lundi 20 juillet 2015

Intégration des logs ProFTPd dans Elasticsearch

Voici quelques configuration pour intégrer les logs ProFTPd au sein de l'elasticsearch.


Sur le serveur hébergeant Proftpd, il faut installer logstash et open-jdk :


Sous Debian :
apt-get install openjdk-7-jre

Puis installation de Logstash via les dépôts officiels :
wget -qO - https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add -
echo "deb http://packages.elasticsearch.org/logstash/1.5/debian stable main" | tee -a /etc/apt/sources.list apt-get update && apt-get install logstash

Préparation de ProFTPd :

Préparons les logs de ProFTPd, à ajouter dans /etc/proftpd/proftpd.conf :
LogFormat default "%h %l %u %t \"%r\" %s %b"
ExtendedLog /var/log/proftpd/proftpd.paranoid_log ALL default

Rajouter la ligne suivante dans /etc/default/proftpd :
export LANG=en_US.UTF-8

Sans cette ligne, le format de date dans les fichiers log est au format FR. Notre pattern Grok ci-dessous parse seulement les dates au format US. Il faudra au préalablement faire un dpkg-reconfigure locales pour intégrer les locales en_US.UTF-8.

Préparation de Logstash :


La configuration de Logstash à déposer dans /etc/logstash/conf.d/logstash.conf :
input {
file {
type => "proftpd-common"
path => "/var/log/proftpd/proftpd.paranoid_log"
}
}
filter {
if [type] == 'proftpd-common' {
grok {
match => [
"message", "%{IPORHOST:clientip} %{WORD:indent} %{USER:ftpuser} \[%{HTTPDATE:timestamp}\] \"%{WORD:command}(?:%{SPACE}%{DATA:request}|%{SPACE})\" (?:%{NUMBER:response}|-) (?:%{NUMBER:bytes}|-)"
]
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
}
}
output {
if [type] == 'proftpd-common' {
elasticsearch {
cluster => "elasticlog"
index => "logstash-proftpd-%{+YYYY.MM.dd}"
}
}
}

Template pour Kibana :
{
"title": "Proftpd access",
"services": {
"query": {
"list": {
"3": {
"id": 3,
"color": "#CCA300",
"alias": "Events",
"pin": true,
"type": "lucene",
"enable": true,
"query": "*"
}
},
"ids": [
3
]
},
"filter": {
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-3h",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
}
},
"ids": [
0
]
}
},
"rows": [
{
"title": "Options",
"height": "250px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"x-axis": true,
"y-axis": true,
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"queries": {
"mode": "selected",
"ids": [
3
]
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"auto_int": true,
"resolution": 100,
"interval": "1m",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1y"
],
"lines": false,
"fill": 0,
"linewidth": 3,
"points": false,
"pointradius": 5,
"bars": true,
"stack": true,
"spyable": true,
"zoomlinks": true,
"options": true,
"legend": true,
"show_query": true,
"interactive": true,
"legend_counts": true,
"timezone": "browser",
"percentage": false,
"zerofill": true,
"derivative": false,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"title": "Events"
}
],
"notice": false
},
{
"title": "Graph",
"height": "250px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 4,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "clientip",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "table",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "selected",
"ids": [
3
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Top sources"
},
{
"error": false,
"span": 4,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "ftpuser",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": true,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "pie",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "selected",
"ids": [
3
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Top users"
},
{
"error": false,
"span": 4,
"editable": true,
"group": [
"default"
],
"type": "terms",
"queries": {
"mode": "selected",
"ids": [
3
]
},
"field": "command",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "bar",
"counter_pos": "above",
"spyable": true,
"title": "Command Types",
"tmode": "terms",
"tstat": "count",
"valuefield": ""
}
],
"notice": false
},
{
"title": "Events",
"height": "650px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"timestamp",
"asc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [
"clientip",
"ftpuser",
"timestamp",
"command",
"request",
"response",
"bytes"
],
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "selected",
"ids": [
3
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 1000,
"normTimes": true,
"title": "Documents",
"all_fields": false,
"localTime": false,
"timeField": "@timestamp"
}
],
"notice": false
}
],
"editable": true,
"index": {
"interval": "day",
"pattern": "[logstash-proftpd-]YYYY.MM.DD",
"default": "logsta",
"warm_fields": true
},
"style": "dark",
"failover": false,
"panel_hints": true,
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"*"
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": true,
"notice": false,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"3h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"enable": true,
"now": true,
"filter_id": 0
}
],
"refresh": "5m"
}

Aucun commentaire:

Publier un commentaire