Intégration des logs ProFTPd dans Elasticsearch

Voici quelques configuration pour intégrer les logs ProFTPd au sein de l'elasticsearch.

Sur le serveur hébergeant Proftpd, il faut installer logstash et open-jdk :


Sous Debian :

apt-get install openjdk-7-jre

Puis installation de Logstash via les dépôts officiels :

wget -qO - https://packages.elasticsearch.org/GPG-KEY-elasticsearch |  apt-key add - 

echo "deb http://packages.elasticsearch.org/logstash/1.5/debian stable main" | tee -a /etc/apt/sources.list
apt-get update && apt-get install logstash

Préparation de ProFTPd :

Préparons les logs de ProFTPd, à ajouter dans /etc/proftpd/proftpd.conf :

LogFormat default "%h %l %u %t \"%r\" %s %b"
ExtendedLog /var/log/proftpd/proftpd.paranoid_log ALL default

Rajouter la ligne suivante dans /etc/default/proftpd :

export LANG=en_US.UTF-8

Sans cette ligne, le format de date dans les fichiers log est au format FR. Notre pattern Grok ci-dessous parse seulement les dates au format US. Il faudra au préalablement faire un dpkg-reconfigure locales pour intégrer les locales en_US.UTF-8.

Préparation de Logstash :

La configuration de Logstash à déposer dans /etc/logstash/conf.d/logstash.conf :

input {
file {
type => "proftpd-common"
path => "/var/log/proftpd/proftpd.paranoid_log"
}
}
filter {
if [type] == 'proftpd-common' {
grok {
match => [
"message", "%{IPORHOST:clientip} %{WORD:indent} %{USER:ftpuser} \[%{HTTPDATE:timestamp}\] \"%{WORD:command}(?:%{SPACE}%{DATA:request}|%{SPACE})\" (?:%{NUMBER:response}|-) (?:%{NUMBER:bytes}|-)"
]
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
}
}
output {
if [type] == 'proftpd-common' {
elasticsearch {
cluster => "elasticlog"
index => "logstash-proftpd-%{+YYYY.MM.dd}"
}
}
}

Template pour Kibana :

{
"title": "Proftpd access",
"services": {
"query": {
"list": {
"3": {
"id": 3,
"color": "#CCA300",
"alias": "Events",
"pin": true,
"type": "lucene",
"enable": true,
"query": "*"
}
},
"ids": [
3
]
},
"filter": {
"list": {
"0": {
"type": "time",
"field": "@timestamp",
"from": "now-3h",
"to": "now",
"mandate": "must",
"active": true,
"alias": "",
"id": 0
}
},
"ids": [
0
]
}
},
"rows": [
{
"title": "Options",
"height": "250px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"span": 12,
"editable": true,
"type": "histogram",
"loadingEditor": false,
"mode": "count",
"time_field": "@timestamp",
"value_field": null,
"x-axis": true,
"y-axis": true,
"scale": 1,
"y_format": "none",
"grid": {
"max": null,
"min": 0
},
"queries": {
"mode": "selected",
"ids": [
3
]
},
"annotate": {
"enable": false,
"query": "*",
"size": 20,
"field": "_type",
"sort": [
"_score",
"desc"
]
},
"auto_int": true,
"resolution": 100,
"interval": "1m",
"intervals": [
"auto",
"1s",
"1m",
"5m",
"10m",
"30m",
"1h",
"3h",
"12h",
"1d",
"1w",
"1y"
],
"lines": false,
"fill": 0,
"linewidth": 3,
"points": false,
"pointradius": 5,
"bars": true,
"stack": true,
"spyable": true,
"zoomlinks": true,
"options": true,
"legend": true,
"show_query": true,
"interactive": true,
"legend_counts": true,
"timezone": "browser",
"percentage": false,
"zerofill": true,
"derivative": false,
"tooltip": {
"value_type": "cumulative",
"query_as_alias": true
},
"title": "Events"
}
],
"notice": false
},
{
"title": "Graph",
"height": "250px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 4,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "clientip",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "table",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "selected",
"ids": [
3
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Top sources"
},
{
"error": false,
"span": 4,
"editable": true,
"type": "terms",
"loadingEditor": false,
"field": "ftpuser",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": true,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "pie",
"counter_pos": "above",
"spyable": true,
"queries": {
"mode": "selected",
"ids": [
3
]
},
"tmode": "terms",
"tstat": "total",
"valuefield": "",
"title": "Top users"
},
{
"error": false,
"span": 4,
"editable": true,
"group": [
"default"
],
"type": "terms",
"queries": {
"mode": "selected",
"ids": [
3
]
},
"field": "command",
"exclude": [],
"missing": false,
"other": false,
"size": 10,
"order": "count",
"style": {
"font-size": "10pt"
},
"donut": false,
"tilt": false,
"labels": true,
"arrangement": "horizontal",
"chart": "bar",
"counter_pos": "above",
"spyable": true,
"title": "Command Types",
"tmode": "terms",
"tstat": "count",
"valuefield": ""
}
],
"notice": false
},
{
"title": "Events",
"height": "650px",
"editable": true,
"collapse": false,
"collapsable": true,
"panels": [
{
"error": false,
"span": 12,
"editable": true,
"group": [
"default"
],
"type": "table",
"size": 100,
"pages": 5,
"offset": 0,
"sort": [
"timestamp",
"asc"
],
"style": {
"font-size": "9pt"
},
"overflow": "min-height",
"fields": [
"clientip",
"ftpuser",
"timestamp",
"command",
"request",
"response",
"bytes"
],
"highlight": [],
"sortable": true,
"header": true,
"paging": true,
"spyable": true,
"queries": {
"mode": "selected",
"ids": [
3
]
},
"field_list": true,
"status": "Stable",
"trimFactor": 1000,
"normTimes": true,
"title": "Documents",
"all_fields": false,
"localTime": false,
"timeField": "@timestamp"
}
],
"notice": false
}
],
"editable": true,
"index": {
"interval": "day",
"pattern": "[logstash-proftpd-]YYYY.MM.DD",
"default": "logsta",
"warm_fields": true
},
"style": "dark",
"failover": false,
"panel_hints": true,
"loader": {
"save_gist": false,
"save_elasticsearch": true,
"save_local": true,
"save_default": true,
"save_temp": true,
"save_temp_ttl_enable": true,
"save_temp_ttl": "30d",
"load_gist": true,
"load_elasticsearch": true,
"load_elasticsearch_size": 20,
"load_local": true,
"hide": false
},
"pulldowns": [
{
"type": "query",
"collapse": false,
"notice": false,
"query": "*",
"pinned": true,
"history": [
"*"
],
"remember": 10,
"enable": true
},
{
"type": "filtering",
"collapse": true,
"notice": false,
"enable": true
}
],
"nav": [
{
"type": "timepicker",
"collapse": false,
"notice": false,
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"3h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"timefield": "@timestamp",
"enable": true,
"now": true,
"filter_id": 0
}
],
"refresh": "5m"
}