Feature: add Logstash-Forwarder as agent

docker-compose.yml

@@ -26,6 +26,15 @@ services:
    - ./logs/apache:/var/log/apache
   links:
    - shipper
+ forwarder:
+  image: apopelo/logstash-forwarder
+  volumes:
+   - ./logstash-forwarder/config:/etc/logstash-forwarder
+   - ./logstash-forwarder/ssl:/etc/ssl
+   - ./logs/random:/var/log/random
+   - ./logs/apache:/var/log/apache
+  links:
+   - shipper
  ####################
  # Logstash shipper #
  ####################
@@ -38,6 +47,7 @@ services:
   volumes:
    - ./logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
    - ./logstash/shipper/pipeline/:/usr/share/logstash/pipeline/
+   - ./logstash/shipper/ssl:/ssl
  ########################
  # Kafka infrastructure #
  ########################

logstash-forwarder/config/config.json

@@ -0,0 +1,17 @@
+{
+    "network": {
+        "servers": [ "shipper:5043" ],
+		"ssl key": "/etc/ssl/selfsigned.key",
+        "ssl ca": "/etc/ssl/selfsigned.crt"
+    },
+    "files": [
+        {
+            "paths":  [ "/var/log/random/*.log" ],
+            "fields": { "type": "random-forwarder" }
+        },
+		{
+            "paths":  [ "/var/log/apache/*.log" ],
+            "fields": { "type": "apache-forwarder" }
+        }
+    ]
+}

logstash-forwarder/ssl/selfsigned.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDEjCCAfqgAwIBAgIQPD0a5WB48xwmuFt+FoT1KDANBgkqhkiG9w0BAQsFADAo
+MRQwEgYDVQQKEwtMb2cgQ291cmllcjEQMA4GA1UEAxMHc2hpcHBlcjAeFw0xNzAz
+MTcxNDE1NDNaFw0yNzAzMTUxNDE1NDNaMCgxFDASBgNVBAoTC0xvZyBDb3VyaWVy
+MRAwDgYDVQQDEwdzaGlwcGVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZWyklfBCGkwTJn
+8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybujUyQWz63irzb
+fWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlWz7mhBXsxFy09
+W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzbbH8kSer9/qIN
+Qx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUdROElKiO4hWou
+Wty046pNWYhum49+zM8r0Q9fSQIDAQABozgwNjAOBgNVHQ8BAf8EBAMCAqQwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsF
+AAOCAQEAO3KmeN/KOFm4OCPKmd5xbZ5I6d/A4V+5lscrFG9tqmF+Ax/hA1RUrnSv
+DPk4YSv8zW8tGrZpX3iLGGX7giqT6EcKH+leyyU0pvO2oUvs+R0cHu4kY7XknUHm
+Lx3QovDap4+uZIZdawI23fGyKSa7PUybKc1fuxrwZNns3zVc+Lp+iSAfsPWI43kX
+ZR0WlMzXIr8EVef+Hz2xLdDsuiwUVrBSWH+tn3pedwPjeEO/YgH+CV899hWlDBl+
+1r8+S11s+Ur+IMuydygLds8QDgLJlFmJVZmvifs+fEd6jPu34CsvA6tF0YqW03iw
+Nmnzx5bMkV9Iffg/rdpYapPIV877DQ==
+-----END CERTIFICATE-----

logstash-forwarder/ssl/selfsigned.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZW
+yklfBCGkwTJn8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybu
+jUyQWz63irzbfWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlW
+z7mhBXsxFy09W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzb
+bH8kSer9/qINQx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUd
+ROElKiO4hWouWty046pNWYhum49+zM8r0Q9fSQIDAQABAoIBAF3BsyHONuA5XjYN
+e4o2D2UwnFLlzbWywAuUW3WyXrD2dMtQfrtQKjDQUPOixNUGErv90I78rGQEi/aK
+GruTwM+O3X4oWSs9zNGdXG8Jgn+x9+hhHHC/XXMGkEcUoHLkH3J5kdhqoI2+dJWs
+bMiXYOd78cYXcaeUM2x+3WuWSjO5o6zNDka4RnwFbVk2k2Zay1EsSzjb+9g78Y+C
+V+bWYz/o5JdWue5ZmnlvWLS/l8YmlYFaIiHePcbQUtiLDGpIB9LYn63lijRIsDdJ
+S+zkxLJfnE9h8b3eViv2h32Ysxd0HCz8on3U6WpvtJ1zTSLZM/wps+clfvHAQxUS
+ad1DJBECgYEA9y3cbh62TX9YuDwVNXSpuszAuqSHxXvVo1AQMux8GQnOGyuBwwww
+0Nj6nAUOLdIIPhs5fY88D1UJa6TH3s4lnRFURrYa5ueeE0DRU9fVDJ5nnwcPlaXg
+bLVRtW7oP3xnFYsB3MQtMANQoHXxqHeO0ttwUI7RhydSY8ctLoZ0cI8CgYEA7o8i
+Z5l+9TSQZefqa/eGhOPsEhYnyWfha9R42KOUkPvEu6awgk3UAAdgzY/MN636Gv1O
+shWcJGC6K8/pQhkc3jhRICJUZutkUh1r5zKdOKp/lFyGqHF1r7nwhouPg4PDMot1
+z0bv2Q8XRxKoXcDZWj7pQog3LAUKWbub7z677qcCgYEAycm40HVmInI5/X2fWtp4
+zbTmCfjTllb4G7D66HfC/7XPesLT1mWXXIgmX0SJRMLYU8tp7aUHRQUJAcWuHahx
+cH57LJbx81tW4yThPu9OUFTpBxx+jo6yXkzP/awjEeD/TskCpjXJJg8uvTBMLloc
+UOswD7PoPcvOKQKrUfYWDOUCgYA0rOhU5eDRtfyYc+AgXPSYDVfIGppIb6anz6+r
+bltUtDMZmFHrQ9Im+3oVicX3GXRbV0l0Ky9iHPWezPbdgHwOWXMYXFw0qY9qTlLQ
+Jy3uID+xe8jEEuGAntcMbm6PBJ4qX/7weOJnFFz4dZc1tGqJ8dBBv6AnGSnS7Ikf
+B8QpiwKBgDW+HqJQTwEZMGLHLq3GNlgQqzDQrSRhGMyueeUK1VYPNRpAQQO/KZy/
+7BjjjJswdoF3gZqGBciKYGfqRvojyNOijadJnicuoVNcLqxtZCB4PbFLUfpz00fM
+9AQkK0ejTa2/Xh0hF19SqdaK5iiEmFbrDNxW/EastBquOYPAiC1m
+-----END RSA PRIVATE KEY-----

logstash/indexer/pipeline/kafka_elasticsearch.conf

@@ -2,7 +2,7 @@ input {
 	kafka {
         codec =>  json{}
         bootstrap_servers => "kafka:9092"
-        topics => ["nginx-access", "random", "apache"]
+        topics => ["random", "apache", "random-forwarder", "apache-forwarder"]
         client_id => "logstash_indexer_1"
 	}
 }
@@ -61,6 +61,36 @@ filter {
 			remove_field => [ "timestamp" ]
 		}
 	}
+	if [type] == "random-forwarder" {
+		grok {
+			match => [ "message" , "(?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) %{NUMBER:pid} %{GREEDYDATA:filename} %{NUMBER:line} %{GREEDYDATA:logger} %{LOGLEVEL:severity} %{GREEDYDATA:quote}"]
+			overwrite => [ "message" ]
+		}
+		date {
+			match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS"]
+			remove_field => [ "timestamp" ]
+		}
+	}
+	if [type] == "apache-forwarder" {
+		grok {
+			match => [ "message" , "%{COMBINEDAPACHELOG}"]
+			overwrite => [ "message" ]
+		}
+		mutate {
+			convert => ["response", "integer"]
+			convert => ["bytes", "integer"]
+			convert => ["responsetime", "float"]
+		}
+		geoip {
+			source => "clientip"
+			target => "geoip"
+			add_tag => [ "apache-geoip" ]
+		}
+		date {
+			match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+			remove_field => [ "timestamp" ]
+		}
+	}
 }
 
 output {
@@ -69,23 +99,32 @@ output {
 			hosts => ["elasticsearch:9200"]
 			index => "nginx-%{+YYYYMM}"
         }
-        stdout {
-          codec => rubydebug
-        }
 	}
 	if [type] == "random" {
         elasticsearch {
 			hosts => ["elasticsearch:9200"]
 			index => "random-%{+YYYYMM}"
         }
+	}
+	if [type] == "apache" {
+        elasticsearch {
+			hosts => ["elasticsearch:9200"]
+			index => "apache-%{+YYYYMM}"
+        }
+	}
+	if [type] == "random-forwarder" {
+        elasticsearch {
+			hosts => ["elasticsearch:9200"]
+			index => "randomforwarder-%{+YYYYMM}"
+        }
         stdout {
           codec => rubydebug
         }
 	}
-	if [type] == "apache" {
+	if [type] == "apache-forwarder" {
         elasticsearch {
 			hosts => ["elasticsearch:9200"]
-			index => "apache-%{+YYYYMM}"
+			index => "apacheforwarder-%{+YYYYMM}"
         }
         stdout {
           codec => rubydebug

logstash/shipper/pipeline/beat_kafka.conf

@@ -2,6 +2,11 @@ input {
   beats {
     port => 5044
   }
+  lumberjack {
+	port => 5043
+	ssl_key => "/ssl/selfsigned.key"
+	ssl_certificate => "/ssl/selfsigned.crt"
+  }
 }
 output {
   kafka {
@@ -9,7 +14,9 @@ output {
     bootstrap_servers => "kafka:9092"
 	topic_id => "%{type}"
   }
-  stdout {
-    codec => rubydebug
+  if [type] == "apache-forwarder" || [type] == "random-forwarder" {
+	  stdout {
+		codec => rubydebug
+	  }
   }
 }

logstash/shipper/ssl/selfsigned.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDEjCCAfqgAwIBAgIQPD0a5WB48xwmuFt+FoT1KDANBgkqhkiG9w0BAQsFADAo
+MRQwEgYDVQQKEwtMb2cgQ291cmllcjEQMA4GA1UEAxMHc2hpcHBlcjAeFw0xNzAz
+MTcxNDE1NDNaFw0yNzAzMTUxNDE1NDNaMCgxFDASBgNVBAoTC0xvZyBDb3VyaWVy
+MRAwDgYDVQQDEwdzaGlwcGVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZWyklfBCGkwTJn
+8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybujUyQWz63irzb
+fWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlWz7mhBXsxFy09
+W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzbbH8kSer9/qIN
+Qx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUdROElKiO4hWou
+Wty046pNWYhum49+zM8r0Q9fSQIDAQABozgwNjAOBgNVHQ8BAf8EBAMCAqQwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsF
+AAOCAQEAO3KmeN/KOFm4OCPKmd5xbZ5I6d/A4V+5lscrFG9tqmF+Ax/hA1RUrnSv
+DPk4YSv8zW8tGrZpX3iLGGX7giqT6EcKH+leyyU0pvO2oUvs+R0cHu4kY7XknUHm
+Lx3QovDap4+uZIZdawI23fGyKSa7PUybKc1fuxrwZNns3zVc+Lp+iSAfsPWI43kX
+ZR0WlMzXIr8EVef+Hz2xLdDsuiwUVrBSWH+tn3pedwPjeEO/YgH+CV899hWlDBl+
+1r8+S11s+Ur+IMuydygLds8QDgLJlFmJVZmvifs+fEd6jPu34CsvA6tF0YqW03iw
+Nmnzx5bMkV9Iffg/rdpYapPIV877DQ==
+-----END CERTIFICATE-----

logstash/shipper/ssl/selfsigned.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA5lbWxKD/8lYzglubCldiG95jTSOXFbFc11BpH0OkDsoy6uZW
+yklfBCGkwTJn8uZRK0RVV02I7ndhzNm9pVD/1wAClOtNnRs3doUuwjdeQCX/tybu
+jUyQWz63irzbfWBB70XxgknddKWy4fP4Xu54wNLoaWvTTwRxejTWmrvEDz7PpqlW
+z7mhBXsxFy09W5aQpaeBlFR5Pdg+C7yXuTL2oAENQWVMgHpJZqaYB7Vll/JWaHzb
+bH8kSer9/qINQx9+RWO1SxtuKcpu40NdavT0Km1ZYc3p8i3OOZAJ4pfcB7g7RvUd
+ROElKiO4hWouWty046pNWYhum49+zM8r0Q9fSQIDAQABAoIBAF3BsyHONuA5XjYN
+e4o2D2UwnFLlzbWywAuUW3WyXrD2dMtQfrtQKjDQUPOixNUGErv90I78rGQEi/aK
+GruTwM+O3X4oWSs9zNGdXG8Jgn+x9+hhHHC/XXMGkEcUoHLkH3J5kdhqoI2+dJWs
+bMiXYOd78cYXcaeUM2x+3WuWSjO5o6zNDka4RnwFbVk2k2Zay1EsSzjb+9g78Y+C
+V+bWYz/o5JdWue5ZmnlvWLS/l8YmlYFaIiHePcbQUtiLDGpIB9LYn63lijRIsDdJ
+S+zkxLJfnE9h8b3eViv2h32Ysxd0HCz8on3U6WpvtJ1zTSLZM/wps+clfvHAQxUS
+ad1DJBECgYEA9y3cbh62TX9YuDwVNXSpuszAuqSHxXvVo1AQMux8GQnOGyuBwwww
+0Nj6nAUOLdIIPhs5fY88D1UJa6TH3s4lnRFURrYa5ueeE0DRU9fVDJ5nnwcPlaXg
+bLVRtW7oP3xnFYsB3MQtMANQoHXxqHeO0ttwUI7RhydSY8ctLoZ0cI8CgYEA7o8i
+Z5l+9TSQZefqa/eGhOPsEhYnyWfha9R42KOUkPvEu6awgk3UAAdgzY/MN636Gv1O
+shWcJGC6K8/pQhkc3jhRICJUZutkUh1r5zKdOKp/lFyGqHF1r7nwhouPg4PDMot1
+z0bv2Q8XRxKoXcDZWj7pQog3LAUKWbub7z677qcCgYEAycm40HVmInI5/X2fWtp4
+zbTmCfjTllb4G7D66HfC/7XPesLT1mWXXIgmX0SJRMLYU8tp7aUHRQUJAcWuHahx
+cH57LJbx81tW4yThPu9OUFTpBxx+jo6yXkzP/awjEeD/TskCpjXJJg8uvTBMLloc
+UOswD7PoPcvOKQKrUfYWDOUCgYA0rOhU5eDRtfyYc+AgXPSYDVfIGppIb6anz6+r
+bltUtDMZmFHrQ9Im+3oVicX3GXRbV0l0Ky9iHPWezPbdgHwOWXMYXFw0qY9qTlLQ
+Jy3uID+xe8jEEuGAntcMbm6PBJ4qX/7weOJnFFz4dZc1tGqJ8dBBv6AnGSnS7Ikf
+B8QpiwKBgDW+HqJQTwEZMGLHLq3GNlgQqzDQrSRhGMyueeUK1VYPNRpAQQO/KZy/
+7BjjjJswdoF3gZqGBciKYGfqRvojyNOijadJnicuoVNcLqxtZCB4PbFLUfpz00fM
+9AQkK0ejTa2/Xh0hF19SqdaK5iiEmFbrDNxW/EastBquOYPAiC1m
+-----END RSA PRIVATE KEY-----