update conf

Makefile

+.PHONY: run restart log-logstash log-es log-kibana stop
+
+run:
+	@docker-compose up -d
+
+restart:
+	@docker-compose restart
+
+log-logstash:
+	@docker-compose logs --tail=100 -f logstash
+
+log-es:
+	@docker-compose logs --tail=100 -f elasticsearch
+
+log-kibana:
+	@docker-compose logs --tail=100 -f kibana
+
+stop:
+	@docker-compose down

elasticsearch/config/elasticsearch.yml

@@ -4,10 +4,12 @@
 #
 cluster.name: "docker-cluster"
 network.host: 0.0.0.0
+cluster.max_shards_per_node: 30000
 
 ## X-Pack settings
 ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
 #
 xpack.license.self_generated.type: trial
+xpack.monitoring.history.duration: 3d
 xpack.security.enabled: false
 xpack.monitoring.collection.enabled: true

logstash/pipeline/logstash.conf

@@ -12,9 +12,15 @@ input {
 
 filter {
 
-	if ("stat.log" in [log][file][path]) {
+	if "stat.log" in [log][file][path] {
         	drop {}
 	}
+	if "chain" in [log][file][path] {
+		drop {}
+        }
+	if "convert" in [log][file][path] {
+                drop {}
+        }
 	mutate {
 		add_field => { "input-type" => "%{[agent][type]}-%{[input][type]}" }
 		remove_field => ["agent","ecs","tags","input","host"]
@@ -24,6 +30,8 @@ filter {
                 skip_on_invalid_json => true
                 target => "json"
         }
+	
+	# version
 	if "v1" in [log][file][path]{
 		mutate { add_field => { "[service][version]" => "v1" } }
 	}else if "v2" in [log][file][path]{
@@ -35,10 +43,25 @@ filter {
 ##		mutate { add_field => { "[env][name]" => "test" } }
 ##	}
 
-
-	mutate { add_field => { "[service][program]" => "backend" } }
-
-	if "gateway" in [log][file][path]{
+	# program
+	if "backend-micro" in [log][file][path]{
+		mutate { add_field => { "[service][program]" => "backend" } }
+        }else if "supervisor" in [log][file][path]{
+		mutate { add_field => { "[service][program]" => "supervisor" } }
+        }else if "chain" in [log][file][path]{
+                mutate { add_field => { "[service][program]" => "chain" } }
+	}else if "backend" in [log][file][path]{
+                mutate { add_field => { "[service][program]" => "backend" } }
+	}else if "externaldb" in [log][file][path]{
+                mutate { add_field => { "[service][program]" => "externaldb" } }
+	}else {
+		mutate { add_field => { "[service][program]" => "other" } }
+	}
+	
+	# services
+        if "supervisor" in [log][file][path]{
+		mutate { update => { "[service][program]" => "supervisor" } }
+	}else if "gateway" in [log][file][path]{
                 mutate { add_field => { "[service][name]" => "gateway" } }
         }else if "swagger" in [log][file][path]{
                 mutate { add_field => { "[service][name]" => "swagger" } }
@@ -56,6 +79,12 @@ filter {
 		mutate { add_field => { "[service][name]" => "tx" } }
         }else if "role-service" in [log][file][path] {
                 mutate { add_field => { "[service][name]" => "role" } }
+	}else if "shorturl" in [log][file][path] {
+                mutate { add_field => { "[service][name]" => "shorturl" } }
+	}else if "express" in [log][file][path] {
+                mutate { add_field => { "[service][name]" => "express" } }
+	}else if "contract" in [log][file][path] {
+                mutate { add_field => { "[service][name]" => "contract" } }
         }else if "user-service" in [log][file][path] {
 		mutate { add_field => { "[service][name]" => "user" } }
 	}else if "department-service" in [log][file][path] {
@@ -67,7 +96,6 @@ filter {
         }else if "trace-service" in [log][file][path] {
                 mutate { add_field => { "[service][name]" => "trace" } }
 	}else if "external" in [log][file][path] {
-		mutate { update => { "[service][program]" => "exdb" } }
 		if "sync" in [log][file][path] {
 	                mutate { add_field => { "[service][name]" => "sync" } } 
 		}else if "convert" in [log][file][path] {
@@ -76,17 +104,15 @@ filter {
                 	mutate { add_field => { "[service][name]" => "rpc" } }
 		}
 	}else if "chain" in [log][file][path] {
-		mutate { update => { "[service][program]" => "chain" } }
         	mutate { add_field => { "[service][name]" => "chain33" } }
         }else {
-		mutate { update => { "[service][program]" => "others" } }
 		mutate { add_field => { "[service][name]" => "other" } }
 	}
 }
 
 output {
 	elasticsearch{
-		index => "%{[env]}-%{[service][program]}-%{[service][version]}-%{[service][name]}-%{+YYYY.MM}"
+		index => "%{[env]}-%{[service][program]}-%{[service][version]}-%{[service][name]}-%{+YYYY.MM}-000000"
 	        hosts => ["elasticsearch:9200"]
         }
 }