Merge branch 'main' into 'test-server'

.env

 ELK_VERSION=7.12.0
+ES_PORT1=19200
+ES_PORT2=19300
+LOGSTASH_PORT1=15044
+LOGSTASH_PORT2=15000
+LOGSTASH_PORT3=15000
+LOGSTASH_PORT4=19600
+KIBANA_PORT=15601
\ No newline at end of file

.idea/.gitignore

+# Default ignored files
+/shelf/
+/workspace.xml
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/

.idea/docker-elk.iml

+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="Go" enabled="true" />
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
\ No newline at end of file

.idea/modules.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/docker-elk.iml" filepath="$PROJECT_DIR$/.idea/docker-elk.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file

.idea/vcs.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file

docker-compose.yml

@@ -15,14 +15,15 @@ services:
         source: elasticsearch
         target: /usr/share/elasticsearch/data
     ports:
-      - "19200:9200"
-      - "19300:9300"
+      - "$ES_PORT1:9200"
+      - "$ES_PORT2:9300"
     environment:
       ES_JAVA_OPTS: "-Xmx1024m -Xms1024m"
       ELASTIC_PASSWORD: changeme
       # Use single node discovery in order to disable production mode and avoid bootstrap checks.
       # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
       discovery.type: single-node
+    restart: always
     networks:
       - elk
 
@@ -41,12 +42,13 @@ services:
         target: /usr/share/logstash/pipeline
         read_only: true
     ports:
-      - "15044:5044"
-      - "15000:5000/tcp"
-      - "15000:5000/udp"
-      - "19600:9600"
+      - "$LOGSTASH_PORT1:5044"
+      - "$LOGSTASH_PORT2:5000/tcp"
+      - "$LOGSTASH_PORT3:5000/udp"
+      - "$LOGSTASH_PORT4:9600"
     environment:
       LS_JAVA_OPTS: "-Xmx512m -Xms512m"
+    restart: always
     networks:
       - elk
     depends_on:
@@ -63,7 +65,8 @@ services:
         target: /usr/share/kibana/config/kibana.yml
         read_only: true
     ports:
-      - "15601:5601"
+      - "$KIBANA_PORT:5601"
+    restart: always
     networks:
       - elk
     depends_on:

init-kibana.sh

+#!/bin/bash
+
+. util.sh
+. .env
+
+kibana_host="localhost:$KIBANA_PORT"
+es_host="localhost:$ES_PORT1"
+
+function setDefaultIndex() {
+  curl --location --request POST "http://$kibana_host/api/kibana/settings" \
+    --header 'kbn-version: 7.12.0' \
+    --header 'Content-Type: application/json' \
+    --data-raw "{\"changes\":{\"defaultIndex\":\"$1\"}}"
+}
+
+function addILMPolicy() {
+  resp=$(
+    curl -s -XPUT "http://$es_host/_ilm/policy/service-log" \
+      --header 'Content-Type: application/json' \
+      --data-raw '{"policy":{"phases":{"hot":{"min_age":"0ms","actions":{"set_priority":{"priority":100}}},"delete":{"min_age":"365d","actions":{"delete":{"delete_searchable_snapshot":true}}}}}}'
+  )
+  echo "$resp"
+  if contain "$resp" '{"acknowledged":true}'; then
+    green "index life cycle import success"
+  else
+    red "index life cycle import failed"
+  fi
+}
+
+function addIndexTemplate() {
+  resp=$(
+    curl -s -XPUT "http://$es_host/_index_template/service-log" \
+      --header 'Content-Type: application/json' \
+      --data-raw '{"template":{"settings":{"index":{"lifecycle":{"name":"service-log"}}},"mappings":{"_source":{"excludes":[],"includes":[],"enabled":true},"_routing":{"required":false},"dynamic":true,"numeric_detection":false,"date_detection":true,"dynamic_date_formats":["strict_date_optional_time","yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"],"dynamic_templates":[]}},"index_patterns":["*-service-*"]}'
+  )
+  echo "$resp"
+  if contain "$resp" '{"acknowledged":true}'; then
+    green "index template import success"
+  else
+    red "index template import failed"
+  fi
+}
+
+function uploadConfig() {
+  resp=$(curl -s -XPOST "$kibana_host/api/saved_objects/_import?overwrite=true" --form "file=@kibana/export.ndjson" --header 'kbn-xsrf: true')
+  echo "$resp"
+  if contain "$resp" 'success'; then
+    green "config upload succeed"
+  else
+    red "config upload failed"
+  fi
+}
+
+function notHealth() {
+  resp=$(curl -sIL -m 5 -w "%{http_code}" -o /dev/null -XGET "$kibana_host/api/features" --header 'kbn-xsrf: true')
+  if [ "$resp" == 200 ]; then
+    return 1
+  else
+    return 0
+  fi
+}
+
+function checkELKRunning(){
+  if [ "$(docker ps | grep -c 'docker-elk' )" -lt 1 ]; then
+    red "==============="
+    red "ELK not running"
+    red "==============="
+    exit 1
+  fi
+}
+
+checkELKRunning
+start=$(date +%s)
+while notHealth; do
+  end=$(date +%s)
+  take=$((100 - (end - start)  ))
+  red "Kibana not ready yet, waiting... time: $take"
+  if [ "$take" -lt 0 ]; then
+      red "time out, please check ELK status"
+      exit 1
+  fi
+  sleep 1s
+done
+green "======================================="
+green "Kibana is ready. Initialization begins."
+green "======================================="
+addILMPolicy
+addIndexTemplate
+uploadConfig
+green ""
+green "========================="
+green "Initialization completed."
+green "========================="
+blue "Visit the following website to discover data"
+blue "http://$kibana_host/app/discover"
+green "End"

kibana/export.ndjson

+{"attributes":{"buildNum":39309,"dateFormat":"YYYY-MM-DD HH:mm:ss.SSS","dateFormat:dow":"Monday","dateFormat:tz":"Asia/Shanghai","dateNanosFormat":"YYYY-MM-DD HH:mm:ss.SSSSSSSSS","defaultColumns":["message"],"defaultIndex":"3dd09b40-b302-11eb-8677-fd4740a7c42a"},"coreMigrationVersion":"7.12.0","id":"7.12.0","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-07-02T05:10:07.453Z","version":"Wzc3NjUxLDExXQ=="}
+{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"role-service*"},"coreMigrationVersion":"7.12.0","id":"a26fc570-b6d7-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-05-17T06:18:02.439Z","version":"WzYwMjE4LDExXQ=="}
+{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"notify-service-v1*"},"coreMigrationVersion":"7.12.0","id":"cf0150c0-bf9e-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-05-28T10:23:56.364Z","version":"WzY0Mjc1LDExXQ=="}
+{"attributes":{"fieldAttrs":"{\"service.version\":{\"count\":2}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"others*"},"coreMigrationVersion":"7.12.0","id":"cb829560-b3d9-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-05-31T02:53:05.046Z","version":"WzcxMjI4LDExXQ=="}
+{"attributes":{"fieldAttrs":"{\"json.content\":{\"count\":13},\"json.duration\":{\"count\":7},\"json.level\":{\"count\":5},\"json.span\":{\"count\":4},\"json.trace\":{\"count\":4},\"log.file.path\":{\"count\":4},\"message\":{\"count\":10},\"service.name\":{\"count\":1},\"service.version\":{\"count\":2}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*-service*"},"coreMigrationVersion":"7.12.0","id":"3dd09b40-b302-11eb-8677-fd4740a7c42a","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-06-22T02:40:13.017Z","version":"Wzc1ODg4LDExXQ=="}
+{"attributes":{"fieldAttrs":"{\"json.content\":{\"count\":2},\"message\":{\"count\":11},\"_id\":{\"count\":8},\"_type\":{\"count\":1},\"json.level\":{\"count\":6}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"trace-service-v2*"},"coreMigrationVersion":"7.12.0","id":"d2394ae0-c1ba-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-06-30T07:33:22.471Z","version":"Wzc2OTk3LDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"auth-service-v1*"},"coreMigrationVersion":"7.12.0","id":"66aa7500-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:20.432Z","version":"Wzc3NTQ2LDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"*-service-v1*"},"coreMigrationVersion":"7.12.0","id":"65763ac0-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:18.412Z","version":"Wzc3NTQyLDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"*-service-v2*"},"coreMigrationVersion":"7.12.0","id":"661030d0-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:19.421Z","version":"Wzc3NTQ0LDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"department-service-v2*"},"coreMigrationVersion":"7.12.0","id":"67441cf0-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:21.439Z","version":"Wzc3NTQ4LDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"gateway-service-v2*"},"coreMigrationVersion":"7.12.0","id":"69105170-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:24.455Z","version":"Wzc3NTU0LDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"gateway-service-v1*"},"coreMigrationVersion":"7.12.0","id":"687c9cf0-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:23.487Z","version":"Wzc3NTUyLDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"process-service-v2*"},"coreMigrationVersion":"7.12.0","id":"6c104b00-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:29.489Z","version":"Wzc3NTY0LDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"file-service-v1*"},"coreMigrationVersion":"7.12.0","id":"67dc8c60-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:22.438Z","version":"Wzc3NTUwLDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"pay-service-v1*"},"coreMigrationVersion":"7.12.0","id":"6b7606d0-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:28.477Z","version":"Wzc3NTYyLDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"proof-service-v1*"},"coreMigrationVersion":"7.12.0","id":"6ca9cbe0-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:30.494Z","version":"Wzc3NTY2LDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"swagger-service-v2*"},"coreMigrationVersion":"7.12.0","id":"6faef590-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:35.561Z","version":"Wzc3NTc0LDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"template-service-v2*"},"coreMigrationVersion":"7.12.0","id":"704198a0-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:36.522Z","version":"Wzc3NTc2LDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"swagger-service-v1*"},"coreMigrationVersion":"7.12.0","id":"6f14ff80-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:34.552Z","version":"Wzc3NTcyLDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"tx-service-v1*"},"coreMigrationVersion":"7.12.0","id":"7173d710-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:38.529Z","version":"Wzc3NTgwLDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"user-service-v1*"},"coreMigrationVersion":"7.12.0","id":"720e4250-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:39.541Z","version":"Wzc3NTgyLDExXQ=="}
+{"attributes":{"timeFieldName":"@timestamp","title":"role-service-v2*"},"coreMigrationVersion":"7.12.0","id":"6e7e8be0-daf2-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"type":"index-pattern","updated_at":"2021-07-02T05:00:33.566Z","version":"Wzc3NTcwLDExXQ=="}
+{"attributes":{"columns":["json.content","json.duration","json.level","json.span","json.trace"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"exists\",\"key\":\"json.content\",\"value\":\"exists\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"exists\":{\"field\":\"json.content\"},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"json","version":1},"coreMigrationVersion":"7.12.0","id":"df93aa50-b30f-11eb-8677-fd4740a7c42a","migrationVersion":{"search":"7.9.3"},"references":[{"id":"3dd09b40-b302-11eb-8677-fd4740a7c42a","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"3dd09b40-b302-11eb-8677-fd4740a7c42a","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"search","updated_at":"2021-05-12T10:50:32.181Z","version":"WzU1NTYxLDExXQ=="}
+{"exportedCount":23,"missingRefCount":0,"missingReferences":[]}
\ No newline at end of file

logstash/pipeline/logstash.conf

 input {
-	beats {
-		port => 5044
-	}
+    beats {
+        port => 5044
+    }
 
-	tcp {
-		port => 5000
-	}
+    tcp {
+        port => 5000
+    }
 }
 
 ## Add your filters / logstash plugins configuration here
 
 filter {
 
-	if ("stat.log" in [log][file][path]) {
-        	drop {}
-	}
-	mutate {
-		add_field => { "input-type" => "%{[agent][type]}-%{[input][type]}" }
-		remove_field => ["agent","ecs","tags","input","host"]
-	}
+    if ("stat.log" in [log][file][path]) {
+        drop {}
+    }
+    mutate {
+        add_field => { "input-type" => "%{[agent][type]}-%{[input][type]}" }
+        remove_field => ["agent","ecs","tags","input","host"]
+    }
         json{
                 source => "message"
                 skip_on_invalid_json => true
                 target => "json"
         }
-	if "v1" in [log][file][path]{
-		mutate { add_field => { "[service][version]" => "v1" } }
-	}else if "v2" in [log][file][path]{
+        if "v1" in [log][file][path]{
+                mutate { add_field => { "[service][version]" => "v1" } }
+        }else if "v2" in [log][file][path]{
                 mutate { add_field => { "[service][version]" => "v2" } }
 	}else {
 		mutate { add_field => { "[service][version]" => "v" } }

start-and-init.sh

+#!/bin/bash
+
+. start.sh
+. init-kibana.sh
\ No newline at end of file

start.sh

+#!/bin/bash
+
+. ./util.sh
+. ./.env
+
+function checkPort(){
+  if [ "$(sudo netstat -nlput | grep -c $1)" -gt 0 ]; then
+    rad "==========================================================="
+    rad "The port $1 is already in use, change it in file \".env\"."
+    rad "==========================================================="
+    exit
+  fi
+}
+
+function checkAllPort() {
+  checkPort "$ES_PORT1"
+  checkPort "$ES_PORT2"
+  checkPort "$LOGSTASH_PORT1"
+  checkPort "$LOGSTASH_PORT2"
+  checkPort "$LOGSTASH_PORT3"
+  checkPort "$LOGSTASH_PORT4"
+  checkPort "$KIBANA_PORT"
+}
+
+function start(){
+  if docker-compose up -d; then
+    green "==============="
+    green "start succeed"
+    green "==============="
+  else
+    red "==============="
+    red "start failed"
+    red "==============="
+    exit 1
+  fi
+}
+
+function checkPS(){
+  if [ "$(docker ps | grep -c 'docker-elk' )" -gt 0 ]; then
+    red "======================="
+    red "ELK are already started"
+    red "======================="
+    exit
+  fi
+}
+
+checkPS
+checkAllPort
+start
\ No newline at end of file

util.sh

+#!/bin/bash
+blue(){
+    echo -e "\033[34m\033[01m$1\033[0m"
+}
+green(){
+    echo -e "\033[32m\033[01m$1\033[0m"
+}
+red(){
+    echo -e "\033[31m\033[01m$1\033[0m"
+}
+
+if [[ -f /etc/redhat-release ]]; then
+    release="centos"
+    systemPackage="yum"
+    systempwd="/usr/lib/systemd/system/"
+elif cat /etc/issue | grep -Eqi "debian"; then
+    release="debian"
+    systemPackage="apt-get"
+    systempwd="/lib/systemd/system/"
+elif cat /etc/issue | grep -Eqi "ubuntu"; then
+    release="ubuntu"
+    systemPackage="apt-get"
+    systempwd="/lib/systemd/system/"
+elif cat /etc/issue | grep -Eqi "centos|red hat|redhat"; then
+    release="centos"
+    systemPackage="yum"
+    systempwd="/usr/lib/systemd/system/"
+elif cat /proc/version | grep -Eqi "debian"; then
+    release="debian"
+    systemPackage="apt-get"
+    systempwd="/lib/systemd/system/"
+elif cat /proc/version | grep -Eqi "ubuntu"; then
+    release="ubuntu"
+    systemPackage="apt-get"
+    systempwd="/lib/systemd/system/"
+elif cat /proc/version | grep -Eqi "centos|red hat|redhat"; then
+    release="centos"
+    systemPackage="yum"
+    systempwd="/usr/lib/systemd/system/"
+fi
+
+function contain() {
+  if [ "$(echo "$1" | grep -c "$2")" -gt 0 ]; then return 0; else return 1; fi
+}