Skip to content

D
docker-elk
Commit 5debeb3d authored by 段孔乐's avatar 段孔乐
Browse files
Options

Merge branch 'merge' into 'main'

Merge See merge request !11
parents 706ca087 e72314e2
Show whitespace changes
Inline Side-by-side
Showing with 41 additions and 27 deletions
.gitignore
View file @ 5debeb3d
...@@ -6,3 +6,4 @@ logstash.log ...@@ -6,3 +6,4 @@ logstash.log
.idea .idea
.vscode .vscode
.github .github
.elasticsearch/data/
\ No newline at end of file
docker-compose.yml
View file @ 5debeb3d
...@@ -11,9 +11,7 @@ services: ...@@ -11,9 +11,7 @@ services:
source: ./elasticsearch/config/elasticsearch.yml source: ./elasticsearch/config/elasticsearch.yml
target: /usr/share/elasticsearch/config/elasticsearch.yml target: /usr/share/elasticsearch/config/elasticsearch.yml
read_only: true read_only: true
- type: volume - "./elasticsearch/data:/usr/share/elasticsearch/data"
source: elasticsearch
target: /usr/share/elasticsearch/data
ports: ports:
- "$ES_PORT1:9200" - "$ES_PORT1:9200"
- "$ES_PORT2:9300" - "$ES_PORT2:9300"
...@@ -76,5 +74,6 @@ networks: ...@@ -76,5 +74,6 @@ networks:
elk: elk:
driver: bridge driver: bridge
volumes: #volumes:
elasticsearch: # elasticsearch:
# driver: local
init-kibana.sh
View file @ 5debeb3d
...@@ -10,14 +10,14 @@ function setDefaultIndex() { ...@@ -10,14 +10,14 @@ function setDefaultIndex() {
curl --location --request POST "http://$kibana_host/api/kibana/settings" \ curl --location --request POST "http://$kibana_host/api/kibana/settings" \
--header 'kbn-version: 7.12.0' \ --header 'kbn-version: 7.12.0' \
--header 'Content-Type: application/json' \ --header 'Content-Type: application/json' \
--data-raw "{\"changes\":{\"defaultIndex\":\"$1\"}}" --data "{\"changes\":{\"defaultIndex\":\"$1\"}}"
} }
function addILMPolicy() { function addILMPolicy() {
resp=$( resp=$(
curl -s -XPUT "http://$es_host/_ilm/policy/service-log" \ curl -s -XPUT "http://$es_host/_ilm/policy/service-log" \
--header 'Content-Type: application/json' \ --header 'Content-Type: application/json' \
--data-raw '{"policy":{"phases":{"hot":{"min_age":"0ms","actions":{"set_priority":{"priority":100}}},"delete":{"min_age":"365d","actions":{"delete":{"delete_searchable_snapshot":true}}}}}}' --data '{"policy":{"phases":{"hot":{"min_age":"0ms","actions":{"set_priority":{"priority":100}}},"delete":{"min_age":"90d","actions":{"delete":{"delete_searchable_snapshot":true}}}}}}'
) )
echo "$resp" echo "$resp"
if contain "$resp" '{"acknowledged":true}'; then if contain "$resp" '{"acknowledged":true}'; then
...@@ -31,7 +31,7 @@ function addIndexTemplate() { ...@@ -31,7 +31,7 @@ function addIndexTemplate() {
resp=$( resp=$(
curl -s -XPUT "http://$es_host/_index_template/service-log" \ curl -s -XPUT "http://$es_host/_index_template/service-log" \
--header 'Content-Type: application/json' \ --header 'Content-Type: application/json' \
--data-raw '{"template":{"settings":{"index":{"lifecycle":{"name":"service-log"}}},"mappings":{"_source":{"excludes":[],"includes":[],"enabled":true},"_routing":{"required":false},"dynamic":true,"numeric_detection":false,"date_detection":true,"dynamic_date_formats":["strict_date_optional_time","yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"],"dynamic_templates":[]}},"index_patterns":["*-service-*"]}' --data '{"priority":0,"template":{"settings":{"index":{"lifecycle":{"name":"service-log"},"number_of_shards":"1","number_of_replicas":"0"}},"mappings":{"_routing":{"required":false},"numeric_detection":false,"dynamic_date_formats":["strict_date_optional_time","yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"],"_source":{"excludes":[],"includes":[],"enabled":true},"dynamic":true,"dynamic_templates":[],"date_detection":true}},"index_patterns":["*-service-*","*backend*","*chian*","*exdb*","*external*","*other*","*supervisor*","*tanxin*"],"composed_of":[]}'
) )
echo "$resp" echo "$resp"
if contain "$resp" '{"acknowledged":true}'; then if contain "$resp" '{"acknowledged":true}'; then
...@@ -42,7 +42,7 @@ function addIndexTemplate() { ...@@ -42,7 +42,7 @@ function addIndexTemplate() {
} }
function uploadConfig() { function uploadConfig() {
resp=$(curl -s -XPOST "$kibana_host/api/saved_objects/_import?overwrite=true" --form "file=@kibana/config.ndjson" --header 'kbn-xsrf: true') resp=$(curl -s -XPOST "$kibana_host/api/saved_objects/_import?overwrite=true" --form "file=@kibana/export.ndjson" --header 'kbn-xsrf: true')
echo "$resp" echo "$resp"
if contain "$resp" 'success'; then if contain "$resp" 'success'; then
green "config upload succeed" green "config upload succeed"
...@@ -51,16 +51,6 @@ function uploadConfig() { ...@@ -51,16 +51,6 @@ function uploadConfig() {
fi fi
} }
function uploadIndexPattern() {
resp=$(curl -s -XPOST "$kibana_host/api/saved_objects/_import?overwrite=true" --form "file=@kibana/index-patterns.ndjson" --header 'kbn-xsrf: true')
echo "$resp"
if contain "$resp" 'success'; then
green "index-patterns upload succeed"
else
red "index-patterns upload failed"
fi
}
function notHealth() { function notHealth() {
resp=$(curl -sIL -m 5 -w "%{http_code}" -o /dev/null -XGET "$kibana_host/api/features" --header 'kbn-xsrf: true') resp=$(curl -sIL -m 5 -w "%{http_code}" -o /dev/null -XGET "$kibana_host/api/features" --header 'kbn-xsrf: true')
if [ "$resp" == 200 ]; then if [ "$resp" == 200 ]; then
...@@ -97,7 +87,6 @@ green "=======================================" ...@@ -97,7 +87,6 @@ green "======================================="
addILMPolicy addILMPolicy
addIndexTemplate addIndexTemplate
uploadConfig uploadConfig
# uploadIndexPattern
green "" green ""
green "=========================" green "========================="
green "Initialization completed." green "Initialization completed."
......
kibana/export.ndjson 0 → 100644
View file @ 5debeb3d
{"attributes":{"fieldAttrs":"{\"message\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*gateway*"},"coreMigrationVersion":"7.12.0","id":"08f1a5f0-360a-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,7],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNTgsNl0="}
{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*test*backend*"},"coreMigrationVersion":"7.12.0","id":"0bedebd0-ef86-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,8],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNTksNl0="}
{"attributes":{"fieldAttrs":"{\"json.level\":{\"count\":1},\"service.name\":{\"count\":2},\"json.content\":{\"count\":1},\"json.content.keyword\":{\"count\":3},\"message\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*tanxin*mingzong*oa*"},"coreMigrationVersion":"7.12.0","id":"0f1c07a0-360e-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,9],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjAsNl0="}
{"attributes":{"fieldAttrs":"{\"json.span\":{\"count\":2},\"json.level\":{\"count\":2},\"@timestamp\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*production*backend*"},"coreMigrationVersion":"7.12.0","id":"189c04b0-ef8c-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,10],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjEsNl0="}
{"attributes":{"fieldAttrs":"{\"@timestamp\":{\"count\":6},\"message\":{\"count\":6}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*test*"},"coreMigrationVersion":"7.12.0","id":"2662e5c0-90a2-11ec-944f-3b9c0870026c","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1647487706520,59],"type":"index-pattern","updated_at":"2022-03-17T03:28:26.520Z","version":"WzYzMTIsNl0="}
{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*demo*externaldb*"},"coreMigrationVersion":"7.12.0","id":"26830990-2cd9-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,11],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjIsNl0="}
{"attributes":{"fieldAttrs":"{\"json.level\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*demo*"},"coreMigrationVersion":"7.12.0","id":"2a4c18f0-00ba-11ec-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645181955089,32],"type":"index-pattern","updated_at":"2022-02-18T10:59:15.089Z","version":"WzYyODUsNl0="}
{"attributes":{"fieldAttrs":"{\"json.level\":{\"count\":1},\"message\":{\"count\":2}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*tanxin*mingzong*talk*"},"coreMigrationVersion":"7.12.0","id":"32d38dd0-360e-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,12],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjMsNl0="}
{"attributes":{"fieldAttrs":"{\"json.content\":{\"count\":1},\"json.span\":{\"count\":1},\"message\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*demo*backend*"},"coreMigrationVersion":"7.12.0","id":"3aaf5950-00ba-11ec-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,13],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjQsNl0="}
{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"timestamp","title":".monitoring-es*"},"coreMigrationVersion":"7.12.0","id":"3bd9a380-1208-11ec-919b-13636c2c5db1","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,14],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjUsNl0="}
{"attributes":{"fieldAttrs":"{\"log.file.path\":{\"count\":2},\"log.offset\":{\"count\":1},\"service.name\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*prod*person*"},"coreMigrationVersion":"7.12.0","id":"3df0eb70-30ac-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1647502604053,62],"type":"index-pattern","updated_at":"2022-03-17T07:36:44.053Z","version":"WzYzMTUsNl0="}
{"attributes":{"fieldAttrs":"{\"message\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*production*externaldb*"},"coreMigrationVersion":"7.12.0","id":"48d00340-2cd9-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,15],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjYsNl0="}
{"attributes":{"fieldAttrs":"{\"service.name\":{\"count\":2},\"log.file.path\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*prod*person*backend*"},"coreMigrationVersion":"7.12.0","id":"52e17c20-30ac-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1647485280941,57],"type":"index-pattern","updated_at":"2022-03-17T02:48:00.941Z","version":"WzYzMTAsNl0="}
{"attributes":{"fieldAttrs":"{\"json.span\":{\"count\":4},\"json.level\":{\"count\":3},\"message\":{\"count\":8},\"log.file.path\":{\"count\":3},\"json.trace\":{\"count\":1},\"json.content\":{\"count\":7},\"service.name\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*test*backend*v2*"},"coreMigrationVersion":"7.12.0","id":"533e9ac0-ef86-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,16],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjcsNl0="}
{"attributes":{"fieldAttrs":"{\"env\":{\"count\":4},\"log.file.path\":{\"count\":7}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*other*"},"coreMigrationVersion":"7.12.0","id":"6f91e2e0-ef86-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,17],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjgsNl0="}
{"attributes":{"fieldAttrs":"{\"service.name\":{\"count\":4},\"log.file.path\":{\"count\":2},\"service.program\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*prod*person*externaldb*"},"coreMigrationVersion":"7.12.0","id":"6feb1bf0-30ac-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,18],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjksNl0="}
{"attributes":{"buildNum":39309,"dateFormat":"YYYY-MM-DD HH:mm:ss.SSS","dateFormat:dow":"Monday","dateFormat:tz":"Asia/Shanghai","dateNanosFormat":"YYYY-MM-DD HH:mm:ss.SSSSSSSSS","defaultColumns":["message"],"defaultIndex":"2662e5c0-90a2-11ec-944f-3b9c0870026c"},"coreMigrationVersion":"7.12.0","id":"7.12.0","migrationVersion":{"config":"7.12.0"},"references":[],"sort":[1645178667957,29],"type":"config","updated_at":"2022-02-18T10:04:27.957Z","version":"WzYyODIsNl0="}
{"attributes":{"fieldAttrs":"{\"json.content\":{\"count\":1},\"json.level\":{\"count\":1},\"message\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"json.@timestamp","title":"*test*backend*v1*"},"coreMigrationVersion":"7.12.0","id":"844b6210-21db-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,19],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNzAsNl0="}
{"attributes":{"fieldAttrs":"{\"message\":{\"count\":2},\"log.file.path\":{\"count\":1},\"@timestamp\":{\"count\":1},\"service.name\":{\"count\":7},\"service.program\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*production*"},"coreMigrationVersion":"7.12.0","id":"de035020-ef85-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1646642618955,45],"type":"index-pattern","updated_at":"2022-03-07T08:43:38.955Z","version":"WzYyOTgsNl0="}
{"exportedCount":19,"missingRefCount":0,"missingReferences":[]}
\ No newline at end of file
logstash/pipeline/logstash.conf
View file @ 5debeb3d
...@@ -11,22 +11,25 @@ input { ...@@ -11,22 +11,25 @@ input {
## Add your filters / logstash plugins configuration here ## Add your filters / logstash plugins configuration here
filter { filter {
if "lvl=dbug msg=dealBlock module=util cur=" in [message]{
drop {}
}
if "stat.log" in [log][file][path] { if "stat.log" in [log][file][path] {
drop {} drop {}
} }
if "error" in[log][file][path] { if "error" in [log][file][path] {
}else if "sync" in [log][file][path]{
}else if "chain" in [log][file][path] { }else if "chain" in [log][file][path] {
drop {} drop {}
}else if "Chain" in [log][file][path] { }else if "Chain" in [log][file][path] {
drop {} drop {}
}else if "convert" in [log][file][path]{ }else if "convert" in [log][file][path]{
if "person" in [env]{ ## if "person" in [env]{
drop{} ##
}else if "demo" in [env] { ## }else if "demo" in [env] {
drop{} ## drop{}
} ## }
} }
if "backend-env-v1" in [log][file][path] { if "backend-env-v1" in [log][file][path] {
drop {} drop {}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment