Skip to content

D
docker-elk
Commit 5debeb3d authored by 段孔乐's avatar 段孔乐
Browse files
Options

Merge branch 'merge' into 'main'

Merge See merge request !11
parents 706ca087 e72314e2
Hide whitespace changes
Inline Side-by-side
Showing with 42 additions and 29 deletions
.gitignore
View file @ 5debeb3d
......@@ -5,4 +5,5 @@ logstash.log
.idea
.vscode
.github
\ No newline at end of file
.github
.elasticsearch/data/
\ No newline at end of file
docker-compose.yml
View file @ 5debeb3d
......@@ -11,9 +11,7 @@ services:
source: ./elasticsearch/config/elasticsearch.yml
target: /usr/share/elasticsearch/config/elasticsearch.yml
read_only: true
- type: volume
source: elasticsearch
target: /usr/share/elasticsearch/data
- "./elasticsearch/data:/usr/share/elasticsearch/data"
ports:
- "$ES_PORT1:9200"
- "$ES_PORT2:9300"
......@@ -76,5 +74,6 @@ networks:
elk:
driver: bridge
volumes:
elasticsearch:
#volumes:
# elasticsearch:
# driver: local
init-kibana.sh
View file @ 5debeb3d
......@@ -10,14 +10,14 @@ function setDefaultIndex() {
curl --location --request POST "http://$kibana_host/api/kibana/settings" \
--header 'kbn-version: 7.12.0' \
--header 'Content-Type: application/json' \
--data-raw "{\"changes\":{\"defaultIndex\":\"$1\"}}"
--data "{\"changes\":{\"defaultIndex\":\"$1\"}}"
}
function addILMPolicy() {
resp=$(
curl -s -XPUT "http://$es_host/_ilm/policy/service-log" \
--header 'Content-Type: application/json' \
--data-raw '{"policy":{"phases":{"hot":{"min_age":"0ms","actions":{"set_priority":{"priority":100}}},"delete":{"min_age":"365d","actions":{"delete":{"delete_searchable_snapshot":true}}}}}}'
--data '{"policy":{"phases":{"hot":{"min_age":"0ms","actions":{"set_priority":{"priority":100}}},"delete":{"min_age":"90d","actions":{"delete":{"delete_searchable_snapshot":true}}}}}}'
)
echo "$resp"
if contain "$resp" '{"acknowledged":true}'; then
......@@ -31,7 +31,7 @@ function addIndexTemplate() {
resp=$(
curl -s -XPUT "http://$es_host/_index_template/service-log" \
--header 'Content-Type: application/json' \
--data-raw '{"template":{"settings":{"index":{"lifecycle":{"name":"service-log"}}},"mappings":{"_source":{"excludes":[],"includes":[],"enabled":true},"_routing":{"required":false},"dynamic":true,"numeric_detection":false,"date_detection":true,"dynamic_date_formats":["strict_date_optional_time","yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"],"dynamic_templates":[]}},"index_patterns":["*-service-*"]}'
--data '{"priority":0,"template":{"settings":{"index":{"lifecycle":{"name":"service-log"},"number_of_shards":"1","number_of_replicas":"0"}},"mappings":{"_routing":{"required":false},"numeric_detection":false,"dynamic_date_formats":["strict_date_optional_time","yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"],"_source":{"excludes":[],"includes":[],"enabled":true},"dynamic":true,"dynamic_templates":[],"date_detection":true}},"index_patterns":["*-service-*","*backend*","*chian*","*exdb*","*external*","*other*","*supervisor*","*tanxin*"],"composed_of":[]}'
)
echo "$resp"
if contain "$resp" '{"acknowledged":true}'; then
......@@ -42,7 +42,7 @@ function addIndexTemplate() {
}
function uploadConfig() {
resp=$(curl -s -XPOST "$kibana_host/api/saved_objects/_import?overwrite=true" --form "file=@kibana/config.ndjson" --header 'kbn-xsrf: true')
resp=$(curl -s -XPOST "$kibana_host/api/saved_objects/_import?overwrite=true" --form "file=@kibana/export.ndjson" --header 'kbn-xsrf: true')
echo "$resp"
if contain "$resp" 'success'; then
green "config upload succeed"
......@@ -51,16 +51,6 @@ function uploadConfig() {
fi
}
function uploadIndexPattern() {
resp=$(curl -s -XPOST "$kibana_host/api/saved_objects/_import?overwrite=true" --form "file=@kibana/index-patterns.ndjson" --header 'kbn-xsrf: true')
echo "$resp"
if contain "$resp" 'success'; then
green "index-patterns upload succeed"
else
red "index-patterns upload failed"
fi
}
function notHealth() {
resp=$(curl -sIL -m 5 -w "%{http_code}" -o /dev/null -XGET "$kibana_host/api/features" --header 'kbn-xsrf: true')
if [ "$resp" == 200 ]; then
......@@ -97,7 +87,6 @@ green "======================================="
addILMPolicy
addIndexTemplate
uploadConfig
# uploadIndexPattern
green ""
green "========================="
green "Initialization completed."
......
kibana/export.ndjson 0 → 100644
View file @ 5debeb3d
{"attributes":{"fieldAttrs":"{\"message\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*gateway*"},"coreMigrationVersion":"7.12.0","id":"08f1a5f0-360a-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,7],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNTgsNl0="}
{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*test*backend*"},"coreMigrationVersion":"7.12.0","id":"0bedebd0-ef86-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,8],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNTksNl0="}
{"attributes":{"fieldAttrs":"{\"json.level\":{\"count\":1},\"service.name\":{\"count\":2},\"json.content\":{\"count\":1},\"json.content.keyword\":{\"count\":3},\"message\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*tanxin*mingzong*oa*"},"coreMigrationVersion":"7.12.0","id":"0f1c07a0-360e-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,9],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjAsNl0="}
{"attributes":{"fieldAttrs":"{\"json.span\":{\"count\":2},\"json.level\":{\"count\":2},\"@timestamp\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*production*backend*"},"coreMigrationVersion":"7.12.0","id":"189c04b0-ef8c-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,10],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjEsNl0="}
{"attributes":{"fieldAttrs":"{\"@timestamp\":{\"count\":6},\"message\":{\"count\":6}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*test*"},"coreMigrationVersion":"7.12.0","id":"2662e5c0-90a2-11ec-944f-3b9c0870026c","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1647487706520,59],"type":"index-pattern","updated_at":"2022-03-17T03:28:26.520Z","version":"WzYzMTIsNl0="}
{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*demo*externaldb*"},"coreMigrationVersion":"7.12.0","id":"26830990-2cd9-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,11],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjIsNl0="}
{"attributes":{"fieldAttrs":"{\"json.level\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*demo*"},"coreMigrationVersion":"7.12.0","id":"2a4c18f0-00ba-11ec-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645181955089,32],"type":"index-pattern","updated_at":"2022-02-18T10:59:15.089Z","version":"WzYyODUsNl0="}
{"attributes":{"fieldAttrs":"{\"json.level\":{\"count\":1},\"message\":{\"count\":2}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*tanxin*mingzong*talk*"},"coreMigrationVersion":"7.12.0","id":"32d38dd0-360e-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,12],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjMsNl0="}
{"attributes":{"fieldAttrs":"{\"json.content\":{\"count\":1},\"json.span\":{\"count\":1},\"message\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*demo*backend*"},"coreMigrationVersion":"7.12.0","id":"3aaf5950-00ba-11ec-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,13],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjQsNl0="}
{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"timestamp","title":".monitoring-es*"},"coreMigrationVersion":"7.12.0","id":"3bd9a380-1208-11ec-919b-13636c2c5db1","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,14],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjUsNl0="}
{"attributes":{"fieldAttrs":"{\"log.file.path\":{\"count\":2},\"log.offset\":{\"count\":1},\"service.name\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*prod*person*"},"coreMigrationVersion":"7.12.0","id":"3df0eb70-30ac-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1647502604053,62],"type":"index-pattern","updated_at":"2022-03-17T07:36:44.053Z","version":"WzYzMTUsNl0="}
{"attributes":{"fieldAttrs":"{\"message\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*production*externaldb*"},"coreMigrationVersion":"7.12.0","id":"48d00340-2cd9-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,15],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjYsNl0="}
{"attributes":{"fieldAttrs":"{\"service.name\":{\"count\":2},\"log.file.path\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*prod*person*backend*"},"coreMigrationVersion":"7.12.0","id":"52e17c20-30ac-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1647485280941,57],"type":"index-pattern","updated_at":"2022-03-17T02:48:00.941Z","version":"WzYzMTAsNl0="}
{"attributes":{"fieldAttrs":"{\"json.span\":{\"count\":4},\"json.level\":{\"count\":3},\"message\":{\"count\":8},\"log.file.path\":{\"count\":3},\"json.trace\":{\"count\":1},\"json.content\":{\"count\":7},\"service.name\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*test*backend*v2*"},"coreMigrationVersion":"7.12.0","id":"533e9ac0-ef86-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,16],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjcsNl0="}
{"attributes":{"fieldAttrs":"{\"env\":{\"count\":4},\"log.file.path\":{\"count\":7}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*other*"},"coreMigrationVersion":"7.12.0","id":"6f91e2e0-ef86-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,17],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjgsNl0="}
{"attributes":{"fieldAttrs":"{\"service.name\":{\"count\":4},\"log.file.path\":{\"count\":2},\"service.program\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*prod*person*externaldb*"},"coreMigrationVersion":"7.12.0","id":"6feb1bf0-30ac-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,18],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNjksNl0="}
{"attributes":{"buildNum":39309,"dateFormat":"YYYY-MM-DD HH:mm:ss.SSS","dateFormat:dow":"Monday","dateFormat:tz":"Asia/Shanghai","dateNanosFormat":"YYYY-MM-DD HH:mm:ss.SSSSSSSSS","defaultColumns":["message"],"defaultIndex":"2662e5c0-90a2-11ec-944f-3b9c0870026c"},"coreMigrationVersion":"7.12.0","id":"7.12.0","migrationVersion":{"config":"7.12.0"},"references":[],"sort":[1645178667957,29],"type":"config","updated_at":"2022-02-18T10:04:27.957Z","version":"WzYyODIsNl0="}
{"attributes":{"fieldAttrs":"{\"json.content\":{\"count\":1},\"json.level\":{\"count\":1},\"message\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"json.@timestamp","title":"*test*backend*v1*"},"coreMigrationVersion":"7.12.0","id":"844b6210-21db-11ec-86c8-c7480f34ef7e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1645176700226,19],"type":"index-pattern","updated_at":"2022-02-18T09:31:40.226Z","version":"WzYyNzAsNl0="}
{"attributes":{"fieldAttrs":"{\"message\":{\"count\":2},\"log.file.path\":{\"count\":1},\"@timestamp\":{\"count\":1},\"service.name\":{\"count\":7},\"service.program\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"*production*"},"coreMigrationVersion":"7.12.0","id":"de035020-ef85-11eb-bdce-75a994460910","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1646642618955,45],"type":"index-pattern","updated_at":"2022-03-07T08:43:38.955Z","version":"WzYyOTgsNl0="}
{"exportedCount":19,"missingRefCount":0,"missingReferences":[]}
\ No newline at end of file
logstash/pipeline/logstash.conf
View file @ 5debeb3d
......@@ -11,22 +11,25 @@ input {
## Add your filters / logstash plugins configuration here
filter {
if "lvl=dbug msg=dealBlock module=util cur=" in [message]{
drop {}
}
if "stat.log" in [log][file][path] {
drop {}
}
if "error" in[log][file][path] {
if "error" in [log][file][path] {
}else if "sync" in [log][file][path]{
}else if "chain" in [log][file][path] {
drop {}
}else if "Chain" in [log][file][path] {
drop {}
}else if "convert" in [log][file][path]{
if "person" in [env]{
drop{}
}else if "demo" in [env] {
drop{}
}
## if "person" in [env]{
##
## }else if "demo" in [env] {
## drop{}
## }
}
if "backend-env-v1" in [log][file][path] {
drop {}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment