Enable Stack Features (X-Pack) and trial by default (#387)

.travis.yml

@@ -4,6 +4,10 @@ services: docker
 env:
   - DOCKER_COMPOSE_VERSION=1.14.0
 
+before_install:
+  - sudo apt-get update
+  - sudo apt-get install -y expect jq
+
 install:
   # Install Docker Compose
   - curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o "$HOME/bin/docker-compose"
@@ -16,38 +20,38 @@ before_script:
 
   # Pull buildpack image (curl 7.52.0+)
   - docker pull buildpack-deps:artful-curl
-  - shopt -s expand_aliases
-  - alias curl='docker run --rm --net=host buildpack-deps:artful-curl curl'
+
+  # Use built-in users
+  - sed -i 's/\(elasticsearch.username:\) elastic/\1 kibana/g' kibana/config/kibana.yml
+  - sed -i 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' logstash/config/logstash.yml
 
 script:
   # Compose
+  - docker-compose up -d elasticsearch
+  - sleep 20
+  - .travis/elasticsearch-setup-passwords.exp
   - docker-compose up -d
-  - sleep 60
-  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9200/'
-  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:5601/api/status'
-  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9600/_node/pipelines/main?pretty'
-  - echo 'dockerelk' | nc localhost 5000
-  - sleep 2
-  - curl -s -XPOST 'http://localhost:9200/_refresh'
-  - curl -s 'http://localhost:9200/_count?q=message:dockerelk' | egrep '^{"count":1,'
-  - curl -s 'http://localhost:9200/_search?q=message:dockerelk&pretty'
+  - sleep 90
+  - .travis/run-tests.sh
   - docker-compose ps
-  - docker-compose logs
+  - docker-compose logs elasticsearch
+  - docker-compose logs kibana
+  - docker-compose logs logstash
   - docker-compose down -v
 
   # Swarm
   - docker swarm init
   - docker stack deploy -c ./docker-stack.yml elk
+  - docker service scale elk_kibana=0 --detach=false
+  - docker service scale elk_logstash=0 --detach=false
   - sleep 60
-  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9200/'
-  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:5601/api/status'
-  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9600/_node/pipelines/main?pretty'
-  - echo 'dockerelk' | nc localhost 5000
-  - sleep 2
-  - curl -s -XPOST 'http://localhost:9200/_refresh'
-  - curl -s 'http://localhost:9200/_count?q=message:dockerelk' | egrep '^{"count":1,'
-  - curl -s 'http://localhost:9200/_search?q=message:dockerelk&pretty'
+  - .travis/elasticsearch-setup-passwords.exp swarm
+  - docker service scale elk_kibana=1 --detach=false
+  - docker service scale elk_logstash=1 --detach=false
+  - sleep 90
+  - .travis/run-tests.sh
   - docker stack services elk
   - docker service logs elk_elasticsearch
   - docker service logs elk_kibana
   - docker service logs elk_logstash
+  - docker stack rm elk

.travis/elasticsearch-setup-passwords.exp

+#!/usr/bin/expect -f
+
+# List of expected users with dummy password
+set user "(elastic|apm_system|kibana|logstash_system|beats_system|remote_monitoring_user)"
+set password "changeme"
+
+# Find elasticsearch container id
+set MODE [lindex $argv 0]
+if { [string match "swarm" $MODE] } {
+	set cid [exec docker ps -q -f label=com.docker.swarm.service.name=elk_elasticsearch]
+} else {
+	set cid [exec docker ps -q -f label=com.docker.compose.service=elasticsearch]
+}
+
+set cmd "docker exec -it $cid bin/elasticsearch-setup-passwords interactive -s -b"
+
+spawn {*}$cmd
+
+expect {
+	-re "(E|Ree)nter password for \\\[$user\\\]: " {
+		send "$password\r"
+		exp_continue
+	}
+	eof
+}

.travis/run-tests.sh

+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+shopt -s expand_aliases
+alias curl="docker run --rm --net=host buildpack-deps:artful-curl curl -s -w '\n'"
+
+function log {
+	echo -e "\n[+] $1\n"
+}
+
+log 'Waiting for Elasticsearch readiness'
+curl -D- 'http://localhost:9200/' \
+	--retry 10 \
+	--retry-delay 5 \
+	--retry-connrefused \
+	-u elastic:changeme
+
+log 'Waiting for Kibana readiness'
+curl -D- 'http://localhost:5601/api/status' \
+	--retry 10 \
+	--retry-delay 5 \
+	--retry-connrefused \
+	-u kibana:changeme
+
+log 'Waiting for Logstash readiness'
+curl -D- 'http://localhost:9600/_node/pipelines/main?pretty' \
+	--retry 10 \
+	--retry-delay 5 \
+	--retry-connrefused
+
+log 'Creating Logstash index pattern in Kibana'
+source .env
+curl -X POST -D- 'http://localhost:5601/api/saved_objects/index-pattern' \
+	-H 'Content-Type: application/json' \
+	-H "kbn-version: ${ELK_VERSION}" \
+	-u kibana:changeme \
+	-d '{"attributes":{"title":"logstash-*","timeFieldName":"@timestamp"}}'
+
+log 'Searching index pattern via Kibana API'
+response="$(curl 'http://localhost:5601/api/saved_objects/_find?type=index-pattern' -u elastic:changeme)"
+echo $response
+count="$(jq -rn --argjson data "${response}" '$data.total')"
+if [[ $count -ne 1 ]]; then
+	echo "Expected 1 index pattern, got ${count}"
+	exit 1
+fi
+
+log 'Sending message to Logstash TCP input'
+echo 'dockerelk' | nc localhost 5000
+
+sleep 1
+curl -X POST 'http://localhost:9200/_refresh' -u elastic:changeme
+
+log 'Searching message in Elasticsearch'
+response="$(curl 'http://localhost:9200/_count?q=message:dockerelk&pretty' -u elastic:changeme)"
+echo $response
+count="$(jq -rn --argjson data "${response}" '$data.count')"
+if [[ $count -ne 1 ]]; then
+	echo "Expected 1 document, got ${count}"
+	exit 1
+fi

docker-compose.yml

@@ -14,6 +14,7 @@ services:
       - "9300:9300"
     environment:
       ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+      ELASTIC_PASSWORD: changeme
     networks:
       - elk
 
@@ -41,7 +42,7 @@ services:
       args:
         ELK_VERSION: $ELK_VERSION
     volumes:
-      - ./kibana/config/:/usr/share/kibana/config:ro
+      - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro
     ports:
       - "5601:5601"
     networks:

docker-stack.yml

@@ -3,7 +3,7 @@ version: '3.3'
 services:
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.6.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.7.0
     ports:
       - "9200:9200"
       - "9300:9300"
@@ -12,6 +12,7 @@ services:
         target: /usr/share/elasticsearch/config/elasticsearch.yml
     environment:
       ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+      ELASTIC_PASSWORD: changeme
     networks:
       - elk
     deploy:
@@ -19,7 +20,7 @@ services:
       replicas: 1
 
   logstash:
-    image: docker.elastic.co/logstash/logstash-oss:6.6.1
+    image: docker.elastic.co/logstash/logstash:6.7.0
     ports:
       - "5000:5000"
       - "9600:9600"
@@ -37,7 +38,7 @@ services:
       replicas: 1
 
   kibana:
-    image: docker.elastic.co/kibana/kibana-oss:6.6.1
+    image: docker.elastic.co/kibana/kibana:6.7.0
     ports:
       - "5601:5601"
     configs:

elasticsearch/Dockerfile

 ARG ELK_VERSION
 
 # https://github.com/elastic/elasticsearch-docker
-FROM docker.elastic.co/elasticsearch/elasticsearch-oss:${ELK_VERSION}
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
 
 # Add your elasticsearch plugins setup here
 # Example: RUN elasticsearch-plugin install analysis-icu

elasticsearch/config/elasticsearch.yml

 ---
 ## Default Elasticsearch configuration from elasticsearch-docker.
-## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml
+## from https://github.com/elastic/elasticsearch-docker/blob/master/.tedi/template/elasticsearch.yml
 #
 cluster.name: "docker-cluster"
 network.host: 0.0.0.0
@@ -14,3 +14,10 @@ discovery.zen.minimum_master_nodes: 1
 ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
 #
 discovery.type: single-node
+
+## X-Pack settings
+## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
+#
+xpack.license.self_generated.type: trial
+xpack.security.enabled: true
+xpack.monitoring.collection.enabled: true

kibana/Dockerfile

 ARG ELK_VERSION
 
 # https://github.com/elastic/kibana-docker
-FROM docker.elastic.co/kibana/kibana-oss:${ELK_VERSION}
+FROM docker.elastic.co/kibana/kibana:${ELK_VERSION}
 
 # Add your kibana plugins setup here
 # Example: RUN kibana-plugin install <name|url>

kibana/config/kibana.yml

 ---
 ## Default Kibana configuration from kibana-docker.
-## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml
+## https://github.com/elastic/kibana-docker/blob/master/.tedi/template/kibana.yml.j2
 #
 server.name: kibana
 server.host: "0"
-elasticsearch.url: http://elasticsearch:9200
+elasticsearch.hosts: [ "http://elasticsearch:9200" ]
+xpack.monitoring.ui.container.elasticsearch.enabled: true
+
+## X-Pack security credentials
+#
+elasticsearch.username: elastic
+elasticsearch.password: changeme

logstash/Dockerfile

 ARG ELK_VERSION
 
 # https://github.com/elastic/logstash-docker
-FROM docker.elastic.co/logstash/logstash-oss:${ELK_VERSION}
+FROM docker.elastic.co/logstash/logstash:${ELK_VERSION}
 
 # Add your logstash plugins setup here
 # Example: RUN logstash-plugin install logstash-filter-json

logstash/config/logstash.yml

 ---
 ## Default Logstash configuration from logstash-docker.
-## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-oss.yml
+## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-full.yml
 #
 http.host: "0.0.0.0"
-path.config: /usr/share/logstash/pipeline
+xpack.monitoring.elasticsearch.url: http://elasticsearch:9200
+
+## X-Pack security credentials
+#
+xpack.monitoring.elasticsearch.username: elastic
+xpack.monitoring.elasticsearch.password: changeme

logstash/pipeline/logstash.conf

@@ -9,5 +9,7 @@ input {
 output {
 	elasticsearch {
 		hosts => "elasticsearch:9200"
+		user => elastic
+		password => changeme
 	}
 }