Support basic Docker Swarm deployment (#262)

.travis.yml

-sudo: required
-
-language: generic
+language: minimal
+services: docker
 
 env:
   - DOCKER_COMPOSE_VERSION=1.14.0
 
 install:
-  # Installing docker
-  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-  - sudo apt-get update
-  - sudo apt-get -y install docker-ce
-
-  # Installing docker-compose
-  - sudo rm /usr/local/bin/docker-compose
-  - curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" > docker-compose
-  - chmod +x docker-compose
-  - sudo mv docker-compose /usr/local/bin
+  # Install Docker Compose
+  - curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o "$HOME/bin/docker-compose"
+  - chmod +x "$HOME/bin/docker-compose"
+  - docker-compose --version
 
 before_script:
-  # Check docker & docker-compose versions
-  - docker --version
-  - docker-compose --version
+  # Build images
+  - docker-compose build
+
+  # Pull buildpack image (curl 7.52.0+)
+  - docker pull buildpack-deps:artful-curl
+  - shopt -s expand_aliases
+  - alias curl='docker run --rm --net=host buildpack-deps:artful-curl curl'
 
 script:
-  - docker-compose build
+  # Compose
   - docker-compose up -d
-
-  # Verifications
-  - sleep 30
+  - sleep 60
+  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9200/'
+  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:5601/api/status'
+  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9600/_node/pipelines/main?pretty'
+  - echo 'dockerelk' | nc localhost 5000
+  - sleep 2
+  - curl -s -XPOST 'http://localhost:9200/_refresh'
+  - curl -s 'http://localhost:9200/_count?q=message:dockerelk' | egrep '^{"count":1,'
+  - curl -s 'http://localhost:9200/_search?q=message:dockerelk&pretty'
+  - docker-compose ps
   - docker-compose logs
-  - curl --retry 10 --retry-delay 5 -D- http://localhost:9200/
-  - curl --retry 10 --retry-delay 5 -D- http://localhost:5601/api/status
+  - docker-compose down -v
+
+  # Swarm
+  - docker swarm init
+  - docker stack deploy -c ./docker-stack.yml elk
+  - sleep 60
+  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9200/'
+  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:5601/api/status'
+  - curl -s --retry 10 --retry-delay 5 --retry-connrefused -D- 'http://localhost:9600/_node/pipelines/main?pretty'
+  - echo 'dockerelk' | nc localhost 5000
+  - sleep 2
+  - curl -s -XPOST 'http://localhost:9200/_refresh'
+  - curl -s 'http://localhost:9200/_count?q=message:dockerelk' | egrep '^{"count":1,'
+  - curl -s 'http://localhost:9200/_search?q=message:dockerelk&pretty'
+  - docker stack services elk
+  - docker service logs elk_elasticsearch
+  - docker service logs elk_kibana
+  - docker service logs elk_logstash

README.md

@@ -43,8 +43,10 @@ Based on the official Docker images from Elastic:
 6. [JVM tuning](#jvm-tuning)
    * [How can I specify the amount of memory used by a service?](#how-can-i-specify-the-amount-of-memory-used-by-a-service)
    * [How can I enable a remote JMX connection to a service?](#how-can-i-enable-a-remote-jmx-connection-to-a-service)
-7. [Updates](#updates)
+7. [Going further](#going-further)
    * [Using a newer stack version](#using-a-newer-stack-version)
+   * [Plugins and integrations](#plugins-and-integrations)
+   * [Docker Swarm](#docker-swarm)
 
 ## Requirements
 
@@ -246,7 +248,7 @@ logstash:
 
 ### How can I enable a remote JMX connection to a service?
 
-As for the Java Heap memory (see above), you can specify JVM options to enable JMX and map the JMX port on the docker
+As for the Java Heap memory (see above), you can specify JVM options to enable JMX and map the JMX port on the Docker
 host.
 
 Update the `{ES,LS}_JAVA_OPTS` environment variable with the following content (I've mapped the JMX service on the port
@@ -260,7 +262,7 @@ logstash:
     LS_JAVA_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=18080 -Dcom.sun.management.jmxremote.rmi.port=18080 -Djava.rmi.server.hostname=DOCKER_HOST_IP -Dcom.sun.management.jmxremote.local.only=false"
 ```
 
-## Updates
+## Going further
 
 ### Using a newer stack version
 
@@ -274,3 +276,28 @@ $ docker-compose up
 
 **NOTE**: Always pay attention to the [upgrade instructions](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html)
 for each individual component before performing a stack upgrade.
+
+### Plugins and integrations
+
+See the following Wiki pages:
+
+* [External applications](https://github.com/deviantony/docker-elk/wiki/External-applications)
+* [Popular integrations](https://github.com/deviantony/docker-elk/wiki/Popular-integrations)
+
+### Docker Swarm
+
+Experimental support for Docker Swarm is provided in the form of a `docker-stack.yml` file, which can be deployed in an
+existing Swarm cluster using the following command:
+
+```console
+$ docker stack deploy -c docker-stack.yml elk
+```
+
+If all components get deployed without any error, the following command will show 3 running services:
+
+```console
+$ docker stack services elk
+```
+
+**NOTE:** to scale Elasticsearch in Swarm mode, configure *zen* to use the DNS name `tasks.elasticsearch` instead of
+`elasticsearch`.

docker-compose.yml

@@ -27,6 +27,7 @@ services:
       - ./logstash/pipeline:/usr/share/logstash/pipeline:ro
     ports:
       - "5000:5000"
+      - "9600:9600"
     environment:
       LS_JAVA_OPTS: "-Xmx256m -Xms256m"
     networks:

docker-stack.yml

+version: '3.3'
+
+services:
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.4.0
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    configs:
+      - source: elastic_config
+        target: /usr/share/elasticsearch/config/elasticsearch.yml
+    environment:
+      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+    networks:
+      - elk
+    deploy:
+      mode: replicated
+      replicas: 1
+
+  logstash:
+    image: docker.elastic.co/logstash/logstash-oss:6.4.0
+    ports:
+      - "5000:5000"
+      - "9600:9600"
+    configs:
+      - source: logstash_config
+        target: /usr/share/logstash/config/logstash.yml
+      - source: logstash_pipeline
+        target: /usr/share/logstash/pipeline/logstash.conf
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+    networks:
+      - elk
+    deploy:
+      mode: replicated
+      replicas: 1
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana-oss:6.4.0
+    ports:
+      - "5601:5601"
+    configs:
+      - source: kibana_config
+        target: /usr/share/kibana/config/kibana.yml
+    networks:
+      - elk
+    deploy:
+      mode: replicated
+      replicas: 1
+
+configs:
+
+  elastic_config:
+    file: ./elasticsearch/config/elasticsearch.yml
+  logstash_config:
+    file: ./logstash/config/logstash.yml
+  logstash_pipeline:
+    file: ./logstash/pipeline/logstash.conf
+  kibana_config:
+    file: ./kibana/config/kibana.yml
+
+networks:
+  elk:
+    driver: overlay