Add App Search extension (#471)

.travis.yml

@@ -22,6 +22,7 @@ before_script:
   - sed -i -e 's/\(elasticsearch.username:\) elastic/\1 kibana/g' -e 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' kibana/config/kibana.yml
   - sed -i -e 's/\(xpack.monitoring.elasticsearch.username:\) elastic/\1 logstash_system/g' -e 's/\(xpack.monitoring.elasticsearch.password:\) changeme/\1 testpasswd/g' logstash/config/logstash.yml
   - sed -i 's/\(password =>\) "changeme"/\1 "testpasswd"/g' logstash/pipeline/logstash.conf
+  - sed -i 's/\(elasticsearch.password:\) changeme/\1 testpasswd/g' extensions/app-search/config/app-search.yml
 
 script:
   # Core Elastic Stack
@@ -36,6 +37,17 @@ script:
   - docker-compose logs logstash
   - docker-compose down -v
 
+  # Extensions
+  - docker-compose up -d elasticsearch
+  - sleep 30
+  - .travis/elasticsearch-setup-passwords.exp
+    # App Search
+  - docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml up -d app-search
+  - .travis/run-tests-app-search.sh
+  - docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml ps
+  - docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml logs app-search
+  - docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml down -v
+
   # Swarm
   - docker swarm init
   - docker stack deploy -c ./docker-stack.yml elk

.travis/lib/testing.sh

@@ -12,6 +12,9 @@ function poll_ready {
 	if [ "$#" -ge 3 ]; then
 		args+=( '-u' "$3" )
 	fi
+	if [ "$#" -ge 4 ]; then
+		args+=( '-o' '/dev/null' )
+	fi
 
 	local label
 	if [ "$MODE" == "swarm" ]; then

.travis/run-tests-app-search.sh

+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+
+source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+
+
+declare MODE=""
+
+log 'Waiting for readiness of Elasticsearch'
+poll_ready elasticsearch 'http://localhost:9200/' 'elastic:testpasswd'
+
+log 'Waiting for readiness of App Search'
+poll_ready app-search 'http://localhost:3002/login' ':' quiet
+
+log 'Retrieving private key from Elasticsearch'
+response="$(curl 'http://localhost:9200/.app-search-actastic-loco_moco_api_tokens/_search?q=name:private-key' -s -u elastic:testpasswd)"
+hits="$(jq -rn --argjson data "${response}" '$data.hits.hits')"
+echo "$hits"
+count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
+if [[ $count -ne 1 ]]; then
+	echo "Private key not found. Expected 1 result, got ${count}"
+	exit 1
+fi
+key="$(jq -rn --argjson data "${hits}" '$data[0]._source.authentication_token')"
+
+log 'Creating App Search engine'
+response="$(curl 'http://localhost:3002/api/as/v1/engines' -s -d '{"name": "dockerelk"}' -H "Authorization: Bearer ${key}")"
+echo "$response"
+name="$(jq -rn --argjson data "${response}" '$data.name')"
+if [[ $name != 'dockerelk' ]]; then
+	echo 'Failed to create engine'
+	exit 1
+fi

extensions/app-search/Dockerfile

+ARG ELK_VERSION
+
+# https://www.docker.elastic.co/
+FROM docker.elastic.co/app-search/app-search:${ELK_VERSION}

extensions/app-search/README.md

+# App Search extension
+
+Elastic App Search provides access to a set of robust APIs and people friendly dashboard controls to deliver amazing
+search experiences, all backed by the Elastic Stack.
+
+## Requirements
+
+* 2 GB of free RAM, on top of the resources required by the other stack components and extensions
+
+App Search exposes the TCP port `3002` for its Web UI and API.
+
+## Usage
+
+To include App Search in the stack, run Docker Compose from the root of the repository with an additional command
+line argument referencing the `app-search-compose.yml` file:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml up
+```
+
+Allow a few minutes for the stack to start, then open your web browser at the address http://localhost:3002 to see the
+App Search home page.
+
+App Search is configured on first boot with the following default credentials:
+
+* user: *app_search*
+* password: *changeme*
+
+## Security
+
+The App Search password is defined inside the Compose file via the `APP_SEARCH_DEFAULT_PASSWORD` environment variable.
+We highly recommend choosing a more secure password than the default one for security reasons.
+
+To do so, change the value `APP_SEARCH_DEFAULT_PASSWORD` environment variable inside the Compose file **before the first
+boot**:
+
+```yaml
+app-search:
+
+  environment:
+    APP_SEARCH_DEFAULT_PASSWORD: {{some strong password}}
+```
+
+> :warning: The default App Search password can only be set during the initial boot. Once the password is persisted in
+> Elasticsearch, it can only be changed via the Elasticsearch API.
+
+For more information, please refer to [Security and User Management][appsearch-security].
+
+## Configuring App Search
+
+The App Search configuration is stored in [`config/app-search.yml`][config-appsearch]. You can modify this file using
+the [Default App Search configuration][appsearch-config] as a reference.
+
+You can also specify the options you want to override by setting environment variables inside the Compose file:
+
+```yaml
+app-search:
+
+  environment:
+    app_search.auth.source: standard
+    worker.threads: '6'
+```
+
+Any change to the App Search configuration requires a restart of the App Search container:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/app-search/app-search-compose.yml restart app-search
+```
+
+Please refer to the following documentation page for more details about how to configure App Search inside a Docker
+container: [Run App Search as a Docker container][appsearch-docker].
+
+## See also
+
+[App Search Self-Managed documentation][appsearch-selfmanaged]
+
+
+[config-appsearch]: ./config/app-search.yml
+
+[appsearch-security]: https://swiftype.com/documentation/app-search/self-managed/security
+[appsearch-config]: https://swiftype.com/documentation/app-search/self-managed/configuration
+[appsearch-docker]: https://swiftype.com/documentation/app-search/self-managed/installation#docker
+[appsearch-selfmanaged]: https://swiftype.com/documentation/app-search/self-managed/overview

extensions/app-search/app-search-compose.yml

+version: '3.2'
+
+services:
+  app-search:
+    build:
+      context: extensions/app-search/
+      args:
+        ELK_VERSION: $ELK_VERSION
+    volumes:
+      - type: bind
+        source: ./extensions/app-search/config/app-search.yml
+        target: /usr/share/app-search/config/app-search.yml
+        read_only: true
+    environment:
+      JAVA_OPTS: -Xmx2g -Xms2g
+      APP_SEARCH_DEFAULT_PASSWORD: changeme
+    ports:
+      - '3002:3002'
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch

extensions/app-search/config/app-search.yml

+---
+## App Search core configuration
+## https://swiftype.com/documentation/app-search/self-managed/configuration
+#
+
+# IP address App Search listens on
+app_search.listen_host: 0.0.0.0
+
+# URL at which users reach App Search
+app_search.external_url: http://localhost:3002
+
+# Elasticsearch URL and credentials
+elasticsearch.host: http://elasticsearch:9200
+elasticsearch.username: elastic
+elasticsearch.password: changeme
+
+# Allow App Search to modify Elasticsearch settings. Used to enable auto-creation of Elasticsearch indexes.
+allow_es_settings_modification: true