ci: Add test for Logspout extension

.github/workflows/ci.yml

@@ -85,8 +85,6 @@ jobs:
           docker-compose logs elasticsearch
           docker-compose logs logstash
           docker-compose logs kibana
-          # next steps don't need Logstash
-          docker-compose stop logstash
 
       ##############################
       #                            #
@@ -95,6 +93,38 @@ jobs:
       ##############################
 
       #
+      # Logspout
+      #
+
+      - name: Execute Logspout test suite
+        run: |
+
+          # Set mandatory Logstash settings
+
+          sed -i '$ a input { udp { port => 5000 codec => json } }' logstash/pipeline/logstash.conf
+
+          # Run Logspout and execute tests.
+          # Logstash will be restarted as a result of building the Logspout
+          # image, so changes above will automatically take effect.
+
+          docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml up -d logspout
+          .github/workflows/scripts/run-tests-logspout.sh
+
+          # Revert changes to Logstash configuration
+
+          sed -i '/input { udp { port => 5000 codec => json } }/d' logstash/pipeline/logstash.conf
+
+      - name: 'debug: Display state and logs (Logspout)'
+        if: always()
+        run: |
+          docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml ps
+          docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml logs logspout
+          # next steps don't need Logspout
+          docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml stop logspout
+          # next steps don't need Logstash
+          docker-compose stop logstash
+
+      #
       # Enterprise Search
       #
 
@@ -155,6 +185,7 @@ jobs:
         run: >-
           docker-compose
           -f docker-compose.yml
+          -f extensions/logspout/logspout-compose.yml
           -f extensions/enterprise-search/enterprise-search-compose.yml
           -f extensions/apm-server/apm-server-compose.yml
           down -v

.github/workflows/scripts/run-tests-logspout.sh

+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+
+source "$(dirname ${BASH_SOURCE[0]})/lib/testing.sh"
+
+
+cid_es="$(container_id elasticsearch)"
+cid_ls="$(container_id logspout)"
+
+ip_es="$(service_ip elasticsearch)"
+ip_ls="$(service_ip logspout)"
+
+log 'Waiting for readiness of Elasticsearch'
+poll_ready "$cid_es" "http://${ip_es}:9200/" -u 'elastic:testpasswd'
+
+log 'Waiting for readiness of Logspout'
+poll_ready "$cid_ls" "http://${ip_ls}/health"
+
+# When Logspout starts, it prints the following log line:
+#   2021/01/07 16:14:52 # logspout v3.2.13-custom by gliderlabs
+#
+# which we expect to find by querying:
+#   docker.image:"docker-elk_logspout" AND message:"logspout gliderlabs"~3
+#
+log 'Searching a log entry forwarded by Logspout'
+
+declare response
+declare -i count
+
+# retry for max 60s (30*2s)
+for _ in $(seq 1 30); do
+	response="$(curl "http://${ip_es}:9200/_count?q=docker.image:%22docker-elk_logspout%22%20AND%20message:%22logspout%20gliderlabs%22~3&pretty" -s -u elastic:testpasswd)"
+	count="$(jq -rn --argjson data "${response}" '$data.count')"
+	if [[ $count -gt 0 ]]; then
+		break
+	fi
+
+	echo -n 'x' >&2
+	sleep 2
+done
+echo -e '\n' >&2
+
+echo "$response"
+# Logspout may restart if Logstash isn't ready yet, so we tolerate multiple
+# results
+if [[ $count -lt 1 ]]; then
+	echo "Expected at least 1 document, got ${count}"
+	exit 1
+fi

extensions/logspout/modules.go

@@ -3,7 +3,8 @@ package main
 // installs the Logstash adapter for Logspout, and required dependencies
 // https://github.com/looplab/logspout-logstash
 import (
-	_ "github.com/looplab/logspout-logstash"
-	_ "github.com/gliderlabs/logspout/transports/udp"
+	_ "github.com/gliderlabs/logspout/healthcheck"
 	_ "github.com/gliderlabs/logspout/transports/tcp"
+	_ "github.com/gliderlabs/logspout/transports/udp"
+	_ "github.com/looplab/logspout-logstash"
 )