diff --git a/.env b/.env index ed7b944..b5c1aef 100644 --- a/.env +++ b/.env @@ -1,2 +1,2 @@ -TAG=6.5.2 -ELASTIC_VERSION=6.5.2 +TAG=6.5.4 +ELASTIC_VERSION=6.5.4 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..efdba87 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto +*.sh text eol=lf diff --git a/Makefile b/Makefile index 2113cec..a94f61a 100644 --- a/Makefile +++ b/Makefile @@ -27,8 +27,5 @@ $(TARGETS:%=%-checkout): (cd stack/$(@:%-checkout=%) && git fetch && git reset --hard && git checkout origin/$(GIT_BRANCH)) $(TARGETS:%=%-clean): - rm -rf stack/$(@:%-clean=%) && find . -name "*.keystore" -exec rm -f {} \; && \ - docker-compose -f docker-compose.setup.yml -f docker-compose.yml down --remove-orphans && \ - docker-compose -f setup.yml down --remove-orphans && \ - docker volume rm stack-docker_es_data + docker-compose down --remove-orphans -v diff --git a/README.md b/README.md index 8247d79..29749da 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,20 @@ Elastic Stack, all running on a single machine under Docker. * Windows and Mac users get Compose installed automatically with Docker for Windows/Mac. + * Ensure that docker-compose version >= 1.21.0, + * Compose introduced a bug wrt project names stripping out hyphens and underscores that was fixed in 1.21.0 + * Linux users can read the [install instructions](https://docs.docker.com/compose/install/#install-compose) or can install via pip: ``` pip install docker-compose ``` +### Windows +Please note that this repo has scripting that's written in `bash`. What this means is the containers that are linux based +are expecting linux style line endings. If you clone this repo and you have windows style line endings, then please +read [The case of Windows line-ending in bash-script](https://techblog.dorogin.com/case-of-windows-line-ending-in-bash-script-7236f056abe). We've included the `.gitattributes` file in this project to help alliviate any issues. But you might need to make +git global config settings. + * Windows Users must set the following 2 ENV vars: * `COMPOSE_CONVERT_WINDOWS_PATHS=1` * `PWD=/path/to/checkout/for/stack-docker` @@ -43,23 +52,38 @@ By default, the amount of Virtual Memory [is not enough](https://www.elastic.co/ First we need to: 1. set default password -2. create keystores to store passwords -3. install dashboards, index patterns, etc.. for beats and apm +2. Create SSL certs +3. create keystores to store passwords +4. install dashboards, index patterns, etc.. for beats and apm This is accomplished using the setup.yml file: ``` -docker-compose -f setup.yml up +docker-compose -f setup/setup.yml run --rm setup ``` Please take note after the setup completes it will output the password that is used for the `elastic` login. +*Please note* that this repository makes the assumption that you will be cloneing the repository into a directory +named `stack-docker`. If you choose to name the directory something else upon cloneing or after, you will have +to specify the `-p stack-docker` flag when running `docker-compose` commands (aside from the setup command). + +From the docker-compose help: +``` +-p, --project-name NAME Specify an alternate project name + (default: directory name) +``` + Now we can launch the stack with `docker-compose up -d` to create a demonstration Elastic Stack with Elasticsearch, Kibana, Logstash, Auditbeat, Metricbeat, Filebeat, Packetbeat, and Heartbeat. Point a browser at [`http://localhost:5601`](http://localhost:5601) to see the results. > *NOTE*: Elasticsearch is now setup with self-signed certs. +> This means anytime you want to interact with elasticsearch by using other tools/clients you must use +> https, and if you want to get the `ca.crt` you can get it by running +> `docker exec -it elasticsearch cat /usr/share/elasticsearch/config/certs/ssl/ca/ca.crt` + Log in with `elastic` and what ever your auto generated elastic password is from the setup. diff --git a/config/apm-server/apm-server.yml b/config/apm-server/apm-server.yml index faba2ef..8fba2ee 100644 --- a/config/apm-server/apm-server.yml +++ b/config/apm-server/apm-server.yml @@ -1,3 +1,5 @@ +keystore.path: "/usr/share/apm-server/config/apm-server.keystore" + apm-server.frontend.enabled: true apm-server.host: "0.0.0.0:8200" @@ -7,7 +9,7 @@ output.elasticsearch: username: elastic # Read PW from apm-server.keystore password: "${ELASTIC_PASSWORD}" - ssl.certificate_authorities: ["/usr/share/apm-server/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] setup.kibana: host: "http://kibana:5601" @@ -15,4 +17,4 @@ setup.kibana: password: "${ELASTIC_PASSWORD}" protocol: "http" ssl.enabled: false - ssl.certificate_authorities: ["/usr/share/apm-server/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] diff --git a/config/auditbeat/auditbeat.yml b/config/auditbeat/auditbeat.yml index 98f1613..62993a9 100644 --- a/config/auditbeat/auditbeat.yml +++ b/config/auditbeat/auditbeat.yml @@ -1,3 +1,5 @@ +keystore.path: "/usr/share/auditbeat/config/auditbeat.keystore" + auditbeat.modules: - module: auditd @@ -19,7 +21,7 @@ output.elasticsearch: username: elastic # Read PW from auditbeat.keystore password: "${ELASTIC_PASSWORD}" - ssl.certificate_authorities: ["/usr/share/auditbeat/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] setup.kibana: host: "http://kibana:5601" @@ -27,6 +29,6 @@ setup.kibana: password: "${ELASTIC_PASSWORD}" protocol: "http" ssl.enabled: false - ssl.certificate_authorities: ["/usr/share/auditbeat/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] xpack.monitoring.enabled: true diff --git a/config/elasticsearch/elasticsearch.p12 b/config/elasticsearch/elasticsearch.p12 deleted file mode 100644 index 7ee72bb..0000000 Binary files a/config/elasticsearch/elasticsearch.p12 and /dev/null differ diff --git a/config/elasticsearch/elasticsearch.yml b/config/elasticsearch/elasticsearch.yml index 0f77d84..6cab4f7 100644 --- a/config/elasticsearch/elasticsearch.yml +++ b/config/elasticsearch/elasticsearch.yml @@ -10,11 +10,11 @@ xpack.license.self_generated.type: trial xpack.security.enabled: true xpack.security.http.ssl.enabled: true xpack.security.http.ssl.verification_mode: certificate -xpack.security.http.ssl.key: certs/elasticsearch/elasticsearch.key -xpack.security.http.ssl.certificate: certs/elasticsearch/elasticsearch.crt -xpack.security.http.ssl.certificate_authorities: [ "certs/ca/ca.crt" ] +xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/ssl/docker-cluster/elasticsearch/elasticsearch.key +xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/ssl/docker-cluster/elasticsearch/elasticsearch.crt +xpack.security.http.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/certs/ssl/ca/ca.crt" ] xpack.security.transport.ssl.enabled: true -xpack.security.transport.ssl.key: certs/elasticsearch/elasticsearch.key -xpack.security.transport.ssl.certificate: certs/elasticsearch/elasticsearch.crt -xpack.security.transport.ssl.certificate_authorities: [ "certs/ca/ca.crt" ] +xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/ssl/docker-cluster/elasticsearch/elasticsearch.key +xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/ssl/docker-cluster/elasticsearch/elasticsearch.crt +xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/certs/ssl/ca/ca.crt" ] diff --git a/config/filebeat/filebeat.yml b/config/filebeat/filebeat.yml index 78ccd0a..f6755b4 100644 --- a/config/filebeat/filebeat.yml +++ b/config/filebeat/filebeat.yml @@ -1,3 +1,5 @@ +keystore.path: "/usr/share/filebeat/config/filebeat.keystore" + filebeat.config: prospectors: path: ${path.config}/prospectors.d/*.yml @@ -15,7 +17,7 @@ output.elasticsearch: username: elastic # Read PW from filebeat.keystore password: "${ELASTIC_PASSWORD}" - ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] setup.kibana: host: "http://kibana:5601" @@ -23,6 +25,6 @@ setup.kibana: password: "${ELASTIC_PASSWORD}" protocol: "http" ssl.enabled: false - ssl.certificate_authorities: ["/usr/share/filebeat/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] xpack.monitoring.enabled: true diff --git a/config/heartbeat/heartbeat.yml b/config/heartbeat/heartbeat.yml index bbdba46..77e82f3 100644 --- a/config/heartbeat/heartbeat.yml +++ b/config/heartbeat/heartbeat.yml @@ -1,3 +1,5 @@ +keystore.path: "/usr/share/heartbeat/config/heartbeat.keystore" + heartbeat.monitors: - type: http schedule: '@every 5s' @@ -5,7 +7,9 @@ heartbeat.monitors: - https://elasticsearch:9200 - http://kibana:5601 ssl: - certificate_authorities: ["/usr/share/heartbeat/certs/ca/ca.crt"] + certificate_authorities: ["/certs/ssl/ca/ca.crt"] + username: elastic + password: "${ELASTIC_PASSWORD}" - type: icmp schedule: '@every 5s' @@ -22,7 +26,7 @@ output.elasticsearch: username: elastic # Read PW from heartbeat.keystore password: "${ELASTIC_PASSWORD}" - ssl.certificate_authorities: ["/usr/share/heartbeat/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] setup.kibana: host: "http://kibana:5601" @@ -30,6 +34,6 @@ setup.kibana: password: "${ELASTIC_PASSWORD}" protocol: "http" ssl.enabled: false - ssl.certificate_authorities: ["/usr/share/heartbeat/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] xpack.monitoring.enabled: true diff --git a/config/ssl/instances.yml b/config/instances.yml similarity index 100% rename from config/ssl/instances.yml rename to config/instances.yml diff --git a/config/kibana/kibana.p12 b/config/kibana/kibana.p12 deleted file mode 100644 index 04f9219..0000000 Binary files a/config/kibana/kibana.p12 and /dev/null differ diff --git a/config/kibana/kibana.yml b/config/kibana/kibana.yml index 37857aa..9409391 100644 --- a/config/kibana/kibana.yml +++ b/config/kibana/kibana.yml @@ -5,8 +5,8 @@ server.host: "0" elasticsearch.url: https://elasticsearch:9200 # elasticsearch.password is stored in `kibana.keystore` elasticsearch.username: kibana -elasticsearch.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca/ca.crt"] +elasticsearch.ssl.certificateAuthorities: ["/certs/ssl/ca/ca.crt"] server.ssl.enabled: false -#server.ssl.certificate: /usr/share/kibana/config/certs/kibana/kibana.crt -#server.ssl.key: /usr/share/kibana/config/certs/kibana/kibana.key +#server.ssl.certificate: /certs/ssl/docker-cluster/kibana/kibana.crt +#server.ssl.key: /certs/ssl/docker-cluster/kibana/kibana.key xpack.monitoring.ui.container.elasticsearch.enabled: true diff --git a/config/logstash/logstash.p12 b/config/logstash/logstash.p12 deleted file mode 100644 index 502fc6d..0000000 Binary files a/config/logstash/logstash.p12 and /dev/null differ diff --git a/config/logstash/logstash.yml b/config/logstash/logstash.yml index b53754f..3856890 100644 --- a/config/logstash/logstash.yml +++ b/config/logstash/logstash.yml @@ -3,5 +3,4 @@ http.host: 0.0.0.0 xpack.monitoring.elasticsearch.password: ${ELASTIC_PASSWORD} xpack.monitoring.elasticsearch.url: https://elasticsearch:9200 xpack.monitoring.elasticsearch.username: logstash_system -xpack.monitoring.elasticsearch.ssl.ca: /usr/share/logstash/config/certs/ca/ca.crt - +xpack.monitoring.elasticsearch.ssl.ca: /certs/ssl/ca/ca.crt diff --git a/config/logstash/pipeline/logstash.conf b/config/logstash/pipeline/logstash.conf index 596ec9a..f9da078 100644 --- a/config/logstash/pipeline/logstash.conf +++ b/config/logstash/pipeline/logstash.conf @@ -11,6 +11,6 @@ output { user => 'elastic' password => "${ELASTIC_PASSWORD}" # read password from logstash.keystore ssl => true - cacert => '/usr/share/logstash/config/certs/ca/ca.crt' + cacert => '/certs/ssl/ca/ca.crt' } } diff --git a/config/logstash/pipelines.yml b/config/logstash/pipelines.yml new file mode 100644 index 0000000..3cbebe2 --- /dev/null +++ b/config/logstash/pipelines.yml @@ -0,0 +1,2 @@ +- pipeline.id: pipeline_1 + path.config: '/usr/share/logstash/pipeline/logstash.conf' diff --git a/config/metricbeat/metricbeat.yml b/config/metricbeat/metricbeat.yml index bfc3a2e..ba4b054 100644 --- a/config/metricbeat/metricbeat.yml +++ b/config/metricbeat/metricbeat.yml @@ -1,3 +1,5 @@ +keystore.path: "/usr/share/metricbeat/config/metricbeat.keystore" + metricbeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false @@ -11,7 +13,7 @@ output.elasticsearch: username: elastic # Read PW from metricbeat.keystore password: "${ELASTIC_PASSWORD}" - ssl.certificate_authorities: ["/usr/share/metricbeat/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] setup.kibana: host: "http://kibana:5601" @@ -19,6 +21,6 @@ setup.kibana: password: "${ELASTIC_PASSWORD}" protocol: "http" ssl.enabled: false - ssl.certificate_authorities: ["/usr/share/metricbeat/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] xpack.monitoring.enabled: true diff --git a/config/packetbeat/packetbeat.yml b/config/packetbeat/packetbeat.yml index 48c2248..abae754 100644 --- a/config/packetbeat/packetbeat.yml +++ b/config/packetbeat/packetbeat.yml @@ -1,3 +1,5 @@ +keystore.path: "/usr/share/packetbeat/config/packetbeat.keystore" + packetbeat.interfaces.device: any packetbeat.flows: @@ -38,19 +40,19 @@ processors: - add_cloud_metadata: output.elasticsearch: - hosts: ['elasticsearch:9200'] + hosts: ['localhost:9200'] # locahost cause we are using network mode = host protocol: "https" username: elastic # Read PW from packetbeat.keystore password: "${ELASTIC_PASSWORD}" - ssl.certificate_authorities: ["/usr/share/packetbeat/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] setup.kibana: - host: "http://kibana:5601" + host: "http://localhost:5601" # locahost cause we are using network mode = host username: elastic password: "${ELASTIC_PASSWORD}" protocol: "http" ssl.enabled: false - ssl.certificate_authorities: ["/usr/share/packetbeat/certs/ca/ca.crt"] + ssl.certificate_authorities: ["/certs/ssl/ca/ca.crt"] xpack.monitoring.enabled: true diff --git a/docker-compose.setup.yml b/docker-compose.setup.yml deleted file mode 100644 index 50acaea..0000000 --- a/docker-compose.setup.yml +++ /dev/null @@ -1,128 +0,0 @@ ---- -version: '3.6' -services: - # Setup Elasticsearch - # * keystore - # * SSL - setup_elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:${TAG} - container_name: setup_elasticsearch - command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-elasticsearch.sh | tr -d "\r" | bash'] - environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] - working_dir: '/config' - volumes: - - './config:/config' - - './scripts/setup-elasticsearch.sh:/usr/local/bin/setup-elasticsearch.sh:ro' - - setup_kibana: - image: docker.elastic.co/kibana/kibana:${TAG} - container_name: setup_kibana - command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-kibana.sh | tr -d "\r" | bash'] - environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] - working_dir: '/config' - volumes: - - './config:/config' - - './scripts/setup-kibana.sh:/usr/local/bin/setup-kibana.sh:ro' - - './config/ssl/ca/ca.crt:/usr/share/kibana/config/ca/ca.crt' - depends_on: ['elasticsearch'] - networks: ['stack'] - - setup_logstash: - image: docker.elastic.co/logstash/logstash:${TAG} - container_name: setup_logstash - command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-logstash.sh | tr -d "\r" | bash'] - environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] - working_dir: '/config' - volumes: - - './config:/config' - - './scripts/setup-logstash.sh:/usr/local/bin/setup-logstash.sh:ro' - - './config/ssl/ca/ca.crt:/usr/share/logstash/config/ca/ca.crt' - depends_on: ['elasticsearch'] - networks: ['stack'] - - setup_auditbeat: - image: docker.elastic.co/beats/auditbeat:${TAG} - container_name: setup_auditbeat - user: root - pid: host - cap_add: ['AUDIT_CONTROL', 'AUDIT_READ'] - volumes: - - './config:/config' - - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' - - './config/auditbeat/auditbeat.yml:/usr/share/auditbeat/auditbeat.yml' - - './config/ssl/ca/ca.crt:/usr/share/auditbeat/certs/ca/ca.crt' - command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-beat.sh | tr -d "\r" | bash -s auditbeat'] - environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] - networks: ['stack'] - depends_on: ['kibana'] - - setup_filebeat: - image: docker.elastic.co/beats/filebeat:${TAG} - container_name: setup_filebeat - user: root - volumes: - - './config:/config' - - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' - - './config/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml' - - './config/ssl/ca/ca.crt:/usr/share/filebeat/certs/ca/ca.crt' - command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-beat.sh | tr -d "\r" | bash -s filebeat'] - environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] - networks: ['stack'] - depends_on: ['kibana'] - - setup_heartbeat: - image: docker.elastic.co/beats/heartbeat:${TAG} - container_name: setup_heartbeat - user: root - volumes: - - './config:/config' - - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' - - './config/heartbeat/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml' - - './config/ssl/ca/ca.crt:/usr/share/heartbeat/certs/ca/ca.crt' - command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-beat.sh | tr -d "\r" | bash -s heartbeat'] - environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] - networks: ['stack'] - depends_on: ['kibana'] - - setup_metricbeat: - image: docker.elastic.co/beats/metricbeat:${TAG} - container_name: setup_metricbeat - user: root - volumes: - - './config:/config' - - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' - - './config/metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml' - - './config/ssl/ca/ca.crt:/usr/share/metricbeat/certs/ca/ca.crt' - command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-beat.sh | tr -d "\r" | bash -s metricbeat'] - environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] - networks: ['stack'] - depends_on: ['kibana'] - - setup_packetbeat: - image: docker.elastic.co/beats/packetbeat:${TAG} - container_name: setup_packetbeat - user: root - cap_add: ['NET_RAW', 'NET_ADMIN'] - command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-beat.sh | tr -d "\r" | bash -s packetbeat'] - volumes: - - './config:/config' - - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' - - './config/packetbeat/packetbeat.yml:/usr/share/packetbeat/packetbeat.yml' - - './config/ssl/ca/ca.crt:/usr/share/packetbeat/certs/ca/ca.crt' - environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] - networks: ['stack'] - depends_on: ['kibana'] - - setup_apm_server: - image: docker.elastic.co/apm/apm-server:${TAG} - container_name: setup_apm_server - user: root - command: ['/bin/bash', '-c', 'cat /usr/local/bin/setup-beat.sh | tr -d "\r" | bash -s apm-server'] - volumes: - - './config:/config' - - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' - - './config/apm-server/apm-server.yml:/usr/share/apm-server/apm-server.yml' - - './config/ssl/ca/ca.crt:/usr/share/apm-server/certs/ca/ca.crt' - environment: ['ELASTIC_PASSWORD=${ELASTIC_PASSWORD}'] - networks: ['stack'] - depends_on: ['kibana'] diff --git a/docker-compose.yml b/docker-compose.yml index 346dd61..e85dcba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,22 +20,18 @@ services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:${TAG} container_name: elasticsearch - secrets: - - source: ca.crt - target: /usr/share/elasticsearch/config/certs/ca/ca.crt - - source: elasticsearch.yml - target: /usr/share/elasticsearch/config/elasticsearch.yml - - source: elasticsearch.keystore - target: /usr/share/elasticsearch/config/elasticsearch.keystore - - source: elasticsearch.key - target: /usr/share/elasticsearch/config/certs/elasticsearch/elasticsearch.key - - source: elasticsearch.crt - target: /usr/share/elasticsearch/config/certs/elasticsearch/elasticsearch.crt ports: ['9200:9200'] networks: ['stack'] volumes: + - 'es_config:/usr/share/elasticsearch/config/' + - 'certs:/usr/share/elasticsearch/config/certs' + - 'ls_config:/logstash' + - 'kb_config:/kibana' - 'es_data:/usr/share/elasticsearch/data' - './scripts/setup-users.sh:/usr/local/bin/setup-users.sh:ro' + - './scripts/setup-elasticsearch.sh:/usr/local/bin/setup-elasticsearch.sh:ro' + - './config/instances.yml:/usr/share/elasticsearch/config/certs/ssl/instances.yml' + - './config/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml' healthcheck: test: curl --cacert /usr/share/elasticsearch/config/certs/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi interval: 30s @@ -45,19 +41,14 @@ services: kibana: image: docker.elastic.co/kibana/kibana:${TAG} container_name: kibana - secrets: - - source: kibana.yml - target: /usr/share/kibana/config/kibana.yml - - source: kibana.keystore - target: /usr/share/kibana/data/kibana.keystore - - source: ca.crt - target: /usr/share/kibana/config/certs/ca/ca.crt - - source: kibana.key - target: /usr/share/kibana/config/certs/kibana/kibana.key - - source: kibana.crt - target: /usr/share/kibana/config/certs/kibana/kibana.crt ports: ['5601:5601'] networks: ['stack'] + volumes: + - 'kb_config:/usr/share/kibana/config' + - 'kb_data:/usr/share/kibana/data/' + - 'certs:/certs' + - './config/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml' + - './scripts/setup-kibana.sh:/usr/local/bin/setup-kibana.sh:ro' depends_on: ['elasticsearch'] healthcheck: test: curl --cacert /usr/share/elasticsearch/config/certs/ca/ca.crt -s https://localhost:5601 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi @@ -68,16 +59,14 @@ services: logstash: image: docker.elastic.co/logstash/logstash:${TAG} container_name: logstash - secrets: - - source: logstash.conf - target: /usr/share/logstash/pipeline/logstash.conf - - source: logstash.yml - target: /usr/share/logstash/config/logstash.yml - - source: logstash.keystore - target: /usr/share/logstash/config/logstash.keystore - - source: ca.crt - target: /usr/share/logstash/config/certs/ca/ca.crt networks: ['stack'] + volumes: + - 'ls_config:/usr/share/logstash/config' + - 'certs:/certs' + - './config/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml' + - './config/logstash/pipelines.yml:/usr/share/logstash/config/pipelines.yml' + - './config/logstash/pipeline/logstash.conf:/usr/share/logstash/pipeline/logstash.conf' + - './scripts/setup-logstash.sh:/usr/local/bin/setup-logstash.sh:ro' depends_on: ['elasticsearch'] healthcheck: test: bin/logstash -t @@ -88,19 +77,16 @@ services: auditbeat: image: docker.elastic.co/beats/auditbeat:${TAG} container_name: auditbeat - command: -e --strict.perms=false # -e flag to log to stderr and disable syslog/file output + hostname: auditbeat + command: -e -c=config/auditbeat.yml # -e flag to log to stderr and disable syslog/file output cap_add: ['AUDIT_CONTROL', 'AUDIT_READ'] - secrets: - - source: auditbeat.yml - target: /usr/share/auditbeat/auditbeat.yml - - source: auditbeat.keystore - target: /usr/share/auditbeat/auditbeat.keystore - - source: ca.crt - target: /usr/share/auditbeat/certs/ca/ca.crt # Auditbeat must run in the main process namespace. pid: host volumes: + - 'ab_config:/usr/share/auditbeat/config' + - 'certs:/certs' - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' + - './config/auditbeat/auditbeat.yml:/usr/share/auditbeat/config/auditbeat.yml' networks: ['stack'] depends_on: ['elasticsearch', 'kibana'] healthcheck: @@ -112,19 +98,16 @@ services: filebeat: image: docker.elastic.co/beats/filebeat:${TAG} container_name: filebeat - command: --strict.perms=false -e # -e flag to log to stderr and disable syslog/file output + hostname: filebeat + command: -e -c=config/filebeat.yml # -e flag to log to stderr and disable syslog/file output # If the host system has logs at "/var/log", mount them at "/mnt/log" # inside the container, where Filebeat can find them. # volumes: ['/var/log:/mnt/log:ro'] - secrets: - - source: filebeat.yml - target: /usr/share/filebeat/filebeat.yml - - source: filebeat.keystore - target: /usr/share/filebeat/filebeat.keystore - - source: ca.crt - target: /usr/share/filebeat/certs/ca/ca.crt volumes: + - 'fb_config:/usr/share/filebeat/config' + - 'certs:/certs' - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' + - './config/filebeat/filebeat.yml:/usr/share/filebeat/config/filebeat.yml' networks: ['stack'] depends_on: ['elasticsearch', 'kibana'] healthcheck: @@ -136,16 +119,13 @@ services: heartbeat: image: docker.elastic.co/beats/heartbeat:${TAG} container_name: heartbeat - command: --strict.perms=false -e # -e flag to log to stderr and disable syslog/file output - secrets: - - source: heartbeat.yml - target: /usr/share/heartbeat/heartbeat.yml - - source: heartbeat.keystore - target: /usr/share/heartbeat/heartbeat.keystore - - source: ca.crt - target: /usr/share/heartbeat/certs/ca/ca.crt + hostname: heartbeat + command: -e -c=config/heartbeat.yml # -e flag to log to stderr and disable syslog/file output volumes: + - 'hb_config:/usr/share/heartbeat/config' + - 'certs:/certs' - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' + - './config/heartbeat/heartbeat.yml:/usr/share/heartbeat/config/heartbeat.yml' networks: ['stack'] depends_on: ['elasticsearch', 'kibana'] healthcheck: @@ -157,6 +137,7 @@ services: metricbeat: image: docker.elastic.co/beats/metricbeat:${TAG} container_name: metricbeat + hostname: metricbeat # The commented sections below enable Metricbeat to monitor the Docker host, # rather than the Metricbeat container. It's problematic with Docker for # Windows, however, since "/proc", "/sys" etc. don't exist on Windows. @@ -165,16 +146,12 @@ services: # - /proc:/hostfs/proc:ro # - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro # - /:/hostfs:ro - command: --strict.perms=false -e # -e flag to log to stderr and disable syslog/file output - secrets: - - source: metricbeat.yml - target: /usr/share/metricbeat/metricbeat.yml - - source: metricbeat.keystore - target: /usr/share/metricbeat/metricbeat.keystore - - source: ca.crt - target: /usr/share/metricbeat/certs/ca/ca.crt + command: -e -c=config/metricbeat.yml # -e flag to log to stderr and disable syslog/file output volumes: + - 'mb_config:/usr/share/metricbeat/config' + - 'certs:/certs' - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' + - './config/metricbeat/metricbeat.yml:/usr/share/metricbeat/config/metricbeat.yml' networks: ['stack'] depends_on: ['elasticsearch', 'kibana'] healthcheck: @@ -186,6 +163,7 @@ services: packetbeat: image: docker.elastic.co/beats/packetbeat:${TAG} container_name: packetbeat + hostname: packetbeat # Packetbeat needs some elevated privileges to capture network traffic. # We'll grant them with POSIX capabilities. cap_add: ['NET_RAW', 'NET_ADMIN'] @@ -197,18 +175,13 @@ services: # that the other containers are connected to, and thus can't resolve the # hostname "elasticsearch". Instead, we'll tell it to find Elasticsearch # on "localhost", which is the Docker host machine in this context. - command: -e -E 'output.elasticsearch.hosts=["localhost:9200"]' depends_on: ['elasticsearch'] - command: --strict.perms=false -e -E output.elasticsearch.hosts="https://localhost:9200" # -e flag to log to stderr and disable syslog/file output - secrets: - - source: packetbeat.yml - target: /usr/share/packetbeat/packetbeat.yml - - source: packetbeat.keystore - target: /usr/share/packetbeat/packetbeat.keystore - - source: ca.crt - target: /usr/share/packetbeat/certs/ca/ca.crt + command: --strict.perms=false -e -c=/usr/share/packetbeat/config/packetbeat.yml # -e flag to log to stderr and disable syslog/file output volumes: + - 'pb_config:/usr/share/packetbeat/config' + - 'certs:/certs' - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' + - './config/packetbeat/packetbeat.yml:/usr/share/packetbeat/config/packetbeat.yml' depends_on: ['elasticsearch', 'kibana'] healthcheck: test: packetbeat test config @@ -219,21 +192,18 @@ services: apm-server: image: docker.elastic.co/apm/apm-server:${TAG} container_name: apm_server + hostname: apm-server ports: ['8200:8200'] networks: ['stack'] - command: --strict.perms=false -e # -e flag to log to stderr and disable syslog/file output - secrets: - - source: apm-server.yml - target: /usr/share/apm-server/apm-server.yml - - source: apm-server.keystore - target: /usr/share/apm-server/apm-server.keystore - - source: ca.crt - target: /usr/share/apm-server/certs/ca/ca.crt + command: --strict.perms=false -e -c=/usr/share/apm-server/config/apm-server.yml # -e flag to log to stderr and disable syslog/file output volumes: + - 'apm_config:/usr/share/apm-server/config' + - 'certs:/certs' - './scripts/setup-beat.sh:/usr/local/bin/setup-beat.sh:ro' + - './config/apm-server/apm-server.yml:/usr/share/apm-server/config/apm-server.yml' depends_on: ['elasticsearch', 'kibana'] healthcheck: - test: curl --cacert /usr/share/elasticsearch/config/certs/ca/ca.crt -s https://localhost:8200/healthcheck >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi + test: curl --cacert /usr/share/apm-server/config/ca/ca.crt -s https://localhost:8200/healthcheck >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi interval: 30s timeout: 10s retries: 5 @@ -241,56 +211,15 @@ services: networks: {stack: {}} # use docker volume to persist ES data outside of a container. volumes: + certs: es_data: - -secrets: - ca.crt: - file: ./config/ssl/ca/ca.crt - logstash.yml: - file: ./config/logstash/logstash.yml - logstash.keystore: - file: ./config/logstash/logstash.keystore - logstash.conf: - file: ./config/logstash/pipeline/logstash.conf - elasticsearch.yml: - file: ./config/elasticsearch/elasticsearch.yml - elasticsearch.keystore: - file: ./config/elasticsearch/elasticsearch.keystore - elasticsearch.key: - file: ./config/elasticsearch/elasticsearch.key - elasticsearch.crt: - file: ./config/elasticsearch/elasticsearch.crt - elasticsearch.p12: - file: ./config/elasticsearch/elasticsearch.p12 - kibana.yml: - file: ./config/kibana/kibana.yml - kibana.keystore: - file: ./config/kibana/kibana.keystore - kibana.key: - file: ./config/kibana/kibana.key - kibana.crt: - file: ./config/kibana/kibana.crt - auditbeat.yml: - file: ./config/auditbeat/auditbeat.yml - auditbeat.keystore: - file: ./config/auditbeat/auditbeat.keystore - filebeat.yml: - file: ./config/filebeat/filebeat.yml - filebeat.keystore: - file: ./config/filebeat/filebeat.keystore - heartbeat.yml: - file: ./config/heartbeat/heartbeat.yml - heartbeat.keystore: - file: ./config/heartbeat/heartbeat.keystore - metricbeat.yml: - file: ./config/metricbeat/metricbeat.yml - metricbeat.keystore: - file: ./config/metricbeat/metricbeat.keystore - packetbeat.yml: - file: ./config/packetbeat/packetbeat.yml - packetbeat.keystore: - file: ./config/packetbeat/packetbeat.keystore - apm-server.yml: - file: ./config/apm-server/apm-server.yml - apm-server.keystore: - file: ./config/apm-server/apm-server.keystore + es_config: + kb_config: + kb_data: + ls_config: + ab_config: + fb_config: + hb_config: + mb_config: + pb_config: + apm_config: diff --git a/scripts/setup-beat.sh b/scripts/setup-beat.sh index 57d6458..13a9e7c 100755 --- a/scripts/setup-beat.sh +++ b/scripts/setup-beat.sh @@ -1,32 +1,30 @@ #!/bin/bash -set -euo pipefail +beat=$(hostname) +command="$beat --c=/usr/share/$beat/config/$beat.yml --strict.perms=false" +echo "Running setup for $beat" -beat=$1 +if [[ $beat == "packetbeat" ]]; then + kibana_domain="localhost" +else + kibana_domain="kibana" +fi -until curl -s "http://kibana:5601/login" | grep "Loading Kibana" > /dev/null; do +until curl -s "http://${kibana_domain}:5601/login" | grep "Loading Kibana" > /dev/null; do echo "Waiting for kibana..." sleep 1 done -chmod go-w /usr/share/$beat/$beat.yml - - echo "Creating keystore..." # create beat keystore -${beat} --strict.perms=false keystore create --force -chown 1000 /usr/share/$beat/$beat.keystore -chmod go-w /usr/share/$beat/$beat.yml +eval "$command keystore create --force" echo "adding ES_PASSWORD to keystore..." -echo "$ELASTIC_PASSWORD" | ${beat} --strict.perms=false keystore add ELASTIC_PASSWORD --stdin -${beat} --strict.perms=false keystore list +echo "$ELASTIC_PASSWORD" | ${command} keystore add ELASTIC_PASSWORD --stdin +eval "$command keystore list" echo "Setting up dashboards..." # Load the sample dashboards for the Beat. # REF: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-sample-dashboards.html -${beat} --strict.perms=false setup -v - -echo "Copy keystore to ./config dir" -cp /usr/share/$beat/$beat.keystore /config/$beat/$beat.keystore -chown 1000:1000 /config/$beat/$beat.keystore +eval "$command setup -v" +chown -R 1000:1000 config diff --git a/scripts/setup-elasticsearch.sh b/scripts/setup-elasticsearch.sh index 13830e8..585b17b 100755 --- a/scripts/setup-elasticsearch.sh +++ b/scripts/setup-elasticsearch.sh @@ -1,10 +1,5 @@ #!/bin/bash - -if [ -f /config/elasticsearch/elasticsearch.keystore ]; then - echo "Keystore already exists, exiting. If you want to re-run please delete config/elasticsearch/elasticsearch.keystore" - exit 0 -fi - +configdir=/usr/share/elasticsearch/config # Determine if x-pack is enabled echo "Determining if x-pack is installed..." if [[ -d /usr/share/elasticsearch/bin/x-pack ]]; then @@ -12,53 +7,48 @@ if [[ -d /usr/share/elasticsearch/bin/x-pack ]]; then echo "=== CREATE Keystore ===" echo "Elastic password is: $ELASTIC_PASSWORD" - if [ -f /config/elasticsearch/elasticsearch.keystore ]; then + if [ -f $configdir/elasticsearch.keystore ]; then echo "Remove old elasticsearch.keystore" - rm /config/elasticsearch/elasticsearch.keystore + rm $configdir/elasticsearch.keystore fi - [[ -f /usr/share/elasticsearch/config/elasticsearch.keystore ]] || (/usr/share/elasticsearch/bin/elasticsearch-keystore create) + [[ -f $configdir/elasticsearch.keystore ]] || (/usr/share/elasticsearch/bin/elasticsearch-keystore create) echo "Setting bootstrap.password..." (echo "$ELASTIC_PASSWORD" | /usr/share/elasticsearch/bin/elasticsearch-keystore add -x 'bootstrap.password') - mv /usr/share/elasticsearch/config/elasticsearch.keystore /config/elasticsearch/elasticsearch.keystore # Create SSL Certs echo "=== CREATE SSL CERTS ===" # check if old docker-cluster-ca.zip exists, if it does remove and create a new one. - if [ -f /config/ssl/docker-cluster-ca.zip ]; then + if [ -f $configdir/certs/ssl/docker-cluster-ca.zip ]; then echo "Remove old ca zip..." - rm /config/ssl/docker-cluster-ca.zip + rm $configdir/certs/ssl/docker-cluster-ca.zip fi echo "Creating docker-cluster-ca.zip..." - /usr/share/elasticsearch/bin/elasticsearch-certutil ca --pem --silent --out /config/ssl/docker-cluster-ca.zip + /usr/share/elasticsearch/bin/elasticsearch-certutil ca --pem --silent --out $configdir/certs/ssl/docker-cluster-ca.zip # check if ca directory exists, if does, remove then unzip new files - if [ -d /config/ssl/ca ]; then + if [ -d $configdir/certs/ssl/ca ]; then echo "CA directory exists, removing..." - rm -rf /config/ssl/ca + rm -rf $configdir/certs/ssl/ca fi echo "Unzip ca files..." - unzip /config/ssl/docker-cluster-ca.zip -d /config/ssl + unzip $configdir/certs/ssl/docker-cluster-ca.zip -d $configdir/certs/ssl # check if certs zip exist. If it does remove and create a new one. - if [ -f /config/ssl/docker-cluster.zip ]; then + if [ -f $configdir/certs/ssl/docker-cluster.zip ]; then echo "Remove old docker-cluster.zip zip..." - rm /config/ssl/docker-cluster.zip + rm $configdir/certs/ssl/docker-cluster.zip fi echo "Create cluster certs zipfile..." - /usr/share/elasticsearch/bin/elasticsearch-certutil cert --silent --pem --in /config/ssl/instances.yml --out /config/ssl/docker-cluster.zip --ca-cert /config/ssl/ca/ca.crt --ca-key /config/ssl/ca/ca.key + /usr/share/elasticsearch/bin/elasticsearch-certutil cert --silent --pem --in $configdir/certs/ssl/instances.yml --out $configdir/certs/ssl/docker-cluster.zip --ca-cert $configdir/certs/ssl/ca/ca.crt --ca-key $configdir/certs/ssl/ca/ca.key - if [ -d /config/ssl/docker-cluster ]; then - rm -rf /config/ssl/docker-cluster + if [ -d $configdir/certs/ssl/docker-cluster ]; then + rm -rf $configdir/certs/ssl/docker-cluster fi echo "Unzipping cluster certs zipfile..." - unzip /config/ssl/docker-cluster.zip -d /config/ssl/docker-cluster + unzip $configdir/certs/ssl/docker-cluster.zip -d $configdir/certs/ssl/docker-cluster - echo "Move logstash certs to logstash config dir..." - mv /config/ssl/docker-cluster/logstash/* /config/logstash/ - echo "Move kibana certs to kibana config dir..." - mv /config/ssl/docker-cluster/kibana/* /config/kibana/ - echo "Move elasticsearch certs to elasticsearch config dir..." - mv /config/ssl/docker-cluster/elasticsearch/* /config/elasticsearch/ + chown 1000:1000 -R $configdir/certs + echo "setup-elasticsearch.sh.... done" fi fi diff --git a/scripts/setup-kibana.sh b/scripts/setup-kibana.sh index e7b1851..2c308d0 100755 --- a/scripts/setup-kibana.sh +++ b/scripts/setup-kibana.sh @@ -2,7 +2,7 @@ set -euo pipefail -cacert=/usr/share/kibana/config/ca/ca.crt +cacert=/certs/ssl/ca/ca.crt # Wait for ca file to exist before we continue. If the ca file doesn't exist # then something went wrong. while [ ! -f $cacert ] @@ -39,4 +39,3 @@ fi echo "Setting elasticsearch.password: $ELASTIC_PASSWORD" echo "$ELASTIC_PASSWORD" | /usr/share/kibana/bin/kibana-keystore add 'elasticsearch.password' -x -mv /usr/share/kibana/data/kibana.keystore /config/kibana/kibana.keystore diff --git a/scripts/setup-logstash.sh b/scripts/setup-logstash.sh index 84a7896..d9569f8 100755 --- a/scripts/setup-logstash.sh +++ b/scripts/setup-logstash.sh @@ -2,7 +2,7 @@ set -euo pipefail -cacert=/usr/share/logstash/config/ca/ca.crt +cacert=/certs/ssl/ca/ca.crt # Wait for ca file to exist before we continue. If the ca file doesn't exist # then something went wrong. while [ ! -f $cacert ] @@ -13,15 +13,15 @@ ls -l $cacert es_url=https://elasticsearch:9200 # Wait for Elasticsearch to start up before doing anything. -while [[ "$(curl -u "elastic:${ELASTIC_PASSWORD}" --cacert $cacert -s -o /dev/null -w '%{http_code}' $es_url)" != "200" ]]; do - sleep 5 +while [[ $(curl -u "elastic:${ELASTIC_PASSWORD}" --cacert $cacert -s -o /dev/null -w '%{http_code}' $es_url) != "200" ]]; do + sleep 5 done # Set the password for the logstash user. # REF: https://www.elastic.co/guide/en/x-pack/6.0/setting-up-authentication.html#set-built-in-user-passwords until curl -u "elastic:${ELASTIC_PASSWORD}" --cacert $cacert -s -H 'Content-Type:application/json' \ -XPUT $es_url/_xpack/security/user/logstash_system/_password \ - -d "{\"password\": \"${ELASTIC_PASSWORD}\"}" + -d '{"password": "${ELASTIC_PASSWORD}"}' do sleep 2 echo Retrying... @@ -36,4 +36,3 @@ fi echo "y" | /usr/share/logstash/bin/logstash-keystore create echo "Setting ELASTIC_PASSWORD..." echo "$ELASTIC_PASSWORD" | /usr/share/logstash/bin/logstash-keystore add 'ELASTIC_PASSWORD' -x -mv /usr/share/logstash/config/logstash.keystore /config/logstash/logstash.keystore diff --git a/scripts/setup-users.sh b/scripts/setup-users.sh index 32c62e9..f002f02 100755 --- a/scripts/setup-users.sh +++ b/scripts/setup-users.sh @@ -1,20 +1,21 @@ #!/bin/bash set -euo pipefail - -cacert=/config/elasticsearch/ca/ca.crt +cacert=/usr/share/elasticsearch/config/certs/ssl/ca/ca.crt # Wait for ca file to exist before we continue. If the ca file doesn't exist # then something went wrong. while [ ! -f $cacert ] do - sleep 2 + echo "No cert found at $cacert" + exit 1 done ls -l $cacert es_url=https://elastic:${ELASTIC_PASSWORD}@elasticsearch:9200 # Wait for Elasticsearch to start up before doing anything. until curl -s --cacert $cacert $es_url -o /dev/null; do - sleep 1 + sleep 3 + echo "Waiting for Elasticsearch..." done # Set the password for the kibana user. @@ -24,7 +25,7 @@ until curl --cacert $cacert -s -H 'Content-Type:application/json' \ -d "{\"password\": \"${ELASTIC_PASSWORD}\"}" do sleep 2 - echo Retrying... + echo Failed to set kibana password, retrying... done until curl --cacert $cacert -s -H 'Content-Type:application/json' \ @@ -32,5 +33,5 @@ until curl --cacert $cacert -s -H 'Content-Type:application/json' \ -d "{\"password\": \"${ELASTIC_PASSWORD}\"}" do sleep 2 - echo Retrying... + echo Failed to set logstash_system password, retrying... done diff --git a/scripts/setup.sh b/scripts/setup.sh index daccbf3..45777cf 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -1,24 +1,31 @@ -#/bin/ash -confdir="${PWD}/config" -chown 1000 -R "$confdir" -find "$confdir" -type f -name "*.keystore" -exec chmod go-wrx {} \; -find "$confdir" -type f -name "*.yml" -exec chmod go-wrx {} \; - -if [ -f "$confdir/elasticsearch/elasticsearch.keystore" ]; then - rm "$confdir/elasticsearch/elasticsearch.keystore" -fi +#/bin/bash PW=$(openssl rand -base64 16;) ELASTIC_PASSWORD="${ELASTIC_PASSWORD:-$PW}" export ELASTIC_PASSWORD -docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_elasticsearch +echo "Running 'setup-elasticsearch.sh'\n" +docker-compose run --rm -e ELASTIC_PASSWORD=$ELASTIC_PASSWORD elasticsearch /usr/local/bin/setup-elasticsearch.sh +echo "Starting Elasticsearch...." + +docker-compose up -d elasticsearch +printf "Running 'setup-users.sh'\n" +docker exec -i -e ELASTIC_PASSWORD=$ELASTIC_PASSWORD elasticsearch /usr/local/bin/setup-users.sh + +## setup kibana +printf "Running 'setup-kibana.sh'\n" +docker-compose run --rm -e ELASTIC_PASSWORD=$ELASTIC_PASSWORD kibana /usr/local/bin/setup-kibana.sh +docker-compose up -d kibana +## setup logstash +printf "Running 'setup-logstash.sh'\n" +docker-compose run --rm -u root -e ELASTIC_PASSWORD=$ELASTIC_PASSWORD logstash /usr/local/bin/setup-logstash.sh -# setup kibana and logstash (and system passwords) -docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_kibana setup_logstash -# setup beats and apm server -docker-compose -f docker-compose.yml -f docker-compose.setup.yml up setup_auditbeat setup_filebeat setup_heartbeat setup_metricbeat setup_packetbeat setup_apm_server +## setup filebeat +for service in auditbeat filebeat heartbeat packetbeat metricbeat apm-server +do + setup_command="docker-compose run --rm -u root -e ELASTIC_PASSWORD=$ELASTIC_PASSWORD $service /usr/local/bin/setup-beat.sh" + eval $setup_command +done +printf "\n\n****************************\n\n" printf "Setup completed successfully. To start the stack please run:\n\t docker-compose up -d\n" -printf "\nIf you wish to remove the setup containers please run:\n\tdocker-compose -f docker-compose.yml -f docker-compose.setup.yml down --remove-orphans\n" -printf "\nYou will have to re-start the stack after removing setup containers.\n" printf "\nYour 'elastic' user password is: $ELASTIC_PASSWORD\n" diff --git a/setup/Dockerfile b/setup/Dockerfile new file mode 100644 index 0000000..bff493b --- /dev/null +++ b/setup/Dockerfile @@ -0,0 +1,2 @@ +FROM docker/compose:1.23.2 +RUN apk update && apk add bash \ No newline at end of file diff --git a/setup.yml b/setup/setup.yml similarity index 62% rename from setup.yml rename to setup/setup.yml index 2419a4f..f6ee415 100644 --- a/setup.yml +++ b/setup/setup.yml @@ -1,7 +1,7 @@ version: "3.6" services: setup: - image: docker/compose:1.21.2 + build: . working_dir: "${PWD}" cap_add: ['SYS_ADMIN'] environment: @@ -9,10 +9,9 @@ services: - "ELASTIC_PASSWORD" - "ELASTIC_VERSION" - "TAG" + - "COMPOSE_PROJECT_NAME=stack-docker" volumes: - "/var/run/docker.sock:/var/run/docker.sock" - "${PWD}:${PWD}" - entrypoint: ["/bin/ash", "-c"] - command: ['cat ./scripts/setup.sh | tr -d "\r" | ash'] - - # command: ["./scripts/setup.sh"] + entrypoint: ["/bin/bash", "-c"] + command: ["./scripts/setup.sh"] \ No newline at end of file