Migrating kafka from zookeper to kraft

rdbms-image/docker-compose.yml

@@ -10,20 +10,32 @@ services:
     profiles:
       - infra
       - all
+  zookeeper-2:
+    image: docker.io/bitnami/zookeeper:3.8
+    ports:
+      - "2182:2182"
+    restart: unless-stopped
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+    profiles:
+      - infra
+      - all
+      - local
 
   kafka:
     image: devicehive/devicehive-kafka:${DH_TAG:-3.5.0}
     ports:
       - "9092:9092"
     links:
       - "zookeeper"
+      - "zookeeper-2"
     restart: unless-stopped
     environment:
-      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT
-      - KAFKA_CFG_LISTENERS=CLIENT://:9092
-      - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:9092
-      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT
-      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181,zookeeper-2:2182
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT
+      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka:9092
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
+      - KAFKA_CFG_LISTENERS=INTERNAL://:9092
       - ALLOW_PLAINTEXT_LISTENER=yes
     healthcheck:
       test: /bin/nc -z localhost 9092
@@ -33,6 +45,29 @@ services:
     profiles:
       - infra
       - all
+  kafka-2:
+    image: devicehive-kafka:local
+    ports:
+      - "9093:9093"
+    links:
+      - "zookeeper"
+      - "zookeeper-2"
+    restart: unless-stopped
+    environment:
+      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181,zookeeper-2:2182
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT
+      - KAFKA_CFG_ADVERTISED_LISTENERS=INTERNAL://kafka-2:9092
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INTERNAL
+      - KAFKA_CFG_LISTENERS=INTERNAL://:9092
+      - ALLOW_PLAINTEXT_LISTENER=yes
+    healthcheck:
+      test: /bin/nc -z localhost 9093
+      interval: 2s
+      timeout: 2s
+      retries: 15
+    profiles:
+      - infra
+      - all
 
   postgres:
     image: postgres:14.4
@@ -55,17 +90,19 @@ services:
       - "3000:3000"
     links:
       - "kafka"
+      - "kafka-2"
 
     depends_on:
       kafka:
         condition: service_healthy
 
     restart: unless-stopped
     environment:
-      FRANZ.KAFKA_HOSTS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
-      PROXY.WEB_SOCKET_SERVER_HOST: 'wsproxy'
-      PROXY.ENABLE_PLUGIN_MANAGER: 'false'
       PROXY.APP_LOG_LEVEL: 'debug'
+      ENVSEPARATOR : _
+      FRANZ_KAFKA_HOSTS: ${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
+      PROXY_WEB_SOCKET_SERVER_HOST: 'wsproxy'
+      PROXY_ENABLE_PLUGIN_MANAGER: 'false'
     profiles:
       - infra
       - all
@@ -118,6 +155,7 @@ services:
       DH_AUTH_URL: http://dh_auth:8090/auth/rest
       REDIS_MASTER_HOST: ${REDIS_MASTER_HOST:-redis_master}
       REDIS_MASTER_PORT: ${REDIS_MASTER_PORT:-6379}
+      REDIS_MASTER_PASSWORD: ${REDIS_MASTER_PASSWORD}
       JWT_SECRET: ${JWT_SECRET}
       SPRING_PROFILES_ACTIVE: ${DH_PLUGIN_SPRING_PROFILES_ACTIVE:-ws-kafka-proxy}
     profiles:
@@ -152,6 +190,9 @@ services:
       DH_AUTH_URL: ${DH_AUTH_URL:-http://dh_auth:8090/auth/rest}
       REDIS_MASTER_HOST: ${REDIS_MASTER_HOST:-redis_master}
       REDIS_MASTER_PORT: ${REDIS_MASTER_PORT:-6379}
+      REDIS_MASTER_PASSWORD: ${REDIS_MASTER_PASSWORD}
+      DH_BACKEND_ADDRESS: ${DH_BACKEND_ADDRESS:-dh_backend}
+      DH_BACKEND_PORT: ${DH_BACKEND_PORT:-8000}
       JWT_SECRET: ${JWT_SECRET}
       SPRING_PROFILES_ACTIVE: ${DH_FE_SPRING_PROFILES_ACTIVE:-ws-kafka-proxy-frontend}
     profiles:
@@ -182,6 +223,7 @@ services:
       ROOT_LOG_LEVEL: ${ROOT_LOG_LEVEL:-WARN}
       REDIS_MASTER_HOST: ${REDIS_MASTER_HOST:-redis_master}
       REDIS_MASTER_PORT: ${REDIS_MASTER_PORT:-6379}
+      REDIS_MASTER_PASSWORD: ${REDIS_MASTER_PASSWORD}
       JWT_SECRET: ${JWT_SECRET}
       DH_WS_PROXY: wsproxy:3000
       SPRING_PROFILES_ACTIVE: ${DH_AUTH_SPRING_PROFILES_ACTIVE:-ws-kafka-proxy}
@@ -241,6 +283,7 @@ services:
       DH_WS_PROXY: wsproxy:3000
       REDIS_MASTER_HOST: ${REDIS_MASTER_HOST:-redis_master}
       REDIS_MASTER_PORT: ${REDIS_MASTER_PORT:-6379}
+      REDIS_MASTER_PASSWORD: ${REDIS_MASTER_PASSWORD}
       ROOT_LOG_LEVEL: ${ROOT_LOG_LEVEL:-INFO}
       SPRING_PROFILES_ACTIVE: ${DH_BE_SPRING_PROFILES_ACTIVE:-ws-kafka-proxy-backend}
     profiles:
@@ -260,6 +303,19 @@ services:
       - dev
       - all
 
+  kafdrop:
+    image: 'obsidiandynamics/kafdrop:4.0.1'
+    restart: "no"
+    ports:
+      - "9000:9000"
+
+    environment:
+      - KAFKA_BROKERCONNECT=${DH_KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
+      - JVM_OPTS=-Xms32M -Xmx64M
+      - SERVER_SERVLET_CONTEXTPATH=/
+
+    container_name: kafdrop
+
 volumes:
   devicehive-db:
   devicehive-redis: