merging local deployment to main

.gitignore

@@ -178,3 +178,7 @@ cython_debug/
 car-to-influx/testing_data/cleaned_can.csv
 
 /.vscode
+# /installer/startup-data-loader/data
+/installer/startup-data-loader/data
+/data
+installer/startup-data-loader/can_metrics.out

installer/.env.example

@@ -12,7 +12,8 @@ SLACK_APP_TOKEN=xapp-1-A08P01YR7M0-8764649895847-57581196f6add241ae4bba782b04500
 
 # Slack Features (Set to 'false' to disable all Slack features)
 ENABLE_SLACK=true
-# SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/WEBHOOK/URL
+SLACK_WEBHOOK_URL=https://hooks.slack.com/services/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
+
 
 # When ENABLE_SLACK=false:
 # - Slackbot container will not start

installer/cloudflared.md

@@ -0,0 +1,8 @@
+# Create DNS routes
+cloudflared tunnel route dns wfr-tunnel explore.0001200.xyz
+cloudflared tunnel route dns wfr-tunnel influxdb3.0001200.xyz
+cloudflared tunnel route dns wfr-tunnel influxdb2.0001200.xyz
+cloudflared tunnel route dns wfr-tunnel grafana.0001200.xyz
+
+# Run the tunnel
+cloudflared tunnel --config .\config.yml run wfr-tunnel

installer/config.yml

@@ -0,0 +1,33 @@
+tunnel: fcc90054-958b-4086-b105-b0417898d206
+credentials-file: C:\Users\haoru\.cloudflared\fcc90054-958b-4086-b105-b0417898d206.json
+
+ingress:
+  - hostname: explore.0001200.xyz
+    service: http://127.0.0.1:8888
+    originRequest:
+      noTLSVerify: true
+      headers:
+        Host: localhost
+
+  - hostname: influxdb3.0001200.xyz
+    service: http://127.0.0.1:9000
+    originRequest:
+      noTLSVerify: true
+      headers:
+        Host: localhost
+
+  - hostname: influxdb2.0001200.xyz
+    service: http://127.0.0.1:8086
+    originRequest:
+      noTLSVerify: true
+      headers:
+        Host: localhost
+
+  - hostname: grafana.0001200.xyz
+    service: http://127.0.0.1:8087
+    originRequest:
+      noTLSVerify: true
+      headers:
+        Host: localhost
+
+  - service: http_status:404

installer/docker-compose-server.yml

@@ -0,0 +1,193 @@
+# SERVER: AWS
+
+networks:
+  datalink:
+    driver: bridge
+
+# 2 GB RAM, 2 vCPUs, 60 GB SSD
+
+volumes:
+  grafana-storage:
+  influxdb3-data:
+  influxdb3-explorer-db:
+  telegraf-data:
+
+# in previous testings, we used influxdb2 as the container name for influxdb2
+# changing it to influxdb2 now to be more standard
+services:
+  influxdb3:
+    image: influxdb:3.5.0-core
+    container_name: influxdb3
+    command: >
+      influxdb3 serve
+      --node-id influxdb3-node
+      --object-store file
+      --data-dir /var/lib/influxdb3
+      --admin-token-file /influxdb3-admin-token.json
+    restart: unless-stopped
+    ports:
+      - "9000:8181"
+    environment:
+      INFLUX_URL: "http://influxdb3:8181"
+      INFLUXDB_INIT_ADMIN_TOKEN: "wfr-admin-token-change-in-production"
+    volumes:
+      - influxdb3-data:/var/lib/influxdb3
+      - ./influxdb3-admin-token.json:/influxdb3-admin-token.json:ro
+    networks:
+      - datalink
+    healthcheck:
+      test: ["CMD-SHELL", "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/localhost/8181' || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          memory: 1g
+          cpus: "1.0"
+
+  influxdb3-explorer:
+    image: influxdata/influxdb3-ui:1.3.0
+    container_name: influxdb3-explorer
+    restart: unless-stopped
+    pull_policy: always
+    ports:
+      - "8888:80"
+    environment:
+      SESSION_SECRET_KEY: "${EXPLORER_SESSION_SECRET:-wfr-explorer-session-key-change-in-production}"
+    volumes:
+      - influxdb3-explorer-db:/db:rw
+      - ./influxdb3-explorer-config:/app-root/config:ro
+    command: ["--mode=admin"]
+    networks:
+      - datalink
+    depends_on:
+      - influxdb3
+    deploy:
+      resources:
+        limits:
+          memory: 1g
+          cpus: "1.0"
+
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    restart: unless-stopped
+    ports:
+      - "8087:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: "admin"
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_SECURITY_ADMIN_EMAIL: daq@westernformularacing.com
+      GF_AUTH_ANONYMOUS_ENABLED: "false"
+      GF_AUTH_BASIC_ENABLED: "true"
+      GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource
+      GF_PATHS_PROVISIONING: /etc/grafana/provisioning
+      INFLUXDB_TOKEN: "${INFLUXDB_ADMIN_TOKEN:-wfr-admin-token-change-in-production}"
+      GF_DASHBOARDS_MIN_REFRESH_INTERVAL: 100ms
+    volumes:
+      - grafana-storage:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    networks:
+      - datalink
+    depends_on:
+      influxdb3:
+        condition: service_healthy
+    #deploy:
+      #resources:
+        #limits:
+          #memory: 1g
+          #cpus: "1.0"
+
+
+
+  slackbot:
+    build: ./slackbot   # put Dockerfile + slack_bot.py here
+    container_name: slackbot
+    restart: always
+    environment:
+      SLACK_BOT_TOKEN: ${SLACK_BOT_TOKEN:-your-slack-bot-token-here}
+      SLACK_APP_TOKEN: ${SLACK_APP_TOKEN:-your-slack-app-token-here}
+      INFLUXDB_ADMIN_TOKEN: "${INFLUXDB_ADMIN_TOKEN:-wfr-admin-token-change-in-production}"
+      INFLUXDB_URL: "http://influxdb3:9000"
+      INFLUXDB_ORG: "WFR"
+      INFLUXDB_BUCKET: "ourCar"
+      SLACK_WEBHOOK_URL: "${SLACK_WEBHOOK_URL:-}"
+    volumes:
+      - ./slackbot:/app
+    working_dir: /app
+    command: python slack_bot.py
+    networks:
+      - datalink
+    deploy:
+      resources:
+        limits:
+          cpus: "0.25"
+
+
+  startup-data-loader:
+    build: ./startup-data-loader
+    container_name: startup-data-loader
+    restart: "no"  # Run once only
+    environment:
+      INFLUXDB_TOKEN: "${INFLUXDB_ADMIN_TOKEN:-wfr-admin-token-change-in-production}"
+      BACKFILL: "1"  # Enable backfill mode for direct InfluxDB writing
+    volumes:
+      - ../data:/data:ro  # Mount data folder as read-only
+      - telegraf-data:/var/lib/telegraf  # Shared volume with telegraf
+      - ./startup-data-loader:/app  # Mount the app directory
+    working_dir: /app
+    networks:
+      - datalink
+    depends_on:
+      influxdb3:
+        condition: service_healthy
+    command: >
+      sh -c "echo 'Waiting additional 5 seconds for InfluxDB to stabilize...' &&
+             sleep 5 &&
+             echo 'Creating telegraf input file (this may take a while for large files)...' &&
+             python load_data_telegraf.py &&
+             echo 'Telegraf input file created successfully'"
+
+    # python load_data_influxdb2.py & python load_data_telegraf.py & wait"
+    deploy:
+      resources:
+        limits:
+          memory: 512m
+          cpus: "0.5"
+
+  file-uploader:
+    build: ./file-uploader
+    container_name: file-uploader
+    ports:
+      - "8084:8084"
+    volumes:
+      - ./file-uploader:/app
+    restart: unless-stopped
+    networks:
+      - datalink
+    environment:
+      - INFLUXDB_TOKEN=${INFLUXDB_ADMIN_TOKEN:-wfr-admin-token-change-in-production}
+      - WEBHOOK_URL=https://hooks.slack.com/services/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
+    deploy:
+      resources:
+        limits:
+          cpus: "0.5"
+          memory: 512m
+
+#  artifact-viewer:
+#    image: artifact-viewer-dev
+#    container_name: artifact-viewer
+#    restart: unless-stopped
+#    ports:
+#      - "2990:2990"
+#    networks:
+#      - datalink
+#    deploy:
+#      resources:
+#        limits:
+#          memory: 512m
+#          cpus: "0.5"