From 7856f3b9b26c58212e0d21c536170a8955c72432 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 17:29:45 +0300 Subject: [PATCH 01/29] 288: traefik to deployment --- infra/deploy.py | 4 +++ infra/docker-compose.yaml | 72 ++++++++++++++++++++++++++------------- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/infra/deploy.py b/infra/deploy.py index 3e509dd4..a6b3662f 100644 --- a/infra/deploy.py +++ b/infra/deploy.py @@ -56,6 +56,10 @@ def get_spec(params: EnvParams) -> deployment.RemoteSpec: "postgres/migrations", "postgres/migrations", ), + deployment.RemoteFile( + "postgres/dockerfile", + "postgres/dockerfile" + ), deployment.RemoteDirectory( "infra/configs/nginx", "configs", diff --git a/infra/docker-compose.yaml b/infra/docker-compose.yaml index ed1f331f..f7d545b8 100644 --- a/infra/docker-compose.yaml +++ b/infra/docker-compose.yaml @@ -10,6 +10,8 @@ services: volumes: - ./postgres/data:/var/lib/postgresql/data restart: unless-stopped + networks: + - hyperleda-network healthcheck: test: psql 'host=localhost port=5432 dbname=hyperleda user=hyperleda password=$${POSTGRES_PASSWORD}' -qtA -c 'select 1;' || exit 1 timeout: 5s @@ -27,6 +29,8 @@ services: depends_on: hyperledadb: condition: service_healthy + networks: + - hyperleda-network wait-for-migrate: image: busybox @@ -34,6 +38,8 @@ services: migrate: condition: service_completed_successfully entrypoint: /bin/sh -c "echo 'Migration completed successfully'" + networks: + - hyperleda-network adminapi: image: ghcr.io/hyperleda/hyperleda:latest @@ -45,11 +51,14 @@ services: volumes: - ./configs:/usr/src/app/configs restart: unless-stopped + networks: + - hyperleda-network healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/ping"] - interval: 5s - timeout: 5s - retries: 5 + interval: 10s + timeout: 10s + retries: 3 + start_period: 30s depends_on: - wait-for-migrate logging: @@ -60,37 +69,36 @@ services: entrypoint: ["uv", "run", "main.py", "dataapi"] environment: - CONFIG=configs/dataapi.yaml - - SERVER_PORT=8081 - STORAGE_PASSWORD=${POSTGRES_PASSWORD:-password} volumes: - ./configs:/usr/src/app/configs restart: unless-stopped + networks: + - hyperleda-network healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8081/ping"] - interval: 5s - timeout: 5s - retries: 5 + test: ["CMD", "curl", "-f", "http://localhost:8080/ping"] + interval: 10s + timeout: 10s + retries: 3 + start_period: 30s depends_on: - - wait-for-migrate - logging: - driver: local + wait-for-migrate: + condition: service_completed_successfully webapp: image: ghcr.io/hyperleda/hyperleda-webapp:latest - depends_on: - dataapi: - condition: service_healthy + networks: + - hyperleda-network healthcheck: - test: ["CMD", "curl", "-f", "-LI", "http://localhost:3000/"] - interval: 5s - timeout: 5s - retries: 5 + test: ["CMD", "curl", "-f", "http://localhost:3000"] + interval: 15s + timeout: 15s + retries: 3 + start_period: 60s restart: unless-stopped nginx: image: nginx:latest - ports: - - "81:80" volumes: - ./configs/nginx.conf:/etc/nginx/conf.d/default.conf - ./logs:/var/log/nginx @@ -100,12 +108,28 @@ services: dataapi: condition: service_healthy webapp: - condition: service_healthy + condition: service_completed_successfully restart: unless-stopped + networks: + - hyperleda-network + - proxynet healthcheck: test: ["CMD", "curl", "-f", "http://localhost:80/ping"] - interval: 5s - timeout: 5s - retries: 5 + interval: 10s + timeout: 10s + retries: 3 + start_period: 40s logging: driver: local + labels: + - "traefik.enable=true" + - "traefik.http.routers.nginx.rule=Host(`leda.sao.ru`)" + - "traefik.http.routers.nginx.entrypoints=https" + - "traefik.http.services.nginx.loadbalancer.server.port=80" + - "traefik.docker.network=proxynet" + +networks: + hyperleda-network: + driver: bridge + proxynet: + external: true \ No newline at end of file From f252f5ebf25c4423efcf49d196b65bf169916a01 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 17:38:34 +0300 Subject: [PATCH 02/29] fix wait --- infra/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/docker-compose.yaml b/infra/docker-compose.yaml index f7d545b8..8ce42b21 100644 --- a/infra/docker-compose.yaml +++ b/infra/docker-compose.yaml @@ -108,7 +108,7 @@ services: dataapi: condition: service_healthy webapp: - condition: service_completed_successfully + condition: service_healthy restart: unless-stopped networks: - hyperleda-network From a9eec8dba2d639ebc3c16807b20faca30806ea0a Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 18:16:19 +0300 Subject: [PATCH 03/29] fix healthchecks --- infra/docker-compose.yaml | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/infra/docker-compose.yaml b/infra/docker-compose.yaml index 8ce42b21..8f332ad1 100644 --- a/infra/docker-compose.yaml +++ b/infra/docker-compose.yaml @@ -55,10 +55,9 @@ services: - hyperleda-network healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/ping"] - interval: 10s - timeout: 10s - retries: 3 - start_period: 30s + interval: 5s + timeout: 5s + retries: 5 depends_on: - wait-for-migrate logging: @@ -76,10 +75,10 @@ services: networks: - hyperleda-network healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/ping"] - interval: 10s - timeout: 10s - retries: 3 + test: ["CMD", "curl", "-f", "http://localhost:8081/ping"] + interval: 5s + timeout: 5s + retries: 5 start_period: 30s depends_on: wait-for-migrate: @@ -90,11 +89,10 @@ services: networks: - hyperleda-network healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000"] - interval: 15s - timeout: 15s - retries: 3 - start_period: 60s + test: ["CMD", "curl", "-f", "-LI", "http://localhost:3000/"] + interval: 5s + timeout: 5s + retries: 5 restart: unless-stopped nginx: @@ -115,10 +113,9 @@ services: - proxynet healthcheck: test: ["CMD", "curl", "-f", "http://localhost:80/ping"] - interval: 10s - timeout: 10s - retries: 3 - start_period: 40s + interval: 5s + timeout: 5s + retries: 5 logging: driver: local labels: From 538f059423fd60b5a2ed35467303b208f986322a Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 18:19:08 +0300 Subject: [PATCH 04/29] add logging drivers --- infra/docker-compose.yaml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/infra/docker-compose.yaml b/infra/docker-compose.yaml index 8f332ad1..70e2a9ce 100644 --- a/infra/docker-compose.yaml +++ b/infra/docker-compose.yaml @@ -17,6 +17,8 @@ services: timeout: 5s interval: 5s retries: 5 + logging: + driver: local migrate: build: @@ -31,6 +33,8 @@ services: condition: service_healthy networks: - hyperleda-network + logging: + driver: local wait-for-migrate: image: busybox @@ -40,6 +44,8 @@ services: entrypoint: /bin/sh -c "echo 'Migration completed successfully'" networks: - hyperleda-network + logging: + driver: local adminapi: image: ghcr.io/hyperleda/hyperleda:latest @@ -79,10 +85,10 @@ services: interval: 5s timeout: 5s retries: 5 - start_period: 30s depends_on: - wait-for-migrate: - condition: service_completed_successfully + - wait-for-migrate + logging: + driver: local webapp: image: ghcr.io/hyperleda/hyperleda-webapp:latest @@ -94,6 +100,8 @@ services: timeout: 5s retries: 5 restart: unless-stopped + logging: + driver: local nginx: image: nginx:latest From 4672326ef2f67f39b6b9c1a861b333d49a4eef9b Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 18:22:21 +0300 Subject: [PATCH 05/29] modify nginx config --- infra/configs/nginx/nginx.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/infra/configs/nginx/nginx.conf b/infra/configs/nginx/nginx.conf index 09a836cb..49ab202b 100644 --- a/infra/configs/nginx/nginx.conf +++ b/infra/configs/nginx/nginx.conf @@ -17,7 +17,7 @@ server { } location /api/ { - proxy_pass http://dataapi:8081/api/; + proxy_pass http://dataapi:8080/api/; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -28,5 +28,8 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_intercept_errors on; + error_page 404 = /index.html; } } \ No newline at end of file From 48410ee31fedf8917cc8b69ace1e93e2c76c84d5 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 18:25:41 +0300 Subject: [PATCH 06/29] fix --- infra/configs/nginx/nginx.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/infra/configs/nginx/nginx.conf b/infra/configs/nginx/nginx.conf index 49ab202b..8e9696af 100644 --- a/infra/configs/nginx/nginx.conf +++ b/infra/configs/nginx/nginx.conf @@ -28,8 +28,5 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - proxy_intercept_errors on; - error_page 404 = /index.html; } } \ No newline at end of file From 75eade68c78bdbeb343dd2c11903c1eebb2b6cdd Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 18:48:27 +0300 Subject: [PATCH 07/29] add deployment tests --- .github/workflows/deployment-testing.yml | 72 ++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/deployment-testing.yml diff --git a/.github/workflows/deployment-testing.yml b/.github/workflows/deployment-testing.yml new file mode 100644 index 00000000..c5dea203 --- /dev/null +++ b/.github/workflows/deployment-testing.yml @@ -0,0 +1,72 @@ +name: Test Deployment + +on: + - push + +jobs: + test-deployment: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install project dependencies + run: uv sync --all-extras --dev + + - name: Generate SSH key pair for testing + run: | + ssh-keygen -t rsa -b 4096 -f /tmp/test_key -N "" -C "test@github.com" + chmod 600 /tmp/test_key + chmod 644 /tmp/test_key.pub + + - name: Start SSH server container for testing + run: | + docker run -d \ + --name test-container \ + --network host \ + -v /tmp:/tmp \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e PUID=1000 \ + -e PGID=1000 \ + -e USER_NAME=testuser \ + -e SUDO_ACCESS=true \ + -e PASSWORD_ACCESS=true \ + -e USER_PASSWORD=testpass \ + linuxserver/openssh-server:latest + + - name: Setup SSH key authentication + run: | + docker exec test-container bash -c " + mkdir -p /home/testuser/.ssh && + cp /tmp/test_key.pub /home/testuser/.ssh/authorized_keys && + chown -R testuser:testuser /home/testuser/.ssh && + chmod 700 /home/testuser/.ssh && + chmod 600 /home/testuser/.ssh/authorized_keys + " + + - name: Wait for SSH to be ready + run: | + for i in {1..30}; do + if ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i /tmp/test_key testuser@localhost "echo 'SSH ready'" 2>/dev/null; then + echo "SSH is ready" + break + fi + echo "Waiting for SSH... ($i/30)" + sleep 2 + done + + - name: Run deployment test + run: | + cd infra + uv run python deploy.py settings/dev.yaml + + - name: Start services and test /ping endpoints + run: | + ssh -o StrictHostKeyChecking=no -i /tmp/test_key testuser@localhost " + cd /tmp/hyperleda_test && + curl -f http://localhost:80/ping || echo 'Nginx ping failed' + " From be0437d8c11d599637911747d7db11bc940c20e9 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 18:51:17 +0300 Subject: [PATCH 08/29] configs --- .github/workflows/deployment-testing.yml | 14 +------------- infra/configs/dev.yaml | 9 +++++++++ infra/deploy.py | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) create mode 100644 infra/configs/dev.yaml diff --git a/.github/workflows/deployment-testing.yml b/.github/workflows/deployment-testing.yml index c5dea203..c1484a65 100644 --- a/.github/workflows/deployment-testing.yml +++ b/.github/workflows/deployment-testing.yml @@ -48,21 +48,9 @@ jobs: chmod 600 /home/testuser/.ssh/authorized_keys " - - name: Wait for SSH to be ready - run: | - for i in {1..30}; do - if ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i /tmp/test_key testuser@localhost "echo 'SSH ready'" 2>/dev/null; then - echo "SSH is ready" - break - fi - echo "Waiting for SSH... ($i/30)" - sleep 2 - done - - name: Run deployment test run: | - cd infra - uv run python deploy.py settings/dev.yaml + uv run python infra/deploy.py infra/configs/dev.yaml - name: Start services and test /ping endpoints run: | diff --git a/infra/configs/dev.yaml b/infra/configs/dev.yaml new file mode 100644 index 00000000..a35a66d7 --- /dev/null +++ b/infra/configs/dev.yaml @@ -0,0 +1,9 @@ +connection: + host: localhost + user: testuser + private_key_filename: /tmp/test_key + +remote_base_path: /tmp/hyperleda_test +configs_env: test +ads_token: test_token +postgres_password: test_password diff --git a/infra/deploy.py b/infra/deploy.py index a6b3662f..d3786533 100644 --- a/infra/deploy.py +++ b/infra/deploy.py @@ -58,7 +58,7 @@ def get_spec(params: EnvParams) -> deployment.RemoteSpec: ), deployment.RemoteFile( "postgres/dockerfile", - "postgres/dockerfile" + "postgres/dockerfile", ), deployment.RemoteDirectory( "infra/configs/nginx", From cbf1082235e453100c3c740d8985e1cc111fa3cc Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 18:56:06 +0300 Subject: [PATCH 09/29] fix ssh-copy-id --- ...ployment-testing.yml => infra-testing.yml} | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) rename .github/workflows/{deployment-testing.yml => infra-testing.yml} (54%) diff --git a/.github/workflows/deployment-testing.yml b/.github/workflows/infra-testing.yml similarity index 54% rename from .github/workflows/deployment-testing.yml rename to .github/workflows/infra-testing.yml index c1484a65..95ef1998 100644 --- a/.github/workflows/deployment-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -1,4 +1,4 @@ -name: Test Deployment +name: Infra testing on: - push @@ -23,7 +23,7 @@ jobs: chmod 600 /tmp/test_key chmod 644 /tmp/test_key.pub - - name: Start SSH server container for testing + - name: Start server container run: | docker run -d \ --name test-container \ @@ -32,29 +32,20 @@ jobs: -v /var/run/docker.sock:/var/run/docker.sock \ -e PUID=1000 \ -e PGID=1000 \ - -e USER_NAME=testuser \ + -e USER_NAME=tester \ -e SUDO_ACCESS=true \ -e PASSWORD_ACCESS=true \ - -e USER_PASSWORD=testpass \ + -e USER_PASSWORD=password \ linuxserver/openssh-server:latest - - name: Setup SSH key authentication + - name: Setup add key to server run: | - docker exec test-container bash -c " - mkdir -p /home/testuser/.ssh && - cp /tmp/test_key.pub /home/testuser/.ssh/authorized_keys && - chown -R testuser:testuser /home/testuser/.ssh && - chmod 700 /home/testuser/.ssh && - chmod 600 /home/testuser/.ssh/authorized_keys - " - - - name: Run deployment test + ssh-copy-id -f -i /tmp/test_key.pub localhost + + - name: Deploy run: | uv run python infra/deploy.py infra/configs/dev.yaml - - name: Start services and test /ping endpoints + - name: Test endpoint run: | - ssh -o StrictHostKeyChecking=no -i /tmp/test_key testuser@localhost " - cd /tmp/hyperleda_test && - curl -f http://localhost:80/ping || echo 'Nginx ping failed' - " + ssh -o StrictHostKeyChecking=no -i /tmp/test_key tester@localhost "curl -f http://localhost:80/ping" From 4f4033c5412ef59ce340ba0bfb228aa27da31c83 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 18:59:35 +0300 Subject: [PATCH 10/29] fix roles --- .github/workflows/infra-testing.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 95ef1998..6f14bf7e 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -40,6 +40,8 @@ jobs: - name: Setup add key to server run: | + mkdir -p ~/.ssh + chmod 700 ~/.ssh ssh-copy-id -f -i /tmp/test_key.pub localhost - name: Deploy From a1d43ba7274a251914c86f3ee34ce4b5da90b1ea Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:00:37 +0300 Subject: [PATCH 11/29] fix host --- .github/workflows/infra-testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 6f14bf7e..072f531c 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -42,6 +42,7 @@ jobs: run: | mkdir -p ~/.ssh chmod 700 ~/.ssh + ssh-keyscan -H localhost >> ~/.ssh/known_hosts ssh-copy-id -f -i /tmp/test_key.pub localhost - name: Deploy From c10ed05006a8f9093c0532fc5b26e598e6cbac65 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:03:16 +0300 Subject: [PATCH 12/29] use sshpass --- .github/workflows/infra-testing.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 072f531c..1707920f 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -17,12 +17,6 @@ jobs: - name: Install project dependencies run: uv sync --all-extras --dev - - name: Generate SSH key pair for testing - run: | - ssh-keygen -t rsa -b 4096 -f /tmp/test_key -N "" -C "test@github.com" - chmod 600 /tmp/test_key - chmod 644 /tmp/test_key.pub - - name: Start server container run: | docker run -d \ @@ -38,12 +32,10 @@ jobs: -e USER_PASSWORD=password \ linuxserver/openssh-server:latest - - name: Setup add key to server + - name: Setup client run: | - mkdir -p ~/.ssh - chmod 700 ~/.ssh + sudo apt-get update && sudo apt-get install -y sshpass ssh-keyscan -H localhost >> ~/.ssh/known_hosts - ssh-copy-id -f -i /tmp/test_key.pub localhost - name: Deploy run: | @@ -51,4 +43,4 @@ jobs: - name: Test endpoint run: | - ssh -o StrictHostKeyChecking=no -i /tmp/test_key tester@localhost "curl -f http://localhost:80/ping" + sshpass -p password ssh -o StrictHostKeyChecking=no tester@localhost "curl -f http://localhost:80/ping" From 6d0ee2d564d3dfc9afafa6586eaaaf1a98c9daa6 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:04:25 +0300 Subject: [PATCH 13/29] ssh dir --- .github/workflows/infra-testing.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 1707920f..bad99141 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -34,7 +34,8 @@ jobs: - name: Setup client run: | - sudo apt-get update && sudo apt-get install -y sshpass + mkdir -p ~/.ssh + chmod 700 ~/.ssh ssh-keyscan -H localhost >> ~/.ssh/known_hosts - name: Deploy From 4e809be8ef4f5c8ad6b8b870718798de9fa57faa Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:06:44 +0300 Subject: [PATCH 14/29] fix key auth --- infra/configs/dev.yaml | 4 ++-- infra/deployment.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/infra/configs/dev.yaml b/infra/configs/dev.yaml index a35a66d7..1a096e41 100644 --- a/infra/configs/dev.yaml +++ b/infra/configs/dev.yaml @@ -1,7 +1,7 @@ connection: host: localhost - user: testuser - private_key_filename: /tmp/test_key + user: tester + password: password remote_base_path: /tmp/hyperleda_test configs_env: test diff --git a/infra/deployment.py b/infra/deployment.py index 8e07bfee..db5ff385 100644 --- a/infra/deployment.py +++ b/infra/deployment.py @@ -76,7 +76,8 @@ def to_files(self) -> list[RemoteFile]: class ConnectionContext: host: str user: str - private_key_filename: str + private_key_filename: str | None = None + password: str | None = None def _run_command( @@ -145,10 +146,16 @@ def __str__(self) -> str: return "\n".join(lines) def apply(self, ctx: ConnectionContext, logger: structlog.stdlib.BoundLogger): + connect_kwargs = {} + if ctx.private_key_filename: + connect_kwargs["key_filename"] = ctx.private_key_filename + if ctx.password: + connect_kwargs["password"] = ctx.password + self.connection = Connection( host=ctx.host, user=ctx.user, - connect_kwargs={"key_filename": ctx.private_key_filename}, + connect_kwargs=connect_kwargs, ) self.logger = logger From da6ac07817f16c90d9827268a9a5dde5bdd039ef Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:12:19 +0300 Subject: [PATCH 15/29] use ssh key to authenticate --- .github/workflows/infra-testing.yml | 12 +++++++++--- infra/configs/dev.yaml | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index bad99141..4032c976 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -17,6 +17,12 @@ jobs: - name: Install project dependencies run: uv sync --all-extras --dev + - name: Generate SSH key pair + run: | + ssh-keygen -t rsa -b 4096 -f ~/.ssh/test_key -N "" -C "test@github-actions" + chmod 600 ~/.ssh/test_key + chmod 644 ~/.ssh/test_key.pub + - name: Start server container run: | docker run -d \ @@ -28,8 +34,7 @@ jobs: -e PGID=1000 \ -e USER_NAME=tester \ -e SUDO_ACCESS=true \ - -e PASSWORD_ACCESS=true \ - -e USER_PASSWORD=password \ + -e PUBLIC_KEY_FILE=/tmp/test_key.pub \ linuxserver/openssh-server:latest - name: Setup client @@ -37,6 +42,7 @@ jobs: mkdir -p ~/.ssh chmod 700 ~/.ssh ssh-keyscan -H localhost >> ~/.ssh/known_hosts + cp ~/.ssh/test_key.pub /tmp/test_key.pub - name: Deploy run: | @@ -44,4 +50,4 @@ jobs: - name: Test endpoint run: | - sshpass -p password ssh -o StrictHostKeyChecking=no tester@localhost "curl -f http://localhost:80/ping" + ssh -i ~/.ssh/test_key -o StrictHostKeyChecking=no tester@localhost "curl -f http://localhost:80/ping" diff --git a/infra/configs/dev.yaml b/infra/configs/dev.yaml index 1a096e41..b0991008 100644 --- a/infra/configs/dev.yaml +++ b/infra/configs/dev.yaml @@ -1,7 +1,7 @@ connection: host: localhost user: tester - password: password + private_key_filename: /tmp/test_key remote_base_path: /tmp/hyperleda_test configs_env: test From 4b43bd69a6cc164bf3e8f94da0485fac705ff3d2 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:38:51 +0300 Subject: [PATCH 16/29] fix key --- .github/workflows/infra-testing.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 4032c976..2e8f6e5b 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -43,6 +43,8 @@ jobs: chmod 700 ~/.ssh ssh-keyscan -H localhost >> ~/.ssh/known_hosts cp ~/.ssh/test_key.pub /tmp/test_key.pub + cp ~/.ssh/test_key /tmp/test_key + chmod 600 /tmp/test_key - name: Deploy run: | From 78214e73e1f43193a7411411cfc88c84f147e3c6 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:41:31 +0300 Subject: [PATCH 17/29] fix location --- .github/workflows/infra-testing.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 2e8f6e5b..4ba03922 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -19,6 +19,8 @@ jobs: - name: Generate SSH key pair run: | + mkdir -p ~/.ssh + chmod 700 ~/.ssh ssh-keygen -t rsa -b 4096 -f ~/.ssh/test_key -N "" -C "test@github-actions" chmod 600 ~/.ssh/test_key chmod 644 ~/.ssh/test_key.pub @@ -37,15 +39,15 @@ jobs: -e PUBLIC_KEY_FILE=/tmp/test_key.pub \ linuxserver/openssh-server:latest - - name: Setup client + - name: Setup and test client run: | - mkdir -p ~/.ssh - chmod 700 ~/.ssh ssh-keyscan -H localhost >> ~/.ssh/known_hosts cp ~/.ssh/test_key.pub /tmp/test_key.pub cp ~/.ssh/test_key /tmp/test_key chmod 600 /tmp/test_key + ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 tester@localhost "echo 'SSH connection successful'" + - name: Deploy run: | uv run python infra/deploy.py infra/configs/dev.yaml From 50636b67a5da924582d020e6a27c144e8736b3b6 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:43:42 +0300 Subject: [PATCH 18/29] change stuff --- .github/workflows/infra-testing.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 4ba03922..fd8a9d9d 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -21,10 +21,15 @@ jobs: run: | mkdir -p ~/.ssh chmod 700 ~/.ssh - ssh-keygen -t rsa -b 4096 -f ~/.ssh/test_key -N "" -C "test@github-actions" + ssh-keygen -m PEM -t rsa -b 2048 -f ~/.ssh/test_key -N "" -C "test@github-actions" chmod 600 ~/.ssh/test_key chmod 644 ~/.ssh/test_key.pub + ssh-keyscan -H localhost >> ~/.ssh/known_hosts + cp ~/.ssh/test_key.pub /tmp/test_key.pub + cp ~/.ssh/test_key /tmp/test_key + chmod 600 /tmp/test_key + - name: Start server container run: | docker run -d \ @@ -41,11 +46,6 @@ jobs: - name: Setup and test client run: | - ssh-keyscan -H localhost >> ~/.ssh/known_hosts - cp ~/.ssh/test_key.pub /tmp/test_key.pub - cp ~/.ssh/test_key /tmp/test_key - chmod 600 /tmp/test_key - ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 tester@localhost "echo 'SSH connection successful'" - name: Deploy From dcbbae02ced4206c838e441b2b95224c5a2669b2 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:45:56 +0300 Subject: [PATCH 19/29] add wait --- .github/workflows/infra-testing.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index fd8a9d9d..563fb111 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -44,6 +44,28 @@ jobs: -e PUBLIC_KEY_FILE=/tmp/test_key.pub \ linuxserver/openssh-server:latest + - name: Wait for SSH server to be ready + run: | + echo "Waiting for SSH server to start..." + for i in {1..30}; do + if docker exec test-container pgrep sshd > /dev/null 2>&1; then + echo "SSH daemon is running" + break + fi + echo "Attempt $i/30: SSH daemon not ready yet" + sleep 2 + done + + echo "Waiting for SSH to accept connections..." + for i in {1..30}; do + if ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 -o BatchMode=yes tester@localhost "echo 'test'" > /dev/null 2>&1; then + echo "SSH connection successful" + break + fi + echo "Attempt $i/30: SSH connection failed" + sleep 2 + done + - name: Setup and test client run: | ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 tester@localhost "echo 'SSH connection successful'" From 836ead4c912c6887a9c1834939b5ad62672e03f5 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:48:06 +0300 Subject: [PATCH 20/29] logs --- .github/workflows/infra-testing.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 563fb111..5dc75ba2 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -42,6 +42,7 @@ jobs: -e USER_NAME=tester \ -e SUDO_ACCESS=true \ -e PUBLIC_KEY_FILE=/tmp/test_key.pub \ + -e SUDO_NOPASSWD=true \ linuxserver/openssh-server:latest - name: Wait for SSH server to be ready @@ -56,6 +57,26 @@ jobs: sleep 2 done + echo "Debugging SSH setup..." + echo "Checking if authorized_keys file exists:" + docker exec test-container ls -la /home/tester/.ssh/ || echo "SSH directory doesn't exist" + + echo "Checking authorized_keys content:" + docker exec test-container cat /home/tester/.ssh/authorized_keys || echo "authorized_keys not found" + + echo "Checking if tester user exists:" + docker exec test-container id tester || echo "tester user not found" + + echo "Checking SSH server logs:" + docker exec test-container tail -20 /var/log/auth.log || echo "No auth log found" + + echo "Manually setting up SSH keys if needed..." + docker exec test-container mkdir -p /home/tester/.ssh + docker exec test-container chmod 700 /home/tester/.ssh + docker exec test-container cp /tmp/test_key.pub /home/tester/.ssh/authorized_keys + docker exec test-container chmod 600 /home/tester/.ssh/authorized_keys + docker exec test-container chown -R tester:tester /home/tester/.ssh + echo "Waiting for SSH to accept connections..." for i in {1..30}; do if ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 -o BatchMode=yes tester@localhost "echo 'test'" > /dev/null 2>&1; then From e7a4a0b7f5f6fc06bfacc308b3825c4002fada70 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:49:05 +0300 Subject: [PATCH 21/29] debug --- .github/workflows/infra-testing.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 5dc75ba2..4ed416aa 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -77,6 +77,24 @@ jobs: docker exec test-container chmod 600 /home/tester/.ssh/authorized_keys docker exec test-container chown -R tester:tester /home/tester/.ssh + echo "Verifying SSH key setup..." + echo "SSH directory contents:" + docker exec test-container ls -la /home/tester/.ssh/ + echo "Authorized keys content:" + docker exec test-container cat /home/tester/.ssh/authorized_keys + echo "SSH directory permissions:" + docker exec test-container ls -la /home/tester/ + + echo "Testing SSH connection with verbose output..." + ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes -v tester@localhost "echo 'test'" 2>&1 || echo "SSH connection failed with verbose output above" + + echo "Checking SSH server configuration..." + docker exec test-container cat /etc/ssh/sshd_config | grep -E "(PubkeyAuthentication|AuthorizedKeysFile|PasswordAuthentication)" || echo "SSH config not found" + + echo "Restarting SSH service..." + docker exec test-container service ssh restart || echo "SSH restart failed" + sleep 3 + echo "Waiting for SSH to accept connections..." for i in {1..30}; do if ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 -o BatchMode=yes tester@localhost "echo 'test'" > /dev/null 2>&1; then From 4f02c52f00c0cfa7d57fd2ab3468e7745995717c Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:50:24 +0300 Subject: [PATCH 22/29] debug --- .github/workflows/infra-testing.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 4ed416aa..852e2f96 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -91,19 +91,12 @@ jobs: echo "Checking SSH server configuration..." docker exec test-container cat /etc/ssh/sshd_config | grep -E "(PubkeyAuthentication|AuthorizedKeysFile|PasswordAuthentication)" || echo "SSH config not found" - echo "Restarting SSH service..." - docker exec test-container service ssh restart || echo "SSH restart failed" - sleep 3 + echo "Enabling public key authentication..." + docker exec test-container sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config + docker exec test-container sed -i 's/#PubkeyAuthentication no/PubkeyAuthentication yes/' /etc/ssh/sshd_config - echo "Waiting for SSH to accept connections..." - for i in {1..30}; do - if ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 -o BatchMode=yes tester@localhost "echo 'test'" > /dev/null 2>&1; then - echo "SSH connection successful" - break - fi - echo "Attempt $i/30: SSH connection failed" - sleep 2 - done + echo "Restarting SSH service..." + docker exec test-container /etc/init.d/ssh restart || docker exec test-container systemctl restart ssh || echo "SSH restart failed" - name: Setup and test client run: | From 9e8642df44eaa875a6851697e81ff4ba8b7b5dc7 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 19:54:23 +0300 Subject: [PATCH 23/29] remove logs --- .github/workflows/infra-testing.yml | 57 ++--------------------------- 1 file changed, 4 insertions(+), 53 deletions(-) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 852e2f96..6fc0470c 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -3,6 +3,10 @@ name: Infra testing on: - push +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-deployment: runs-on: ubuntu-latest @@ -45,59 +49,6 @@ jobs: -e SUDO_NOPASSWD=true \ linuxserver/openssh-server:latest - - name: Wait for SSH server to be ready - run: | - echo "Waiting for SSH server to start..." - for i in {1..30}; do - if docker exec test-container pgrep sshd > /dev/null 2>&1; then - echo "SSH daemon is running" - break - fi - echo "Attempt $i/30: SSH daemon not ready yet" - sleep 2 - done - - echo "Debugging SSH setup..." - echo "Checking if authorized_keys file exists:" - docker exec test-container ls -la /home/tester/.ssh/ || echo "SSH directory doesn't exist" - - echo "Checking authorized_keys content:" - docker exec test-container cat /home/tester/.ssh/authorized_keys || echo "authorized_keys not found" - - echo "Checking if tester user exists:" - docker exec test-container id tester || echo "tester user not found" - - echo "Checking SSH server logs:" - docker exec test-container tail -20 /var/log/auth.log || echo "No auth log found" - - echo "Manually setting up SSH keys if needed..." - docker exec test-container mkdir -p /home/tester/.ssh - docker exec test-container chmod 700 /home/tester/.ssh - docker exec test-container cp /tmp/test_key.pub /home/tester/.ssh/authorized_keys - docker exec test-container chmod 600 /home/tester/.ssh/authorized_keys - docker exec test-container chown -R tester:tester /home/tester/.ssh - - echo "Verifying SSH key setup..." - echo "SSH directory contents:" - docker exec test-container ls -la /home/tester/.ssh/ - echo "Authorized keys content:" - docker exec test-container cat /home/tester/.ssh/authorized_keys - echo "SSH directory permissions:" - docker exec test-container ls -la /home/tester/ - - echo "Testing SSH connection with verbose output..." - ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes -v tester@localhost "echo 'test'" 2>&1 || echo "SSH connection failed with verbose output above" - - echo "Checking SSH server configuration..." - docker exec test-container cat /etc/ssh/sshd_config | grep -E "(PubkeyAuthentication|AuthorizedKeysFile|PasswordAuthentication)" || echo "SSH config not found" - - echo "Enabling public key authentication..." - docker exec test-container sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/' /etc/ssh/sshd_config - docker exec test-container sed -i 's/#PubkeyAuthentication no/PubkeyAuthentication yes/' /etc/ssh/sshd_config - - echo "Restarting SSH service..." - docker exec test-container /etc/init.d/ssh restart || docker exec test-container systemctl restart ssh || echo "SSH restart failed" - - name: Setup and test client run: | ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 tester@localhost "echo 'SSH connection successful'" From be695780f776e863f6a548fb0bd4cd1dd3a6b221 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Fri, 26 Sep 2025 20:11:27 +0300 Subject: [PATCH 24/29] fix stuff --- .github/workflows/infra-testing.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml index 6fc0470c..651e3760 100644 --- a/.github/workflows/infra-testing.yml +++ b/.github/workflows/infra-testing.yml @@ -3,10 +3,6 @@ name: Infra testing on: - push -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - jobs: test-deployment: runs-on: ubuntu-latest @@ -36,13 +32,7 @@ jobs: - name: Start server container run: | - docker run -d \ - --name test-container \ - --network host \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e PUID=1000 \ - -e PGID=1000 \ + docker run -d -v /tmp:/tmp -p 2222:22 \ -e USER_NAME=tester \ -e SUDO_ACCESS=true \ -e PUBLIC_KEY_FILE=/tmp/test_key.pub \ @@ -51,7 +41,7 @@ jobs: - name: Setup and test client run: | - ssh -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 tester@localhost "echo 'SSH connection successful'" + ssh -p 2222 -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 tester@localhost "echo 'SSH connection successful'" - name: Deploy run: | From 62acc6ed222fd505dd8ecae8218ba860e7420ad1 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Sun, 28 Sep 2025 13:01:11 +0300 Subject: [PATCH 25/29] add traefik labels to database --- .github/workflows/infra-testing.yml | 52 ----------------------------- infra/docker-compose.yaml | 16 +++++---- 2 files changed, 9 insertions(+), 59 deletions(-) delete mode 100644 .github/workflows/infra-testing.yml diff --git a/.github/workflows/infra-testing.yml b/.github/workflows/infra-testing.yml deleted file mode 100644 index 651e3760..00000000 --- a/.github/workflows/infra-testing.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Infra testing - -on: - - push - -jobs: - test-deployment: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install project dependencies - run: uv sync --all-extras --dev - - - name: Generate SSH key pair - run: | - mkdir -p ~/.ssh - chmod 700 ~/.ssh - ssh-keygen -m PEM -t rsa -b 2048 -f ~/.ssh/test_key -N "" -C "test@github-actions" - chmod 600 ~/.ssh/test_key - chmod 644 ~/.ssh/test_key.pub - - ssh-keyscan -H localhost >> ~/.ssh/known_hosts - cp ~/.ssh/test_key.pub /tmp/test_key.pub - cp ~/.ssh/test_key /tmp/test_key - chmod 600 /tmp/test_key - - - name: Start server container - run: | - docker run -d -v /tmp:/tmp -p 2222:22 \ - -e USER_NAME=tester \ - -e SUDO_ACCESS=true \ - -e PUBLIC_KEY_FILE=/tmp/test_key.pub \ - -e SUDO_NOPASSWD=true \ - linuxserver/openssh-server:latest - - - name: Setup and test client - run: | - ssh -p 2222 -i /tmp/test_key -o StrictHostKeyChecking=no -o ConnectTimeout=5 tester@localhost "echo 'SSH connection successful'" - - - name: Deploy - run: | - uv run python infra/deploy.py infra/configs/dev.yaml - - - name: Test endpoint - run: | - ssh -i ~/.ssh/test_key -o StrictHostKeyChecking=no tester@localhost "curl -f http://localhost:80/ping" diff --git a/infra/docker-compose.yaml b/infra/docker-compose.yaml index 70e2a9ce..6a977e16 100644 --- a/infra/docker-compose.yaml +++ b/infra/docker-compose.yaml @@ -5,20 +5,24 @@ services: POSTGRES_DB: "hyperleda" POSTGRES_USER: "hyperleda" POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} - ports: - - "5432:5432" volumes: - ./postgres/data:/var/lib/postgresql/data restart: unless-stopped networks: - hyperleda-network + - proxynet healthcheck: test: psql 'host=localhost port=5432 dbname=hyperleda user=hyperleda password=$${POSTGRES_PASSWORD}' -qtA -c 'select 1;' || exit 1 timeout: 5s interval: 5s retries: 5 - logging: - driver: local + labels: + - "traefik.enable=true" + - "traefik.tcp.routers.hyperledadb.rule=HostSNI(`database.leda.sao.ru`)" + - "traefik.tcp.routers.hyperledadb.entrypoints=https" + - "traefik.tcp.routers.hyperledadb.tls=true" + - "traefik.tcp.services.hyperledadb.loadbalancer.server.port=5432" + - "traefik.docker.network=proxynet" migrate: build: @@ -33,8 +37,6 @@ services: condition: service_healthy networks: - hyperleda-network - logging: - driver: local wait-for-migrate: image: busybox @@ -137,4 +139,4 @@ networks: hyperleda-network: driver: bridge proxynet: - external: true \ No newline at end of file + external: true From 56b2f1f8c86d8064b1caa8299205904a66ce4a06 Mon Sep 17 00:00:00 2001 From: Artyom Zaporozhets Date: Sun, 28 Sep 2025 13:10:05 +0300 Subject: [PATCH 26/29] fixes --- infra/configs/dev.yaml | 9 --------- infra/deployment.py | 8 +------- 2 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 infra/configs/dev.yaml diff --git a/infra/configs/dev.yaml b/infra/configs/dev.yaml deleted file mode 100644 index b0991008..00000000 --- a/infra/configs/dev.yaml +++ /dev/null @@ -1,9 +0,0 @@ -connection: - host: localhost - user: tester - private_key_filename: /tmp/test_key - -remote_base_path: /tmp/hyperleda_test -configs_env: test -ads_token: test_token -postgres_password: test_password diff --git a/infra/deployment.py b/infra/deployment.py index db5ff385..b8d3f327 100644 --- a/infra/deployment.py +++ b/infra/deployment.py @@ -146,16 +146,10 @@ def __str__(self) -> str: return "\n".join(lines) def apply(self, ctx: ConnectionContext, logger: structlog.stdlib.BoundLogger): - connect_kwargs = {} - if ctx.private_key_filename: - connect_kwargs["key_filename"] = ctx.private_key_filename - if ctx.password: - connect_kwargs["password"] = ctx.password - self.connection = Connection( host=ctx.host, user=ctx.user, - connect_kwargs=connect_kwargs, + connect_kwargs={"key_filename": ctx.private_key_filename}, ) self.logger = logger From 1f9715fe315d2b415468bdd442b8a4628acd0c8e Mon Sep 17 00:00:00 2001 From: kraysent Date: Sat, 8 Nov 2025 08:44:44 +0000 Subject: [PATCH 27/29] fix deps and ports --- infra/docker-compose.yaml | 4 +++- pyproject.toml | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/infra/docker-compose.yaml b/infra/docker-compose.yaml index 6a977e16..cb34fb59 100644 --- a/infra/docker-compose.yaml +++ b/infra/docker-compose.yaml @@ -8,6 +8,8 @@ services: volumes: - ./postgres/data:/var/lib/postgresql/data restart: unless-stopped + ports: + - "5433:5432" networks: - hyperleda-network - proxynet @@ -19,7 +21,7 @@ services: labels: - "traefik.enable=true" - "traefik.tcp.routers.hyperledadb.rule=HostSNI(`database.leda.sao.ru`)" - - "traefik.tcp.routers.hyperledadb.entrypoints=https" + - "traefik.tcp.routers.hyperledadb.entrypoints=postgres" - "traefik.tcp.routers.hyperledadb.tls=true" - "traefik.tcp.services.hyperledadb.loadbalancer.server.port=5432" - "traefik.docker.network=proxynet" diff --git a/pyproject.toml b/pyproject.toml index 8f3aa7e2..afe6c5e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,14 +2,14 @@ name = "hyperleda-backend" version = "0.1.0" readme = "README.md" -requires-python = ">=3.13" +requires-python = "~=3.13.0" dependencies = [ - "astropy>=7.0.0", - "astroquery>=0.4.9.post1", + "astropy>=7.1.0", + "astroquery>=0.4.10", "bcrypt>=4.2.1", "click>=8.1.8", "jinja2>=3.1.5", - "pandas>=2.2.3", + "pandas>=2.3.2", "psycopg[binary,pool]>=3.2.4", "pyyaml>=6.0.2", "requests>=2.32.3", @@ -28,7 +28,7 @@ dependencies = [ [project.optional-dependencies] test = [ - "pandas-stubs>=2.2.3.241126", + "pandas-stubs>=2.3.2.250827", "parameterized>=0.9.0", "pytest>=8.3.4", "testcontainers-postgres>=0.0.1rc1", From c5e1b5e86a1a567d62b60c90de5686307edda340 Mon Sep 17 00:00:00 2001 From: kraysent Date: Wed, 12 Nov 2025 14:03:17 +0000 Subject: [PATCH 28/29] moved deployment to separate repo --- .github/workflows/app-deploy.yaml | 39 +------ infra/.env | 1 - infra/.gitignore | 1 - infra/configs/nginx/nginx.conf | 32 ----- infra/deploy.py | 90 -------------- infra/deployment.py | 188 ------------------------------ infra/docker-compose.yaml | 144 ----------------------- infra/makefile | 5 - 8 files changed, 1 insertion(+), 499 deletions(-) delete mode 100644 infra/.env delete mode 100644 infra/.gitignore delete mode 100644 infra/configs/nginx/nginx.conf delete mode 100644 infra/deploy.py delete mode 100644 infra/deployment.py delete mode 100644 infra/docker-compose.yaml delete mode 100644 infra/makefile diff --git a/.github/workflows/app-deploy.yaml b/.github/workflows/app-deploy.yaml index 67fb2232..78d5ce1d 100644 --- a/.github/workflows/app-deploy.yaml +++ b/.github/workflows/app-deploy.yaml @@ -1,4 +1,4 @@ -name: Release +name: Release image on: workflow_dispatch: @@ -55,40 +55,3 @@ jobs: docker push ghcr.io/hyperleda/hyperleda:$GIT_VERSION docker tag ghcr.io/hyperleda/hyperleda:$GIT_VERSION ghcr.io/hyperleda/hyperleda:latest docker push ghcr.io/hyperleda/hyperleda:latest - - deploy-test: - name: Deploy to test environment - needs: push-docker - environment: testing - runs-on: ubuntu-latest - env: - HOST: ${{ secrets.BACKEND_HOST }} - BACKEND_USER: ${{ secrets.BACKEND_USER }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install the project - run: uv sync --all-extras --dev - - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Write private key and config - run: | - mkdir infra/settings - echo "${{ secrets.TEST_DEPLOY_CONFIG }}" >> infra/settings/test.yaml - echo "${{ secrets.SSH_PRIVATE_KEY }}" >> infra/hyperleda_rsa - - - name: Add backend to known hosts - run: | - ssh-keyscan -H ${{ secrets.BACKEND_HOST }} >> ~/.ssh/known_hosts - - - name: Copy files and restart the backend - run: | - make deploy-test diff --git a/infra/.env b/infra/.env deleted file mode 100644 index 80834719..00000000 --- a/infra/.env +++ /dev/null @@ -1 +0,0 @@ -HOST=127.0.0.1 diff --git a/infra/.gitignore b/infra/.gitignore deleted file mode 100644 index fe5dd4a2..00000000 --- a/infra/.gitignore +++ /dev/null @@ -1 +0,0 @@ -settings \ No newline at end of file diff --git a/infra/configs/nginx/nginx.conf b/infra/configs/nginx/nginx.conf deleted file mode 100644 index 8e9696af..00000000 --- a/infra/configs/nginx/nginx.conf +++ /dev/null @@ -1,32 +0,0 @@ -server { - listen 80 default_server; - server_name _; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - location /ping { - return 200 'pong'; - } - - location /admin/api/ { - proxy_pass http://adminapi:8080/admin/api/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/ { - proxy_pass http://dataapi:8080/api/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location / { - proxy_pass http://webapp:3000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } -} \ No newline at end of file diff --git a/infra/deploy.py b/infra/deploy.py deleted file mode 100644 index d3786533..00000000 --- a/infra/deploy.py +++ /dev/null @@ -1,90 +0,0 @@ -import os -import pathlib -import subprocess -import sys -from dataclasses import dataclass - -import deployment -import structlog - - -def get_git_version() -> str: - result = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, check=True) - return result.stdout.strip() - - -@dataclass -class EnvParams: - connection: deployment.ConnectionContext - remote_base_path: str - configs_env: str - ads_token: str = "" - postgres_password: str = "" - - ads_token_env = "CLIENTS_ADS_TOKEN" - postgres_password_env = "POSTGRES_PASSWORD" - - @classmethod - def from_yaml(cls, path: str) -> "EnvParams": - import yaml - - with pathlib.Path(path).open("r") as f: - data = yaml.safe_load(f) - - data["connection"] = deployment.ConnectionContext(**data["connection"]) - - params = cls(**data) - - params.ads_token = params.ads_token or os.getenv(params.ads_token_env) or "" - params.postgres_password = params.postgres_password or os.getenv(params.postgres_password_env) or "" - - return params - - -def get_spec(params: EnvParams) -> deployment.RemoteSpec: - return deployment.RemoteSpec( - [ - deployment.RemoteFile( - "infra/docker-compose.yaml", - "docker-compose.yaml", - ), - deployment.RemoteContent( - get_git_version(), - "version.txt", - ), - deployment.RemoteDirectory( - "postgres/migrations", - "postgres/migrations", - ), - deployment.RemoteFile( - "postgres/dockerfile", - "postgres/dockerfile", - ), - deployment.RemoteDirectory( - "infra/configs/nginx", - "configs", - ), - deployment.RemoteDirectory( - pathlib.Path("configs") / params.configs_env, - "configs", - ), - ], - root_dir=pathlib.Path(params.remote_base_path), - env_vars={ - "CLIENTS_ADS_TOKEN": params.ads_token, - "POSTGRES_PASSWORD": params.postgres_password, - }, - ) - - -if __name__ == "__main__": - logger = structlog.get_logger() - - config = sys.argv[1] - params = EnvParams.from_yaml(config) - - spec = get_spec(params) - print(spec) - - spec.apply(params.connection, logger) - spec.reload() diff --git a/infra/deployment.py b/infra/deployment.py deleted file mode 100644 index b8d3f327..00000000 --- a/infra/deployment.py +++ /dev/null @@ -1,188 +0,0 @@ -import io -import os -import pathlib -from dataclasses import dataclass, field -from typing import IO - -import structlog -from fabric import Connection - - -@dataclass -class RemoteFile: - local_path: pathlib.Path | str - remote_path: pathlib.Path | str - - def __str__(self) -> str: - return f"{self.local_path} --> {self.remote_path}" - - -@dataclass -class RemoteContent: - content: str - remote_path: pathlib.Path | str - - def __str__(self) -> str: - return f'"{self.content}" --> {self.remote_path}' - - -@dataclass -class RemoteDirectory: - local_path: pathlib.Path | str - remote_path: pathlib.Path | str - - def _get_filenames(self) -> list[str]: - filenames: list[str] = [] - - for _, _, fnames in os.walk(self.local_path): - filenames.extend(fnames) - - filenames.sort() - - return filenames - - def __str__(self) -> str: - filenames = self._get_filenames() - if len(filenames) == 0: - return "" - - lines = [] - lines.append(f"{self.local_path} --> {self.remote_path}") - - for filename in filenames: - lines.append(f"\t/{filename}") - - return "\n".join(lines) - - def to_files(self) -> list[RemoteFile]: - files: list[RemoteFile] = [] - - local_base = pathlib.Path(self.local_path) - remote_base = pathlib.Path(self.remote_path) - filenames = self._get_filenames() - - for filename in filenames: - local_file = local_base / filename - remote_file = remote_base / filename - files.append(RemoteFile(local_file, remote_file)) - - return files - - -RemoteData = RemoteFile | RemoteContent | RemoteDirectory - - -@dataclass -class ConnectionContext: - host: str - user: str - private_key_filename: str | None = None - password: str | None = None - - -def _run_command( - logger: structlog.stdlib.BoundLogger, - connection: Connection, - cmd: str, - params: dict[str, str] | None = None, -): - logger.debug("Running command", cmd=cmd) - - if params: - cmd = cmd.format(**params) - - connection.run(cmd) - - -def _apply_item( - logger: structlog.stdlib.BoundLogger, - connection: Connection, - path_on_remote: str | pathlib.Path, - file_like: IO | str, -): - remote_path = pathlib.Path(path_on_remote) - - if isinstance(file_like, str): - logger.debug("Copying file", src=str(file_like), dst=str(remote_path)) - else: - logger.debug("Writing file", dst=str(remote_path)) - - connection.put(file_like, str(remote_path)) - - -@dataclass -class RemoteSpec: - data: list[RemoteData] - root_dir: pathlib.Path - env_vars: dict[str, str] = field(default_factory=dict) - - def add(self, data: RemoteData | list[RemoteData]): - if isinstance(data, list): - self.data.extend(data) - return - - self.data.append(data) - - def __str__(self) -> str: - lines = [ - "------- Params -------", - f"Root directory: {self.root_dir}", - "", - "------- Environment variables -------", - ] - - lines.extend(self.env_vars.keys()) - - lines.extend( - [ - "", - "------- Directory structure -------", - ] - ) - - for entry in self.data: - lines.append(str(entry)) - - return "\n".join(lines) - - def apply(self, ctx: ConnectionContext, logger: structlog.stdlib.BoundLogger): - self.connection = Connection( - host=ctx.host, - user=ctx.user, - connect_kwargs={"key_filename": ctx.private_key_filename}, - ) - self.logger = logger - - for item in self.data: - if isinstance(item, RemoteFile): - path = self.root_dir / item.remote_path - _run_command(self.logger, self.connection, f"mkdir -p {str(path.parent)}") - - _apply_item(self.logger, self.connection, path, str(item.local_path)) - elif isinstance(item, RemoteContent): - path = self.root_dir / item.remote_path - _run_command(self.logger, self.connection, f"mkdir -p {str(path.parent)}") - - _apply_item(self.logger, self.connection, path, io.StringIO(item.content)) - elif isinstance(item, RemoteDirectory): - _run_command(self.logger, self.connection, f"mkdir -p {str(self.root_dir / item.remote_path)}") - files = item.to_files() - - for file in files: - _apply_item(self.logger, self.connection, self.root_dir / file.remote_path, str(file.local_path)) - - _run_command(self.logger, self.connection, f"cd {self.root_dir} && docker compose pull") - - def reload(self): - env_strings = [] - params = {} - - for key, value in self.env_vars.items(): - env_strings.append(f"{key}=" + "{" + key + "}") - params[key] = value - - env_string = " ".join(env_strings) - - _run_command( - self.logger, self.connection, f"cd {self.root_dir} && {env_string} docker compose up -d", params=params - ) diff --git a/infra/docker-compose.yaml b/infra/docker-compose.yaml deleted file mode 100644 index cb34fb59..00000000 --- a/infra/docker-compose.yaml +++ /dev/null @@ -1,144 +0,0 @@ -services: - hyperledadb: - image: postgis/postgis:17-3.5 - environment: - POSTGRES_DB: "hyperleda" - POSTGRES_USER: "hyperleda" - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} - volumes: - - ./postgres/data:/var/lib/postgresql/data - restart: unless-stopped - ports: - - "5433:5432" - networks: - - hyperleda-network - - proxynet - healthcheck: - test: psql 'host=localhost port=5432 dbname=hyperleda user=hyperleda password=$${POSTGRES_PASSWORD}' -qtA -c 'select 1;' || exit 1 - timeout: 5s - interval: 5s - retries: 5 - labels: - - "traefik.enable=true" - - "traefik.tcp.routers.hyperledadb.rule=HostSNI(`database.leda.sao.ru`)" - - "traefik.tcp.routers.hyperledadb.entrypoints=postgres" - - "traefik.tcp.routers.hyperledadb.tls=true" - - "traefik.tcp.services.hyperledadb.loadbalancer.server.port=5432" - - "traefik.docker.network=proxynet" - - migrate: - build: - dockerfile: postgres/dockerfile - environment: - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} - entrypoint: /bin/sh -c "cd database/postgres && pgmigrate -c \"postgresql://hyperledadb:5432/hyperleda?user=hyperleda&password=$${POSTGRES_PASSWORD}\" -t latest migrate" - volumes: - - .:/database - depends_on: - hyperledadb: - condition: service_healthy - networks: - - hyperleda-network - - wait-for-migrate: - image: busybox - depends_on: - migrate: - condition: service_completed_successfully - entrypoint: /bin/sh -c "echo 'Migration completed successfully'" - networks: - - hyperleda-network - logging: - driver: local - - adminapi: - image: ghcr.io/hyperleda/hyperleda:latest - entrypoint: ["uv", "run", "main.py", "adminapi"] - environment: - - CONFIG=configs/adminapi.yaml - - CLIENTS_ADS_TOKEN - - STORAGE_PASSWORD=${POSTGRES_PASSWORD:-password} - volumes: - - ./configs:/usr/src/app/configs - restart: unless-stopped - networks: - - hyperleda-network - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/ping"] - interval: 5s - timeout: 5s - retries: 5 - depends_on: - - wait-for-migrate - logging: - driver: local - - dataapi: - image: ghcr.io/hyperleda/hyperleda:latest - entrypoint: ["uv", "run", "main.py", "dataapi"] - environment: - - CONFIG=configs/dataapi.yaml - - STORAGE_PASSWORD=${POSTGRES_PASSWORD:-password} - volumes: - - ./configs:/usr/src/app/configs - restart: unless-stopped - networks: - - hyperleda-network - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8081/ping"] - interval: 5s - timeout: 5s - retries: 5 - depends_on: - - wait-for-migrate - logging: - driver: local - - webapp: - image: ghcr.io/hyperleda/hyperleda-webapp:latest - networks: - - hyperleda-network - healthcheck: - test: ["CMD", "curl", "-f", "-LI", "http://localhost:3000/"] - interval: 5s - timeout: 5s - retries: 5 - restart: unless-stopped - logging: - driver: local - - nginx: - image: nginx:latest - volumes: - - ./configs/nginx.conf:/etc/nginx/conf.d/default.conf - - ./logs:/var/log/nginx - depends_on: - adminapi: - condition: service_healthy - dataapi: - condition: service_healthy - webapp: - condition: service_healthy - restart: unless-stopped - networks: - - hyperleda-network - - proxynet - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:80/ping"] - interval: 5s - timeout: 5s - retries: 5 - logging: - driver: local - labels: - - "traefik.enable=true" - - "traefik.http.routers.nginx.rule=Host(`leda.sao.ru`)" - - "traefik.http.routers.nginx.entrypoints=https" - - "traefik.http.services.nginx.loadbalancer.server.port=80" - - "traefik.docker.network=proxynet" - -networks: - hyperleda-network: - driver: bridge - proxynet: - external: true diff --git a/infra/makefile b/infra/makefile deleted file mode 100644 index 4d7d1cd3..00000000 --- a/infra/makefile +++ /dev/null @@ -1,5 +0,0 @@ -copy-files: - set -a && . .env.local && set +a && uv run scripts.py copy-files - -deploy: - set -a && . .env.local && set +a && uv run scripts.py deploy From a22229ed952e5ed3b6c36aae7f852ee348c5100f Mon Sep 17 00:00:00 2001 From: kraysent Date: Wed, 12 Nov 2025 14:06:20 +0000 Subject: [PATCH 29/29] fix make file --- makefile | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/makefile b/makefile index 2a4f275f..3c451619 100644 --- a/makefile +++ b/makefile @@ -102,7 +102,7 @@ fix: fix-unsafe: uvx ruff check --config=pyproject.toml --unsafe-fixes --fix -## Deploy +## Release GIT_VERSION = `git rev-parse --short master` @@ -113,9 +113,3 @@ image-build: image-push: docker push ghcr.io/hyperleda/hyperleda:$(GIT_VERSION) docker push ghcr.io/hyperleda/hyperleda:latest - -deploy-test: - uv run infra/deploy.py infra/settings/test.yaml - -deploy-prod: - uv run infra/deploy.py infra/settings/prod.yaml