diff --git a/.github/build-v2-wip.yaml b/.github/build-v2-wip.yaml deleted file mode 100644 index ed7bf37..0000000 --- a/.github/build-v2-wip.yaml +++ /dev/null @@ -1,173 +0,0 @@ -# -# -# This is work in progress! Goal: parallelize builds and cache docker layers. Ideally not rebuild images that have no changed commits, -# e.g. version each component separately vs. versioning all in one package based on workspaces-stack's version. -# -# - - -name: "Release v2" -on: - workflow_dispatch: - inputs: - environment: - description: 'Environment' - required: true - default: 'dev' - -jobs: - build: - runs-on: ubuntu-latest - name: Build, Push, Deploy - environment: "dev" - timeout-minutes: 45 - steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - -# - name: Generate release notes -# run: -# ./gen-rel-notes.sh > CHANGELOG - - - name: Set commit SHA as code version - run: echo "CODE_VERSION=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV - - - uses: docker/login-action@v3 - with: - registry: ${{ vars.WS_DOCKER_REGISTRY }} - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - uses: docker/setup-buildx-action@v3 - name: Set up Docker Buildx - - - name: Build frontend - uses: docker/build-push-action@v6 - with: - context: ./frontend - push: true - tags: ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-frontend:${{ vars.ENV }}, ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-frontend:${{ env.CODE_VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build osm-cgimap - uses: docker/build-push-action@v6 - with: - context: ./osm-cgimap - push: true - file: ./osm-cgimap/docker/ubuntu/Dockerfile2404 - tags: ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-osm-cgimap:${{ vars.ENV }}, ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-osm-cgimap:${{ env.CODE_VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build osm-log-proxy - uses: docker/build-push-action@v6 - with: - context: ./osm-log-proxy - push: true - tags: ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-osm-log-proxy:${{ vars.ENV }}, ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-osm-log-proxy:${{ env.CODE_VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build osm-rails - uses: docker/build-push-action@v6 - with: - context: ./osm-rails - push: true - file: ./osm-rails/Dockerfile.prod - tags: ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-osm-rails:${{ vars.ENV }}, ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-osm-rails:${{ env.CODE_VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build osm-web - uses: docker/build-push-action@v6 - with: - context: ./osm-web - push: true - tags: ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-osm-web:${{ vars.ENV }}, ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-osm-web:${{ env.CODE_VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build pathways-editor - uses: docker/build-push-action@v6 - with: - context: ./pathways-editor - push: true - tags: ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-pathways-editor:${{ vars.ENV }}, ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-pathways-editor:${{ env.CODE_VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build rapid - uses: docker/build-push-action@v6 - with: - context: ./rapid - push: true - tags: ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-rapid:${{ vars.ENV }}, ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-rapid:${{ env.CODE_VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build tasks-backend - uses: docker/build-push-action@v6 - with: - context: ./tasking-manager - push: true - file: ./tasking-manager/scripts/docker/Dockerfile.backend - target: prod - build-args: | - APP_UID=1000 - tags: ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-tasks-backend:${{ vars.ENV }}, ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-tasks-backend:${{ env.CODE_VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Build tasks-frontend - uses: docker/build-push-action@v6 - with: - context: ./tasking-manager - push: true - file: ./tasking-manager/scripts/docker/Dockerfile.frontend - build-args: | - TM_APP_API_URL="https://tasks.workspaces-${{ vars.ENV }}.sidewalks.washington.edu/" - tags: ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-tasks-frontend:${{ vars.ENV }}, ${{ vars.WS_DOCKER_REGISTRY }}/workspaces-tasks-frontend:${{ env.CODE_VERSION }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Deploy - env: - ENV: ${{ vars.ENV }} - WS_DOCKER_REGISTRY: ${{ vars.WS_DOCKER_REGISTRY }} - - # machine to deploy to - AZURE_DOCKER_HOST_IP: ${{ vars.AZURE_DOCKER_HOST_IP }} - - # tag to deploy - WS_DOCKER_TAG: ${{ env.CODE_VERSION }} - - # passwords - WS_API_DB_PASS: ${{ secrets.WS_API_DB_PASS }} - WS_MAPBOX_ACCESS_TOKEN: ${{ secrets.WS_MAPBOX_ACCESS_TOKEN }} - WS_OSM_DB_PASS: ${{ secrets.WS_OSM_DB_PASS }} - WS_OSM_SECRET_KEY_BASE: ${{ secrets.WS_OSM_SECRET_KEY_BASE }} - WS_SMTP_PASS: ${{ secrets.WS_SMTP_PASS }} - WS_TASKS_DB_PASS: ${{ secrets.WS_TASKS_DB_PASS }} - WS_TASKS_SECRET_KEY: ${{ secrets.WS_TASKS_SECRET_KEY }} - run: | - mkdir -p ~/.ssh - eval `ssh-agent -s` - - ssh-keyscan ${{ vars.AZURE_DOCKER_HOST_IP }} >> ~/.ssh/known_hosts - - echo "${{ secrets.GH_ACTIONS_SSH_KEY }}" > ~/.ssh/github_actions - chmod 600 ~/.ssh/github_actions - ssh-add ~/.ssh/github_actions - - docker context create dev --docker "host=ssh://github-actions@${{ vars.AZURE_DOCKER_HOST_IP }}" - docker context use dev - - # remove images no longer in use--may need disk space to pull - docker image prune -a -f - - # pull new images, recreate containers - docker compose -f docker-compose.deploy.yml --env-file tdei_uw.env up -d --force-recreate --remove-orphans - - \ No newline at end of file diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 550b66a..e2d8b08 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,5 +1,10 @@ name: "Release" on: + push: + branches: + - dev + - stage + workflow_dispatch: inputs: build_flag: @@ -13,9 +18,9 @@ on: jobs: build: runs-on: ubuntu-latest - name: Build, Push, Deploy (dev) - environment: "dev" - timeout-minutes: 45 + name: Build, Push, Deploy + environment: ${{ github.ref_name }} + timeout-minutes: 30 steps: - uses: actions/checkout@v2 with: @@ -27,13 +32,19 @@ jobs: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: "Build Changelog" - id: build_changelog - uses: mikepenz/release-changelog-builder-action@v5 - with: - fromTag: "0.1" - toTag: ${{ github.sha }} - mode: "HYBRID" +# - name: Get last tag +# run: echo "TAG_LAST=$(git describe --tags --abbrev=0 --always)" >> $GITHUB_ENV + +# - name: "Build Changelog" +# id: build_changelog +# uses: mikepenz/release-changelog-builder-action@v5 +# with: +# toTag: "${{ github.ref }}" +# mode: "HYBRID" +# outputFile: "CHANGELOG.md" + +# - name: Print CHANGELOG.md +# run: cat CHANGELOG.md - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -44,9 +55,12 @@ jobs: docker compose -f docker-compose.build.yml --env-file tdei_uw.env push env: ENV: ${{ vars.ENV }} - WS_DOCKER_REGISTRY: ${{ vars.WS_DOCKER_REGISTRY }} CODE_VERSION: ${{ github.sha }} - + + # hosts + WS_DOCKER_REGISTRY: ${{ vars.WS_DOCKER_REGISTRY }} + WS_POSM_DB_HOST: ${{ vars.WS_POSM_DB_HOST }} + # passwords WS_API_DB_PASS: ${{ secrets.WS_API_DB_PASS }} WS_MAPBOX_ACCESS_TOKEN: ${{ secrets.WS_MAPBOX_ACCESS_TOKEN }} @@ -59,7 +73,6 @@ jobs: - name: Deploy env: ENV: ${{ vars.ENV }} - WS_DOCKER_REGISTRY: ${{ vars.WS_DOCKER_REGISTRY }} CODE_VERSION: ${{ github.sha }} # machine to deploy to @@ -67,7 +80,11 @@ jobs: # tag to deploy WS_DOCKER_TAG: ${{ github.sha }} - + + # hosts + WS_DOCKER_REGISTRY: ${{ vars.WS_DOCKER_REGISTRY }} + WS_POSM_DB_HOST: ${{ vars.WS_POSM_DB_HOST }} + # passwords WS_API_DB_PASS: ${{ secrets.WS_API_DB_PASS }} WS_MAPBOX_ACCESS_TOKEN: ${{ secrets.WS_MAPBOX_ACCESS_TOKEN }} @@ -88,8 +105,8 @@ jobs: ssh-add ~/.ssh/github_actions echo Configuring docker context - docker context create dev --docker "host=ssh://github-actions@${{ vars.AZURE_DOCKER_HOST_IP }}" - docker context use dev + docker context create ${{ vars.ENV }} --docker "host=ssh://github-actions@${{ vars.AZURE_DOCKER_HOST_IP }}" + docker context use ${{ vars.ENV }} # remove images no longer in use--may need disk space to pull echo Pruning old docker images @@ -103,6 +120,5 @@ jobs: sleep 30s echo Deploy list - docker ps - - \ No newline at end of file + docker ps + diff --git a/README.md b/README.md index cc360df..632b7d4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,19 @@ -# Contents of This Repo +# Dev Standards -## Frontend Tier +### Branch Strategy + +* **main**: day to day work goes here. +* **dev**: current code of the dev environment. No commits allowed directly; you'll need to open a PR from main (or your feature branch). Committing/pushing here causes a deployment to the dev infrastructure. +* **stage**: current code of the stage environment. No commits allowed directly; you'll need to open a PR from dev. Committing/pushing here causes a deployment to the stage infrastructure. +* **prod**: current code of the prod environment. No commits allowed directly; you'll need to open a PR from stage. Committing/pushing here causes a deployment to the prod infrastructure. + +### GH Actions + +See GitHub's actions tab in this repo for more deployment examples/steps. + +# System Overview + +### Frontend Tier * ```frontend```: The Workspaces frontend (VueJS app). * Code here: https://github.com/TaskarCenterAtUW/workspaces-frontend. * Uses the other components within this repo, e.g. ```osm-rails```, ```osm-cgimap``` and ```tasking-manager``` as an API backend. @@ -10,30 +23,87 @@ * Workspaces version in the ```workspaces``` branch. Need to pull from public version periodically to keep up to date. * Public version: tries to be a match to ```main``` of upstream. -## Backend Tier +### Backend Tier * ```osm-web```: Reverse proxy that dispatches requests to osm-cgimap or osm-rails depending on performance requirements. * ```osm-log-proxy```: sits in front of the below two services, logs requests for debugging purposes. * ```osm-rails```: “reference implementation” for OSM API for editing. Slow. * ```osm-cgimap```: Faster version of osm-rails. API format the same as osm-rails. Only some functions of osm-rails. * ```tasking-manager```: Not currently using, but task manager for OSM. - -# To build images + +# Getting Started + +### To build images for deploy ```git clone --recursive https://github.com/TaskarCenterAtUW/workspaces-stack.git``` ```docker-compose -f docker-compose.build.yml --env-file XXX.env build``` -```docker-compose --env-file XXX.env push``` +```docker-compose -f docker-compose.build.yml push``` Replace XXX.env with the environment definition file of the environment for which you want to build an image. This can be overridden when running (see below). -# To update docker daemon to run with latest images +### To update docker daemon to run with latest images ```docker-compose -f docker-compose.deploy.yml --env-file XXX.env up -d -pull always --force-recreate --remove-orphans``` Replace XXX.env with the environment definition file of the environment for which you are deploying. -# Other Files +### Other Files * ```example.env```: template of the .env file required by the Docker compose YAML files -* ```tdei_uw.env```: .env for deployment at UW's TDEI center. Only UW should use this, new users should adapt example.env to suit their needs. LTG: remove this file from this repo. +* ```tdei_uw.env```: .env for deployment at UW's TDEI center. Only UW should use this, new users should adapt example.env to suit their needs. LTG: remove this file from this repo. + +# Local Development + +### To build images for local development + +You will need access to prepopulated .env files that are not in version control. Specifically local.dev.env for the rest of these steps, which is assumed to be found in the workspaces-stack root directory. + +Add (or merge) the following to your ```/etc/hosts``` file: + +```127.0.0.1 localhost workspaces.local api.workspaces.local rapid.workspaces.local osm.workspaces.local pathways.workspaces.local tasks.workspaces.local``` + +You will also need to increase the total amount of system memory available to Docker containers to 10+ GB. In Docker Desktop you find this in Settings -> Resources -> Memory Limit. + +```git clone --recursive https://github.com/TaskarCenterAtUW/workspaces-stack.git``` + +Comment out the ```osm-log-proxy``` section from ```docker-compose.yml``` as aggregate logging is not available locally. + +```docker-compose build``` + +```cp local.dev.env .env``` + +```docker-compose up -d``` + +Now we are ready to finalize configuration on each container. + +``` +docker-compose run --rm --entrypoint=bash frontend +npm i +exit +``` + +``` +docker-compose run --rm --entrypoint=bash rapid +npm i +exit +``` + +``` +docker-compose run --rm --entrypoint=bash pathways-editor +npm i +exit +``` + +``` +docker-compose run --rm --entrypoint=bash osm-rails +cp /config/example.storage.yml /config/storage.yml +bundle install +exit +``` + +Note that there are two different ways to run individual containers depending on your dev context. + +Using the frontend container as an example: +* To debug / step through run ```docker-compose run --rm frontend``` +* To run while working on another service run ```docker-compose up -d frontend``` diff --git a/docker-buildx.build.hcl b/docker-buildx.build.hcl deleted file mode 100644 index 42a05da..0000000 --- a/docker-buildx.build.hcl +++ /dev/null @@ -1,201 +0,0 @@ -group "default" { - targets = [ - "osm-web", - "pathways-editor", - "rapid", - "osm-cgimap", - "frontend", - "tasks-frontend", - "tasks-backend", - "osm-log-proxy", - "osm-rails" - ] -} - -variable "WS_DOCKER_REGISTRY" { - default = "opensidewalksdev.azurecr.io" -} - -variable "ENV" { - default = "dev" -} - -variable "CODE_VERSION" { - default = "0" -} - -variable "VITE_API_URL" { - default = "https://api.workspaces-${ENV}.sidewalks.washington.edu/api/v1/" -} - -variable "VITE_OSM_URL" { - default = "https://osm.workspaces-${ENV}.sidewalks.washington.edu/" -} - -variable "VITE_PATHWAYS_EDITOR_URL" { - default = "https://pathways.workspaces-${ENV}.sidewalks.washington.edu/" -} - -variable "VITE_RAPID_URL" { - default = "https://rapid.workspaces-${ENV}.sidewalks.washington.edu/" -} - -variable "VITE_TDEI_API_URL" { - default = "https://tdei-api-${ENV}.azurewebsites.net/api/v1/" -} - -variable "VITE_TDEI_USER_API_URL" { - default = "https://tdei-usermanagement-be-${ENV}.azurewebsites.net/api/v1/" -} - -target "frontend" { - context = "frontend" - dockerfile = "Dockerfile" -# args = { -# CODE_VERSION = ${CODE_VERSION} -# VITE_API_URL = ${VITE_API_URL} -# VITE_OSM_URL = ${VITE_OSM_URL} -# VITE_PATHWAYS_EDITOR_URL = ${VITE_PATHWAYS_EDITOR_URL} -# VITE_RAPID_URL = ${VITE_RAPID_URL} -# VITE_TDEI_API_URL = ${VITE_TDEI_API_URL} -# VITE_TDEI_USER_API_URL = ${VITE_TDEI_USER_API_URL} -# } - tags = [ - "${WS_DOCKER_REGISTRY}/workspaces-frontend:${ENV}", - "${WS_DOCKER_REGISTRY}/workspaces-frontend:${CODE_VERSION}" - ] - cache-to = [ - "type=gha" - ] - cache-from = [ - "type=gha" - ] - -} - -target "osm-cgimap" { - context = "osm-cgimap" - dockerfile = "docker/ubuntu/Dockerfile2404" - tags = [ - "${WS_DOCKER_REGISTRY}/workspaces-osm-cgimap:${ENV}", - "${WS_DOCKER_REGISTRY}/workspaces-osm-cgimap:${CODE_VERSION}" - ] - cache-to = [ - "type=gha" - ] - cache-from = [ - "type=gha" - ] -} - -target "osm-log-proxy" { - context = "osm-log-proxy" - dockerfile = "Dockerfile" - tags = [ - "${WS_DOCKER_REGISTRY}/workspaces-osm-log-proxy:${ENV}", - "${WS_DOCKER_REGISTRY}/workspaces-osm-log-proxy:${CODE_VERSION}" - ] - cache-to = [ - "type=gha" - ] - cache-from = [ - "type=gha" - ] -} - -target "osm-rails" { - context = "osm-rails" - dockerfile = "Dockerfile.prod" - tags = [ - "${WS_DOCKER_REGISTRY}/workspaces-osm-rails:${ENV}", - "${WS_DOCKER_REGISTRY}/workspaces-osm-rails:${CODE_VERSION}" - ] - cache-to = [ - "type=gha" - ] - cache-from = [ - "type=gha" - ] -} - -target "osm-web" { - context = "osm-web" - dockerfile = "Dockerfile" - tags = [ - "${WS_DOCKER_REGISTRY}/workspaces-osm-web:${ENV}", - "${WS_DOCKER_REGISTRY}/workspaces-osm-web:${CODE_VERSION}" - ] - cache-to = [ - "type=gha" - ] - cache-from = [ - "type=gha" - ] -} - -target "pathways-editor" { - context = "pathways-editor" - dockerfile = "Dockerfile" - tags = [ - "${WS_DOCKER_REGISTRY}/workspaces-pathways-editor:${ENV}", - "${WS_DOCKER_REGISTRY}/workspaces-pathways-editor:${CODE_VERSION}" - ] - cache-to = [ - "type=gha" - ] - cache-from = [ - "type=gha" - ] -} - -target "rapid" { - context = "rapid" - dockerfile = "Dockerfile" - tags = [ - "${WS_DOCKER_REGISTRY}/workspaces-rapid:${ENV}", - "${WS_DOCKER_REGISTRY}/workspaces-rapid:${CODE_VERSION}" - ] - cache-to = [ - "type=gha" - ] - cache-from = [ - "type=gha" - ] -} - -target "tasks-backend" { - context = "tasking-manager" - dockerfile = "./scripts/docker/Dockerfile.backend" - tags = [ - "${WS_DOCKER_REGISTRY}/workspaces-tasks-backend:${ENV}", - "${WS_DOCKER_REGISTRY}/workspaces-tasks-backend:${CODE_VERSION}" - ] - target = "prod" - args = { - APP_UID = "1000" - } - cache-to = [ - "type=gha" - ] - cache-from = [ - "type=gha" - ] -} - -target "tasks-frontend" { - context = "tasking-manager" - dockerfile = "./scripts/docker/Dockerfile.frontend" - tags = [ - "${WS_DOCKER_REGISTRY}/workspaces-tasks-frontend:${ENV}", - "${WS_DOCKER_REGISTRY}/workspaces-tasks-frontend:${CODE_VERSION}" - ] - args = { - TM_APP_API_URL = "https://tasks.workspaces-${ENV}.sidewalks.washington.edu/" - } - cache-to = [ - "type=gha" - ] - cache-from = [ - "type=gha" - ] -} \ No newline at end of file diff --git a/tdei_uw.env b/tdei_uw.env index caae22a..2b74124 100644 --- a/tdei_uw.env +++ b/tdei_uw.env @@ -18,10 +18,10 @@ WS_PLATFORM_PRIVACY_POLICY=https://sidewalks.washington.edu/privacy # Web ############################################################################### -WS_TDEI_FRONTEND_URL=https://tdei-usermanagement-web-${ENV}.azurewebsites.net/ -WS_TDEI_BACKEND_URL=https://tdei-usermanagement-be-${ENV}.azurewebsites.net/api/v1/ -WS_TDEI_API_URL=https://tdei-api-${ENV}.azurewebsites.net/api/v1/ -WS_TDEI_OIDC_URL=https://tdei-keycloak.azurewebsites.net/ +WS_TDEI_FRONTEND_URL=https://portal-${ENV}.tdei.us/ +WS_TDEI_BACKEND_URL=https://portal-api-${ENV}.tdei.us/api/v1/ +WS_TDEI_API_URL=https://api-${ENV}.tdei.us/api/v1/ +WS_TDEI_OIDC_URL=https://account-{$ENV}.tdei.us/ WS_TDEI_OIDC_REALM=tdei WS_FRONTEND_HOST=workspaces-${ENV}.sidewalks.washington.edu @@ -54,13 +54,13 @@ WS_API_DB_USER=workspaces WS_API_DB_PASS=${WS_API_DB_PASS} WS_API_DB_NAME=workspaces -WS_OSM_DB_HOST=opensidewalks-${ENV}.postgres.database.azure.com +WS_OSM_DB_HOST=${WS_POSM_DB_HOST} WS_OSM_DB_PORT=5432 WS_OSM_DB_USER=workspaces-osm-${ENV} WS_OSM_DB_PASS=${WS_OSM_DB_PASS} WS_OSM_DB_NAME=workspaces-osm-${ENV} -WS_TASKS_DB_HOST=opensidewalks-${ENV}.postgres.database.azure.com +WS_TASKS_DB_HOST=${WS_POSM_DB_HOST} WS_TASKS_DB_PORT=5432 WS_TASKS_DB_USER=workspaces-tasks-${ENV} WS_TASKS_DB_PASS=${WS_TASKS_DB_PASS}