diff --git a/infrastructure/modules/catcolab/services.nix b/infrastructure/modules/catcolab/services.nix index f8fa6384e..b76e4237b 100644 --- a/infrastructure/modules/catcolab/services.nix +++ b/infrastructure/modules/catcolab/services.nix @@ -17,26 +17,16 @@ let automergePortStr = builtins.toString cfg.automerge.port; # idempotent script for intializing the catcolab database + # Wraps the shared database-setup.sh script, extracting the password from DATABASE_URL databaseSetupScript = pkgs.writeShellScriptBin "database-setup" '' #!/usr/bin/env bash set -ex - # Extract the password from the secret file - password=$(echo $DATABASE_URL | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p') - - # Create the user only if it doesn't already exist. - if ! ${pkgs.postgresql}/bin/psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='catcolab'" | grep -q 1; then - ${pkgs.postgresql}/bin/psql -c "CREATE USER catcolab WITH ENCRYPTED PASSWORD '$password';" - fi + export PATH="${pkgs.postgresql}/bin:$PATH" - # Create the database only if it doesn't already exist. - if ! ${pkgs.postgresql}/bin/psql -tAc "SELECT 1 FROM pg_database WHERE datname='catcolab'" | grep -q 1; then - ${pkgs.postgresql}/bin/psql -c "CREATE DATABASE catcolab;" - fi + password=$(echo $DATABASE_URL | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p') - ${pkgs.postgresql}/bin/psql -c "alter database catcolab owner to catcolab;" - ${pkgs.postgresql}/bin/psql -c "grant all privileges on database catcolab to catcolab;" - ${pkgs.postgresql}/bin/psql -d catcolab -c "grant all on schema public to catcolab;" + ${pkgs.writeShellScript "database-setup-impl" (builtins.readFile ../../scripts/database-setup.sh)} "$password" ''; in with lib; diff --git a/infrastructure/scripts/cc-utils-db.sh b/infrastructure/scripts/cc-utils-db.sh index 1d3785538..474ff8bd2 100644 --- a/infrastructure/scripts/cc-utils-db.sh +++ b/infrastructure/scripts/cc-utils-db.sh @@ -8,6 +8,7 @@ function print_db_help { echo "Usage: $0 db [options]" echo "" echo "Subcommands:" + echo " setup Set up the local database (idempotent)" echo " reset Reset the local database" echo " load Load a dump into a target (local|staging|production)" echo " dump Dump a target (local|staging|production) database" @@ -55,6 +56,12 @@ function print_reset_help { echo " -h, --help Print this help message" } +function print_setup_help { + echo "Usage: $0 db setup" + echo "" + echo "Set up the local database for local development" +} + function run_db() { local subcommand="${1-}" shift || true @@ -69,6 +76,9 @@ function run_db() { load) run_load "$@" ;; + setup) + run_setup "$@" + ;; reset) run_reset "$@" ;; @@ -290,21 +300,30 @@ function run_reset() { fi load_target_env local - local pwd="$PGPASSWORD" + local catcolab_db_password="$PGPASSWORD" load_target_env local_superuser - if ! psql --dbname=postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='catcolab'" | grep -q 1; then - psql --dbname=postgres --command "CREATE USER catcolab WITH ENCRYPTED PASSWORD '$pwd';" - fi - psql --dbname=postgres --command "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='catcolab' AND pid<>pg_backend_pid();" psql --dbname=postgres --command "DROP DATABASE IF EXISTS catcolab;" - psql --dbname=postgres --command "CREATE DATABASE catcolab;" - psql --dbname=postgres --command "ALTER DATABASE catcolab OWNER TO catcolab;" - psql --dbname=catcolab --command "GRANT ALL ON SCHEMA public TO catcolab;" + database-setup.sh "$catcolab_db_password" if [[ "$skip_migrate" != "true" ]]; then run_local_migrations fi } + +function run_setup() { + if [[ "${1-}" == "-h" || "${1-}" == "--help" ]]; then + print_setup_help + exit 0 + fi + + load_target_env local + local catcolab_db_password="$PGPASSWORD" + load_target_env local_superuser + + database-setup.sh "$catcolab_db_password" + + run_local_migrations +} diff --git a/infrastructure/scripts/cc-utils-lib.sh b/infrastructure/scripts/cc-utils-lib.sh index bdbb7dc18..846a23f8e 100644 --- a/infrastructure/scripts/cc-utils-lib.sh +++ b/infrastructure/scripts/cc-utils-lib.sh @@ -38,7 +38,7 @@ function load_env() { content=$(<"$env_file") fi - # extract VAR= + # get the contents of the varname variable local url=$(printf '%s\n' "$content" | grep -E "^${varname}=" | cut -d '=' -f2-) if [[ -z $url ]]; then echo "Error: '$varname' missing in $env_file." >&2 @@ -89,7 +89,28 @@ function wait_for_port() { function load_target_env() { case "$1" in local_superuser) - load_env DATABASE_SUPERUSER_URL "$(find_git_root)/packages/backend/.env" + local env_file="$(find_git_root)/packages/backend/.env" + + if grep -q "^DATABASE_SUPERUSER_URL=" "$env_file" 2>/dev/null; then + load_env DATABASE_SUPERUSER_URL "$env_file" + else + # Fall back to constructing from DATABASE_URL + prompted password + echo "Tip: Add 'DATABASE_SUPERUSER_URL=postgresql://postgres:@localhost:5432/postgres' to $env_file to configure postgres superuser access" >&2 + echo "" >&2 + + load_env DATABASE_URL "$env_file" + local db_host="$PGHOST" + local db_port="$PGPORT" + + read -s -p "Enter postgres user password: " pg_password + echo "" + + export PGUSER="postgres" + export PGPASSWORD="$pg_password" + export PGHOST="$db_host" + export PGPORT="$db_port" + export PGDATABASE="postgres" + fi ;; local) load_env DATABASE_URL "$(find_git_root)/packages/backend/.env" diff --git a/infrastructure/scripts/database-setup.sh b/infrastructure/scripts/database-setup.sh new file mode 100755 index 000000000..776508d07 --- /dev/null +++ b/infrastructure/scripts/database-setup.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Usage: database-setup.sh +# +# Environment: +# Expects to be run as the postgres superuser or with PostgreSQL environment +# variables for the superuser (PGUSER, PGHOST, PGPORT, etc.). + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +password="$1" + +# Create the user only if it doesn't already exist. +if ! psql --dbname=postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='catcolab'" | grep -q 1; then + psql --dbname=postgres -c "CREATE USER catcolab WITH ENCRYPTED PASSWORD '$password';" +fi + +# Create the database only if it doesn't already exist. +if ! psql --dbname=postgres -tAc "SELECT 1 FROM pg_database WHERE datname='catcolab'" | grep -q 1; then + psql --dbname=postgres -c "CREATE DATABASE catcolab;" +fi + +psql --dbname=postgres -c "ALTER DATABASE catcolab OWNER TO catcolab;" +psql --dbname=postgres -c "GRANT ALL PRIVILEGES ON DATABASE catcolab TO catcolab;" +psql --dbname=catcolab -c "GRANT ALL ON SCHEMA public TO catcolab;" diff --git a/packages/backend/.env.development b/packages/backend/.env.development index 9bc8624ca..460856707 100644 --- a/packages/backend/.env.development +++ b/packages/backend/.env.development @@ -1,2 +1,3 @@ -DATABASE_URL=postgres://postgres-user:password@localhost:5432/catcolab +DATABASE_URL=postgres://catcolab:password@localhost:5432/catcolab +DATABASE_SUPERUSER_URL=postgresql://postgres:password@localhost:5432/postgres FIREBASE_PROJECT_ID=catcolab-next diff --git a/packages/backend/README.md b/packages/backend/README.md index 4224282d0..bdcd4007f 100644 --- a/packages/backend/README.md +++ b/packages/backend/README.md @@ -9,15 +9,8 @@ You can find the auto-generated documentation for this Rust crate at [next.catco ## Setup 1. Install Rust, say by using [rustup](https://rustup.rs/) -2. Install and run PostgreSQL and create a new database named `catcolab` - - (E.g. by using Docker) - ```sh - docker run --name catcolab-postgres -e POSTGRES_USER=postgres-user \ - -e POSTGRES_PASSWORD=password -e POSTGRES_DB=catcolab -p 5432:5432 -d postgres:15 - ``` - -3. Make sure the required packages are built and installed: +2. Make sure the required packages are built and installed: ```sh cd packages/notebook-types @@ -26,10 +19,44 @@ You can find the auto-generated documentation for this Rust crate at [next.catco pnpm install ``` -4. Change to the migrator directory: `cd ../backend` -5. Copy the .env.development to both folders (`cp .env.development .env && cp .env.development ../migrator/.env`) and update the `DATABASE_URL` variable with - database username, password, and port. (If you used the above Docker command _as is_ it should already be correct.) -6. Run the initial database migration: `cargo run -p migrator apply` +3. Set up PostgreSQL (choose one of the options below) + + - **Option A: Using Docker (Recommended)** + + Run PostgreSQL in a Docker container: + + ```sh + docker run --name catcolab-postgres -e POSTGRES_PASSWORD=password \ + -p 5432:5432 -d postgres:15 + ``` + + This creates a PostgreSQL instance with superuser `postgres` and password `password`. The `catcolab` database and user will be created automatically by the setup script in step 6. + + The default `.env.development` file is pre-configured to work with this Docker setup. + + - **Option B: Using local PostgreSQL installation** + + Install and run PostgreSQL locally. Ensure you have superuser access (typically the `postgres` user). + + **Note:** The default `.env.development` assumes the PostgreSQL superuser is `postgres` with password `password`. + If your local PostgreSQL has different superuser credentials, you'll need to update `DATABASE_SUPERUSER_URL` + in your `.env` file (see next step). + +4. Change to the backend directory: `cd packages/backend` +5. Copy the `.env.development` file: `cp .env.development .env` + - If you're using the Docker command above as-is, the defaults will work perfectly + - If you're using local PostgreSQL with different superuser credentials, you can either: + - Update `DATABASE_SUPERUSER_URL` in `.env` to match your postgres superuser credentials, OR + - Remove `DATABASE_SUPERUSER_URL` from `.env` and you'll be prompted for your postgres password when running the setup + - **If NOT using Nix shell:** Also copy the `.env` file to the migrator directory: `cp .env ../migrator/.env` + - **If using Nix shell:** The environment is automatically configured and you don't need to copy `.env` files manually +6. Run the database setup: + - If using Nix shell: `cc-utils db setup` (the script is in your PATH) + - Otherwise: `../../infrastructure/scripts/cc-utils db setup` + - This command connects as the PostgreSQL superuser and automatically creates: + - The `catcolab` database user with appropriate permissions + - The `catcolab` database owned by that user + - It then runs all database migrations 7. Build the backend binary: `cargo build` 8. Run the unit tests: `cargo test` @@ -72,16 +99,16 @@ pnpm run dev ## Running migrations -This package runs databaes migrations using `migrator` subcommand which uses the +Migrations are in the `migrator` package which uses the [sqlx_migrator](https://github.com/iamsauravsharma/sqlx_migrator) framework. -### Usage The migrator tool can be run from any directory using the `cargo run -p backend migrator ...` command. The migrator tool uses the default CLI interface provided by `sqlx_migrator`, which is very similar to the `sqlx` CLI. The `DATABASE_URL` environment variable must be set for the target database. This is typically configured -automatically by the Nix dev shell defined in the repository's `flake.nix`. +automatically by the Nix dev shell defined in the repository's `flake.nix`. Alternatively it is read from +the `.env` file in the current directory. To view available commands, run