From 2e7abfcb08a0662cc281c559ff597f59dc667c2e Mon Sep 17 00:00:00 2001 From: s0dyy Date: Thu, 1 Sep 2022 11:58:16 +0200 Subject: [PATCH 1/4] docker: refacto (#43) --- .env.example | 8 +++++++- .gitignore | 1 + Dockerfile | 19 +++++++++++++++---- docker-compose.yml | 31 +++---------------------------- entrypoint.sh | 12 ++++++++++++ src/schema.rs | 24 +++++++++++++----------- 6 files changed, 51 insertions(+), 44 deletions(-) create mode 100644 entrypoint.sh diff --git a/.env.example b/.env.example index 0dd3b32..c352a8b 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,9 @@ +SECRET_TOKEN= + +DATABASE_URL=postgres://postgres:postgres@db/api +DATABASE_URL_TEST=postgres://postgres:postgres@db_test/api_test +DATABASE_MAX_CONNECTIONS=30 + AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= -AWS_REGION= \ No newline at end of file +AWS_REGION= diff --git a/.gitignore b/.gitignore index 88635ae..54a0f2e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ schemaspy/.DS_Store .DS_Store .vscode .env +shell.nix diff --git a/Dockerfile b/Dockerfile index 186a936..7e56c1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,21 @@ -FROM rustlang/rust:nightly +FROM ubuntu:22.04 WORKDIR /app ADD src src COPY Cargo.toml . -RUN cargo install cargo-watch -RUN cargo install diesel_cli --no-default-features --features postgres -RUN cargo build +# Dependencies +RUN apt-get -y update +RUN apt-get -y install curl build-essential libpq-dev pkg-config netcat + +# Rustup install (not yet packaged on Ubuntu) +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +# Rust toolchain +RUN ${HOME}/.cargo/bin/rustup install 1.63.0 + +# Cargo stuff +RUN ${HOME}/.cargo/bin/cargo install cargo-watch +RUN ${HOME}/.cargo/bin/cargo install diesel_cli --no-default-features --features postgres +RUN ${HOME}/.cargo/bin/cargo build diff --git a/docker-compose.yml b/docker-compose.yml index 7a66cb5..e0b73c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,12 +3,8 @@ services: api: container_name: api - image: rustlang/rust:nightly - environment: - SECRET_TOKEN: 'Xqv8jTGLxT' - DATABASE_URL: 'postgres://postgres:postgres@db/api' - DATABASE_URL_TEST: 'postgres://postgres:postgres@db_test/api_test' - DATABASE_MAX_CONNECTIONS: '30' + env_file: + - .env build: context: ./ volumes: @@ -19,50 +15,29 @@ services: depends_on: - db - db_test - command: > - bash -c "diesel setup && diesel migration run && diesel migration run --database-url=$${DATABASE_URL_TEST} && cargo watch -x run" + entrypoint: ["/bin/bash", "./entrypoint.sh"] db: container_name: db image: postgres:14.2 environment: - POSTGRES_USER: 'postgres' POSTGRES_PASSWORD: 'postgres' POSTGRES_DB: 'api' PGDATA: /data/postgres volumes: - db:/data/postgres - ports: - - "5433:5432" restart: always db_test: container_name: db_test image: postgres:14.2 environment: - POSTGRES_USER: 'postgres' POSTGRES_PASSWORD: 'postgres' POSTGRES_DB: 'api_test' PGDATA: /data/postgres volumes: - db_test:/data/postgres - ports: - - "5434:5432" restart: always - schemaspy: - image: schemaspy/schemaspy:snapshot - volumes: - - ./schemaspy/output:/output - - ./schemaspy/config:/config - container_name: "schemaspy_local" - command: [ - "-configFile", - "/config/schemaspy.properties", - "-imageformat", - "svg" - ] - depends_on: - - api volumes: db: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..95c614a --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,12 @@ +# Wait for PostgreSQL +for d in db db_test; do + until nc -z $d 5432 + do + echo "Waiting for connection to MySQL database..." + sleep 2 + done +done + +${HOME}/.cargo/bin/diesel migration run && +${HOME}/.cargo/bin/diesel migration run --database-url=${DATABASE_URL_TEST} && +${HOME}/.cargo/bin/cargo watch -x run diff --git a/src/schema.rs b/src/schema.rs index 5928ae3..0cde552 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -1,4 +1,6 @@ -table! { +// @generated automatically by Diesel CLI. + +diesel::table! { agents (id) { id -> Int4, name -> Varchar, @@ -6,7 +8,7 @@ table! { } } -table! { +diesel::table! { lambdas (id) { id -> Int4, aws_lambda_region -> Varchar, @@ -15,7 +17,7 @@ table! { } } -table! { +diesel::table! { metrics (id) { id -> Int4, load_average_1 -> Nullable, @@ -29,7 +31,7 @@ table! { } } -table! { +diesel::table! { monitors (id) { id -> Int4, name -> Varchar, @@ -45,14 +47,14 @@ table! { } } -table! { +diesel::table! { organizations (id) { id -> Int4, name -> Varchar, } } -table! { +diesel::table! { organizations_users (id) { id -> Int4, id_organization -> Nullable, @@ -60,14 +62,14 @@ table! { } } -table! { +diesel::table! { roles (id) { id -> Int4, name -> Varchar, } } -table! { +diesel::table! { users (id) { id -> Uuid, email -> Varchar, @@ -77,10 +79,10 @@ table! { } } -joinable!(monitors -> lambdas (id_lambda)); -joinable!(users -> roles (id_role)); +diesel::joinable!(monitors -> lambdas (id_lambda)); +diesel::joinable!(users -> roles (id_role)); -allow_tables_to_appear_in_same_query!( +diesel::allow_tables_to_appear_in_same_query!( agents, lambdas, metrics, From 55682ed53c1a2257ca38237b11bd1e7a8e4617d3 Mon Sep 17 00:00:00 2001 From: s0dyy Date: Wed, 14 Sep 2022 21:40:47 +0200 Subject: [PATCH 2/4] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'naersk': 'github:nix-community/naersk/c6a45e4277fa58abd524681466d3450f896dc094' (2022-08-04) → 'github:nix-community/naersk/6944160c19cb591eb85bbf9b2f2768a935623ed3' (2022-09-03) • Updated input 'naersk/nixpkgs': 'github:NixOS/nixpkgs/adf66cc31390f920854340679d7505ac577a5a8b' (2022-08-28) → 'github:NixOS/nixpkgs/125efbd96af28ea5d60e00a3eed832ea3f49a93b' (2022-09-14) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/adf66cc31390f920854340679d7505ac577a5a8b' (2022-08-28) → 'github:NixOS/nixpkgs/125efbd96af28ea5d60e00a3eed832ea3f49a93b' (2022-09-14) --- flake.lock | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 flake.lock diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..10cce96 --- /dev/null +++ b/flake.lock @@ -0,0 +1,74 @@ +{ + "nodes": { + "naersk": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1662220400, + "narHash": "sha256-9o2OGQqu4xyLZP9K6kNe1pTHnyPz0Wr3raGYnr9AIgY=", + "owner": "nix-community", + "repo": "naersk", + "rev": "6944160c19cb591eb85bbf9b2f2768a935623ed3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "naersk", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1663146586, + "narHash": "sha256-VCGdyEc5TF0uq+gdU+jesjZcNxptomDunvGtElmAD9I=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "125efbd96af28ea5d60e00a3eed832ea3f49a93b", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1663146586, + "narHash": "sha256-VCGdyEc5TF0uq+gdU+jesjZcNxptomDunvGtElmAD9I=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "125efbd96af28ea5d60e00a3eed832ea3f49a93b", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "root": { + "inputs": { + "naersk": "naersk", + "nixpkgs": "nixpkgs_2", + "utils": "utils" + } + }, + "utils": { + "locked": { + "lastModified": 1659877975, + "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} From 8391e3fcd9082a0036493555fd23d50295d7fa14 Mon Sep 17 00:00:00 2001 From: s0dyy Date: Thu, 15 Sep 2022 09:08:49 +0200 Subject: [PATCH 3/4] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'naersk/nixpkgs': 'github:NixOS/nixpkgs/125efbd96af28ea5d60e00a3eed832ea3f49a93b' (2022-09-14) → 'github:NixOS/nixpkgs/e731e6638c7726241c352c74bc7f860872e4cbd2' (2022-09-15) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/125efbd96af28ea5d60e00a3eed832ea3f49a93b' (2022-09-14) → 'github:NixOS/nixpkgs/e731e6638c7726241c352c74bc7f860872e4cbd2' (2022-09-15) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 10cce96..d049345 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1663146586, - "narHash": "sha256-VCGdyEc5TF0uq+gdU+jesjZcNxptomDunvGtElmAD9I=", + "lastModified": 1663202367, + "narHash": "sha256-62sI03nVTWKMULPnjUC+Ig952PqPDnkCSJ56MLoFvDI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "125efbd96af28ea5d60e00a3eed832ea3f49a93b", + "rev": "e731e6638c7726241c352c74bc7f860872e4cbd2", "type": "github" }, "original": { @@ -34,11 +34,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1663146586, - "narHash": "sha256-VCGdyEc5TF0uq+gdU+jesjZcNxptomDunvGtElmAD9I=", + "lastModified": 1663202367, + "narHash": "sha256-62sI03nVTWKMULPnjUC+Ig952PqPDnkCSJ56MLoFvDI=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "125efbd96af28ea5d60e00a3eed832ea3f49a93b", + "rev": "e731e6638c7726241c352c74bc7f860872e4cbd2", "type": "github" }, "original": { From 631ec9e6da6961f39636af63a4560655d0882efc Mon Sep 17 00:00:00 2001 From: s0dyy Date: Fri, 16 Sep 2022 10:24:30 +0200 Subject: [PATCH 4/4] monitors --- Dockerfile | 2 +- docker-compose.yml | 4 + flake.nix | 17 +++++ src/main.rs | 3 +- src/monitors/mod.rs | 49 +++--------- src/monitors/model.rs | 173 ++++++++++++++++++++++++------------------ 6 files changed, 131 insertions(+), 117 deletions(-) create mode 100644 flake.nix diff --git a/Dockerfile b/Dockerfile index 7e56c1d..c0c02ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ COPY Cargo.toml . # Dependencies RUN apt-get -y update -RUN apt-get -y install curl build-essential libpq-dev pkg-config netcat +RUN apt-get -y install curl build-essential libpq-dev pkg-config netcat systemd # Rustup install (not yet packaged on Ubuntu) RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y diff --git a/docker-compose.yml b/docker-compose.yml index e0b73c8..534e4f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,10 @@ services: api: container_name: api + init: true + privileged: true + devices: + - /sys/fs/cgroup:/sys/fs/cgroup:ro env_file: - .env build: diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..ca19b89 --- /dev/null +++ b/flake.nix @@ -0,0 +1,17 @@ +{ + inputs = { + utils.url = "github:numtide/flake-utils"; + naersk.url = "github:nix-community/naersk"; + }; + + outputs = { self, nixpkgs, utils, naersk }: + utils.lib.eachDefaultSystem (system: let + pkgs = nixpkgs.legacyPackages."${system}"; + naersk-lib = naersk.lib."${system}"; + in rec { + # `nix develop` + devShell = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ rustc cargo rust-analyzer ]; + }; + }); +} diff --git a/src/main.rs b/src/main.rs index cac8122..e4b1ded 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,14 +50,13 @@ async fn main() -> std::io::Result<()> { .route("/", web::get().to(index)) .service(metrics::add_metrics) .service(organizations::add_organization) - .service(monitors::add_monitor) - // .service(monitors::get_all_monitors_of_user) .service(agents::add_agents) .service(roles::add_role) .service(roles::get_roles) .service(users::register) .service(users::login) .service(users::user_informations) + .service(monitors::add_monitor) }) .bind(("0.0.0.0", 8080))? .run() diff --git a/src/monitors/mod.rs b/src/monitors/mod.rs index 1e11ad2..5621f90 100644 --- a/src/monitors/mod.rs +++ b/src/monitors/mod.rs @@ -1,44 +1,15 @@ -use self::model::{FormMonitor, InsertableMonitor, Monitor}; -use super::DbPool; -use crate::middlewares::auth::AuthorizationService; -use actix_web::{get, post, web, Error, HttpRequest, HttpResponse}; -pub mod model; +use actix_web::{post, web, Result, HttpResponse, Error}; +use uuid::Uuid; -/// Inserts new monitor with name, id_agent, id_organization defined in form. -#[post("/monitors")] -pub async fn add_monitor( - pool: web::Data, - form: web::Json, - _req: HttpRequest, - _: AuthorizationService, -) -> Result { - let token = crate::users::model::User::get_token_from_request(&_req); +use crate::monitors::model::*; +mod model; +mod util; - let monitor = web::block(move || { - let conn = pool.get()?; - Monitor::insert_new_monitor(&token, &form, &conn) - }) - .await? - .map_err(actix_web::error::ErrorInternalServerError)?; - Ok(HttpResponse::Created().json(monitor)) +#[post("/monitors")] +async fn add_monitor(monitor: web::Json) -> Result { + Monitor::write_new_monitor(monitor)?; + //Monitor::insert_new_monitor(monitor)?; + Ok(HttpResponse::Ok().body("Service created")) } -// Get all monitors of the user -// #[get("/monitors")] -// pub async fn get_all_monitors_of_user( -// pool: web::Data, -// _req: HttpRequest, -// _: AuthorizationService, -// ) -> Result { -// let token = crate::users::model::User::get_token_from_request(&_req); - -// let monitors = web::block(move || { -// let conn = pool.get()?; -// Monitor::get_all_monitors_of_user(&token, &conn) -// }) -// .await? -// .map_err(actix_web::error::ErrorInternalServerError)?; - -// Ok(HttpResponse::Ok().json(monitors)) -// } diff --git a/src/monitors/model.rs b/src/monitors/model.rs index f6444b1..baa83d6 100644 --- a/src/monitors/model.rs +++ b/src/monitors/model.rs @@ -1,88 +1,111 @@ -use crate::diesel::RunQueryDsl; -use crate::schema::monitors; -use diesel::prelude::*; -use diesel::{AsChangeset, Queryable}; -use serde_derive::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; +use std::io::prelude::*; +use std::fs; +use std::fs::File; +use std::path::Path; +use log::{info, error, debug}; use uuid::Uuid; +use actix_web::web; +use std::process::Command; -type DbError = Box; - -#[derive(Serialize, Deserialize, Queryable, AsChangeset, Insertable)] -#[table_name = "monitors"] +#[derive(Debug, Serialize, Deserialize)] pub struct Monitor { - pub id: i32, - pub id_agent: Option, - pub id_lambda: Option, - pub id_user: Uuid, - pub id_organization: Option, - pub name: String, - pub aws_eventbridge_region: String, - pub aws_eventbridge_name: String, - pub aws_eventbridge_description: String, - pub aws_eventbridge_event_bus_name: String, - pub aws_eventbridge_schedule_expression: String, + pub user_uuid: String, + pub url: String, + pub inactive_sec: String, } -#[derive(Debug, Clone, Serialize, Deserialize, Insertable, Queryable)] -#[table_name = "monitors"] +#[derive(Debug, Serialize, Deserialize)] pub struct InsertableMonitor { - pub id_agent: Option, - pub id_lambda: Option, - pub id_user: Uuid, - pub id_organization: Option, - pub name: String, - pub aws_eventbridge_region: String, - pub aws_eventbridge_name: String, - pub aws_eventbridge_description: String, - pub aws_eventbridge_event_bus_name: String, - pub aws_eventbridge_schedule_expression: String, -} - -#[derive(Debug, Clone, Serialize, Queryable, Deserialize, Insertable)] -#[table_name = "monitors"] -pub struct FormMonitor { - pub name: String, - pub id_organization: Option, - pub id_agent: Option, + pub monitor: Monitor, + pub monitor_uuid: Uuid, } impl Monitor { - pub fn insert_new_monitor( - token: &str, - form: &FormMonitor, - conn: &PgConnection, - ) -> Result { - use crate::schema::monitors::dsl::*; - let uid = crate::users::model::User::get_uid_from_token(token, conn); - let monitor = InsertableMonitor { - name: form.name.clone(), - aws_eventbridge_region: "wip".to_owned(), - aws_eventbridge_name: "wip".to_owned(), - aws_eventbridge_description: "wip".to_owned(), - aws_eventbridge_event_bus_name: "wip".to_owned(), - aws_eventbridge_schedule_expression: "wip".to_owned(), - id_agent: form.id_agent, - id_lambda: None, - id_user: uid.unwrap(), - id_organization: form.id_organization, + // TODO: Maybe put this at the start of the api + fn check_systemd_units_path(path: &String) -> std::io::Result<()> { + match fs::create_dir_all(path) { + Ok(..) => { + info!("The {} path has been successfully created", path); + return Ok(()) + }, + Err(e) => { + error!("The {} path cannot be created", path); + return Err(e) + } + } + } + + fn write_systemd_file(path: &str, contents: &str) -> std::io::Result<()> { + match File::create(&path) { + Ok(mut file) => { + match file.write_all(contents.as_bytes()) { + Ok(..) => { + debug!("The file has been written"); + return Ok(()) + }, + Err(e) => { + error!("Unable to write to the file"); + return Err(e) + } + } + } + Err(e) => { + error!("Unable to create the file"); + return Err(e) + } + }; + } + + pub fn contents_systemd_service(path: &String, insertable_monitor: &InsertableMonitor) -> std::io::Result<()> { + let path = format!("{}/mon_{}.service", path, insertable_monitor.monitor_uuid); + let unit = format!("[Unit]\nDescription=Monitor owned by user {}\n\n", insertable_monitor.monitor.user_uuid); + let service = format!("[Service]\nType=oneshot\nExecStart={}\n\n", insertable_monitor.monitor.url); + let install = format!("[Install]\nWantedBy=multi-user.target"); + let contents = format!("{}{}{}", &unit, &service, &install); + + Monitor::write_systemd_file(&path, &contents)?; + Ok(()) + } + + pub fn contents_systemd_timer(path: &String, insertable_monitor: &InsertableMonitor) -> std::io::Result<()> { + let path = format!("{}/mon_{}.timer", path, insertable_monitor.monitor_uuid); + let unit = format!("[Unit]\nDescription=Monitor owned by user {}\n\n", insertable_monitor.monitor.user_uuid); + let timer = format!("[Timer]\nOnStartupSec=1\nOnUnitInactiveSec={}\n\n", insertable_monitor.monitor.inactive_sec); + let install = format!("[Install]\nWantedBy=multi-user.target"); + let contents = format!("{}{}{}", &unit, &timer, &install); + + Monitor::write_systemd_file(&path, &contents)?; + Ok(()) + } + + pub fn systemd_timer_state(path: &String, insertable_monitor: &InsertableMonitor, state: &str) -> std::io::Result<()> { + let timer = format!("{}/mon_{}.service", path, insertable_monitor.monitor_uuid); + + Command::new("systemctl").args(["--user", state, "--now", &timer]).output().expect("Unable to start the timer"); + Ok(()) + } + + pub fn write_new_monitor(monitor: web::Json) -> std::io::Result<()> { + let path = String::from("/home/skuareview/.config/systemd/user"); + + // Create insertable monitor with new uuid + let insertable_monitor = InsertableMonitor { + monitor: monitor.into_inner(), + monitor_uuid: Uuid::new_v4() }; - diesel::insert_into(monitors) - .values(&monitor) - .execute(conn)?; - Ok(monitor) + // Check systemd units path + Monitor::check_systemd_units_path(&path)?; + + // Create and write systemd service / timer contents + Monitor::contents_systemd_service(&path, &insertable_monitor)?; + Monitor::contents_systemd_timer(&path, &insertable_monitor)?; + + // Start timer + Monitor::systemd_timer_state(&path, &insertable_monitor, "start")?; + + Ok(()) } - // pub fn get_all_monitors_of_user( - // token: &str, - // conn: &PgConnection, - // ) -> Result, DbError> { - // let uid = crate::users::model::User::get_uid_from_token(token, conn); - // use crate::schema::monitors::dsl::*; - - // let all_monitors = monitors - // .filter(id_user.eq(uid.unwrap())) - // .load::(conn) - // .unwrap(); - // Ok(all_monitors) - // } + }