From 61a148fb0eca962a3e700f3d46a315156992a0ef Mon Sep 17 00:00:00 2001 From: Ali Naqvi Date: Fri, 22 Nov 2024 09:57:22 +0800 Subject: [PATCH 1/4] feat: PPT-1701 added extra endpoints for recompile and reload --- OPENAPI_DOC.yml | 120 ++++++++++++++++++++++++++++- src/api/drivers.cr | 30 ++++++++ src/placeos-core/driver_manager.cr | 33 +++++++- 3 files changed, 178 insertions(+), 5 deletions(-) diff --git a/OPENAPI_DOC.yml b/OPENAPI_DOC.yml index 352f7654..d91d98a8 100644 --- a/OPENAPI_DOC.yml +++ b/OPENAPI_DOC.yml @@ -3,7 +3,7 @@ openapi: 3.0.3 info: description: Internal core API. Handles driver management and comms title: core - version: 4.14.3 + version: 4.15.1 paths: /api/core/v1/chaos/terminate: post: @@ -300,6 +300,122 @@ paths: application/json: schema: $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ParameterError' + /api/core/v1/drivers/{file_name}/recompile: + post: + summary: Force re-compile driver + tags: + - Drivers + operationId: PlaceOS::Core::Api::Drivers_recompile + parameters: + - name: file_name + in: path + description: the name of the file in the repository + example: drivers/place/meet.cr + required: true + schema: + type: string + - name: commit + in: query + description: the commit hash of the driver to check is compiled + example: e901494 + required: true + schema: + type: string + - name: tag + in: query + description: the driver database id + example: driver-GFEaAlJB5 + required: true + schema: + type: string + responses: + 200: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/String' + 404: + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS__Core__Api__Application__CommonError' + 406: + description: Not Acceptable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ContentError' + 415: + description: Unsupported Media Type + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ContentError' + 422: + description: Unprocessable Entity + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ParameterError' + 400: + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ParameterError' + /api/core/v1/drivers/{driver_id}/reload: + post: + summary: Fetch and reload compiled driver + tags: + - Drivers + operationId: PlaceOS::Core::Api::Drivers_reload + parameters: + - name: driver_id + in: path + description: the driver database id + example: driver-GFEaAlJB5 + required: true + schema: + type: string + responses: + 200: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/String' + 404: + description: Not Found + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS__Core__Api__Application__CommonError' + 406: + description: Not Acceptable + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ContentError' + 415: + description: Unsupported Media Type + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ContentError' + 422: + description: Unprocessable Entity + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ParameterError' + 400: + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/PlaceOS__Core__Api__Application__ParameterError' /api/core/v1/drivers/{file_name}/details: get: summary: Returns the details of a driver @@ -733,6 +849,8 @@ components: schemas: Bool: type: boolean + String: + type: string PlaceOS__Model__Version: type: object properties: diff --git a/src/api/drivers.cr b/src/api/drivers.cr index d8984cb7..0d063334 100644 --- a/src/api/drivers.cr +++ b/src/api/drivers.cr @@ -23,6 +23,36 @@ module PlaceOS::Core::Api store.compiled?(driver_file, commit, repository.branch, repository.uri) end + # Force re-compile driver + @[AC::Route::POST("/:file_name/recompile")] + def recompile( + @[AC::Param::Info(name: "file_name", description: "the name of the file in the repository", example: "drivers/place/meet.cr")] + driver_file : String, + @[AC::Param::Info(description: "the commit hash of the driver to check is compiled", example: "e901494")] + commit : String, + @[AC::Param::Info(description: "the driver database id", example: "driver-GFEaAlJB5")] + tag : String + ) : String + driver = Model::Driver.find!(tag) + repository = driver.repository! + result = store.compile(driver_file, repository.uri, commit, repository.branch, true, repository.username, repository.decrypt_password, false) + if result.success + render text: "OK" + else + render :internal_server_error, text: result.output + end + end + + # Fetch and reload compiled driver + @[AC::Route::POST("/:driver_id/reload")] + def reload( + @[AC::Param::Info(description: "the driver database id", example: "driver-GFEaAlJB5")] + driver_id : String + ) : String + result = store.reload_driver(driver_id) + render status: result[:status], text: result[:message] + end + # Returns the details of a driver @[AC::Route::GET("/:file_name/details")] def details( diff --git a/src/placeos-core/driver_manager.cr b/src/placeos-core/driver_manager.cr index 52bc97b7..15d3b2f1 100644 --- a/src/placeos-core/driver_manager.cr +++ b/src/placeos-core/driver_manager.cr @@ -25,10 +25,14 @@ module PlaceOS::Core !ret.nil? end - def compile(file_name : String, url : String, commit : String, branch : String, force : Bool, username : String? = nil, password : String? = nil) : Result + def compile(file_name : String, url : String, commit : String, branch : String, force : Bool, username : String? = nil, password : String? = nil, fetch : Bool = true) : Result Log.info { {message: "Requesting build service to compile driver", driver_file: file_name, branch: branch, repository: url} } begin resp = BuildApi.compile(file_name, url, commit, branch, force, username, password) + unless fetch + return Result.new(success: true) + end + resp = resp.not_nil! unless resp.success? Log.error { {message: resp.body, status_code: resp.status_code, driver: file_name, commit: commit, branch: branch, force: force} } return Result.new(output: resp.body, name: file_name) @@ -86,6 +90,25 @@ module PlaceOS::Core {driver_source, commit, Core::ARCH}.join("_").downcase end + def reload_driver(driver_id : String) + if driver = Model::Driver.find?(driver_id) + repo = driver.repository! + + if compiled?(driver.file_name, driver.commit, repo.branch, repo.uri) + manager = ModuleManager.instance + stale_path = manager.reload_modules(driver) + if path = stale_path + File.delete(path) rescue nil if File.exists?(path) + end + else + return {status: 404, message: "Driver not compiled or not available on S3"} + end + else + return {status: 404, message: "Driver with id #{driver_id} not found "} + end + {status: 404, message: "OK"} + end + private def fetch_binary(link : LinkData) : String url = URI.parse(link.url) driver_file = Path[url.path].basename @@ -164,7 +187,7 @@ module PlaceOS::Core end end - def self.compile(file_name : String, url : String, commit : String, branch : String, force : Bool, username : String? = nil, password : String? = nil) + def self.compile(file_name : String, url : String, commit : String, branch : String, force : Bool, username : String? = nil, password : String? = nil, fetch : Bool = true) host = URI.parse(Core.build_host) file_name = URI.encode_www_form(file_name) headers = HTTP::Headers.new @@ -202,8 +225,10 @@ module PlaceOS::Core raise "Build API end-point #{link} returned invalid response code #{resp.status_code}, expected 303" unless resp.status_code == 303 raise "Build API end-point #{link} returned invalid state #{task["state"]}, expected 'done'" unless task["state"] == "done" hdr = resp.headers["Location"] rescue raise "Build API returned compilation done, but missing Location URL" - ConnectProxy::HTTPClient.new(host) do |client| - client.get(hdr) + if fetch + ConnectProxy::HTTPClient.new(host) do |client| + client.get(hdr) + end end end end From 91e66d19f0c5e6ad9e6eb2f32e00dd441e8d73a8 Mon Sep 17 00:00:00 2001 From: Ali Naqvi Date: Fri, 22 Nov 2024 10:22:22 +0800 Subject: [PATCH 2/4] chore: dockerfile linting + shard.yml --- Dockerfile | 37 ++++++++++++++++++++----------------- shard.lock | 4 ++++ shard.yml | 2 ++ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 083ab66a..d322a985 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,21 +18,22 @@ ENV USER=appuser # See https://stackoverflow.com/a/55757473/12429735 RUN adduser \ - --disabled-password \ - --gecos "" \ - --home "/nonexistent" \ - --shell "/sbin/nologin" \ - --no-create-home \ - --uid "${UID}" \ - "${USER}" + --disabled-password \ + --gecos "" \ + --home "/nonexistent" \ + --shell "/sbin/nologin" \ + --no-create-home \ + --uid "${UID}" \ + "${USER}" # Install additional libs required for drivers +# hadolint ignore=DL3059 RUN apk add \ --update \ --no-cache \ - 'apk-tools>=2.10.8-r0' \ - 'expat>=2.2.10-r1' \ - 'libcurl>=7.79.1-r0' + 'apk-tools>=2.10.8-r0' \ + 'expat>=2.2.10-r1' \ + 'libcurl>=7.79.1-r0' # Install shards for caching COPY shard.yml shard.yml @@ -49,17 +50,17 @@ ENV UNAME_AT_COMPILE_TIME=true # hadolint ignore=SC2086 RUN PLACE_VERSION=$PLACE_VERSION \ - PLACE_COMMIT=$PLACE_COMMIT \ - shards build $TARGET \ - --error-trace \ - --production \ - --static + PLACE_COMMIT=$PLACE_COMMIT \ + shards build $TARGET \ + --error-trace \ + --production \ + --static SHELL ["/bin/ash", "-eo", "pipefail", "-c"] # Create binary directories -RUN mkdir -p repositories bin/drivers tmp -RUN chown appuser -R /app +RUN mkdir -p repositories bin/drivers tmp \ + && chown appuser -R /app ############################################################################### @@ -89,7 +90,9 @@ COPY --from=build --chown=0:0 /app/bin/drivers /app/bin/drivers COPY --from=build /bin /bin COPY --from=build /lib/ld-musl-* /lib/ RUN chmod -R a+rwX /tmp +# hadolint ignore=DL3059 RUN chmod -R a+rwX /app/bin/drivers +# hadolint ignore=SC2114,DL3059 RUN rm -rf /bin /lib # Copy the app into place diff --git a/shard.lock b/shard.lock index cbcf37d1..bb740d72 100644 --- a/shard.lock +++ b/shard.lock @@ -81,6 +81,10 @@ shards: git: https://github.com/crystal-community/future.cr.git version: 1.0.0 + git-repository: + git: https://github.com/place-labs/git-repository.git + version: 1.4.1 + google: git: https://github.com/placeos/google.git version: 3.5.0 diff --git a/shard.yml b/shard.yml index 8c105106..a49e4373 100644 --- a/shard.yml +++ b/shard.yml @@ -75,6 +75,8 @@ dependencies: github: spider-gazelle/connect-proxy version: ~> 2.0 + git-repository: + github: place-labs/git-repository development_dependencies: ameba: github: crystal-ameba/ameba From 40da3231d2dbeadb80c6993e35b461b9dd0b9243 Mon Sep 17 00:00:00 2001 From: Ali Naqvi Date: Mon, 25 Nov 2024 16:09:09 +0800 Subject: [PATCH 3/4] chore: move git-repository to dev deps --- shard.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shard.yml b/shard.yml index a49e4373..c3bee083 100644 --- a/shard.yml +++ b/shard.yml @@ -75,11 +75,12 @@ dependencies: github: spider-gazelle/connect-proxy version: ~> 2.0 - git-repository: - github: place-labs/git-repository development_dependencies: ameba: github: crystal-ameba/ameba faker: github: askn/faker + + git-repository: + github: place-labs/git-repository From 73c9319fd167a3baf801f6be9e80a7719e6148b1 Mon Sep 17 00:00:00 2001 From: Ali Naqvi Date: Tue, 26 Nov 2024 09:09:21 +0800 Subject: [PATCH 4/4] chore: bump shard versions --- shard.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shard.lock b/shard.lock index bb740d72..82b030ef 100644 --- a/shard.lock +++ b/shard.lock @@ -7,7 +7,7 @@ shards: action-controller: git: https://github.com/spider-gazelle/action-controller.git - version: 7.4.4 + version: 7.5.0 active-model: git: https://github.com/spider-gazelle/active-model.git @@ -15,7 +15,7 @@ shards: ameba: git: https://github.com/crystal-ameba/ameba.git - version: 1.6.3 + version: 1.6.4 backtracer: git: https://github.com/sija/backtracer.cr.git @@ -215,7 +215,7 @@ shards: protobuf: git: https://github.com/jeromegn/protobuf.cr.git - version: 2.3.0 + version: 2.3.1 redis: git: https://github.com/stefanwille/crystal-redis.git