add GitlabStore

.github/actions/common-setup/action.yaml

@@ -1,10 +1,6 @@
 name: Common Project Setup
 description: Setup Python, install uv, and install dependencies
 inputs:
-  python-version:
-    description: which python to install
-    required: false
-    default: '3.10'
   uv-version:
     description: which uv to use
     required: false
@@ -16,10 +12,9 @@ runs:
       uses: astral-sh/setup-uv@v6
       with:
         version: ${{ inputs.uv-version }}
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ inputs.python-version }}
     - name: Install the project
       shell: bash
       run: uv sync --all-packages --locked 
+    - name: Print python version
+      shell: bash
+      run: uv run python --version

.github/workflows/ci.yml

@@ -69,6 +69,6 @@ jobs:
       - name: tests/third_party/s3-tests
         run: |
           cd tests/third_party/s3-tests
-          uv run tox -- s3tests_boto3/functional/test_s3.py -m boxdrive
+          uv run tox -- s3tests_boto3/functional/test_s3.py -m inmemory
         env:
           S3TEST_CONF: s3tests.conf

.gitignore

@@ -213,3 +213,4 @@ __marimo__/
 pyrightconfig.json
 tmp
 .tmp
+.volumes

.python-version

@@ -1 +1 @@
-3.13
+3.12

README.md

@@ -8,16 +8,17 @@
 S3-compatible API with **Abstract Object Store** in Python (FastAPI).
 Work in progress.
 
-## Installation
+## Built-in stores
+- `InMemoryStore`
+- `GitlabStore` (with some limitations)
 
+## Quick Start
+
+0. install `boxdrive` from [pypi](https://pypi.org/project/boxdrive/) using your favorite package manager:
 ```bash
 uv add boxdrive
 ```
 
-## Quick Start
-
-### Basic Usage
-
 1. create `main.py`:
 ```python
 from boxdrive import create_app
@@ -71,7 +72,7 @@ class MyCustomStore(ObjectStore):
     async def put_object(
         self, bucket_name: BucketName, key: Key, data: bytes, content_type: ContentType | None = None
     ) -> ObjectInfo: ...
-    async def delete_object(self, bucket_name: BucketName, key: Key) -> ObjectInfo: ...
+    async def delete_object(self, bucket_name: BucketName, key: Key) -> None: ...
     async def head_object(self, bucket_name: BucketName, key: Key) -> ObjectInfo: ...
     async def list_objects(
         self,
@@ -99,22 +100,34 @@ class MyCustomStore(ObjectStore):
 
 ## Development
 
-### Tests
+### Start API
+
+0. start monitoring pipeline:
+```sh
+docker compose -f docker-compose-monitoring.yaml up --detach --wait
+```
+
+1. start api:
+```sh
+uv run fastapi dev examples/inmemory.py
+```
+
+### Run Tests
 
 #### unit
 ```bash
 uv run pytest tests/unit
 ```
 
 #### e2e
-API should run in the background.
+`API` should run in the background.
 ```bash
 export S3_ENDPOINT_URL=http://127.0.0.1:8000
 uv run run pytest tests/e2e
 ```
 
 #### third_party/s3-tests
-API should run in the background.
+`API` should run in the background.
 ```bash
 cd tests/third_party/s3-tests
 export S3TEST_CONF=s3tests.conf
@@ -123,7 +136,7 @@ uv run tox -- s3tests_boto3/functional/test_s3.py -m boxdrive
 See [tests/third_party/s3-tests/boxdrive.md](./tests/third_party/s3-tests/boxdrive.md)
 for additional info.
 
-### Code Quality
+### Check Code Quality
 
 ```bash
 uv run ruff format .

docker-compose-monitoring.yaml

@@ -0,0 +1,38 @@
+networks:
+  internal:
+    driver: bridge
+
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server:25.6.3
+    ports:
+      - 8123:8123   # HTTP interface
+      - 9000:9000   # Native interface
+    environment:
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    volumes:
+      - .volumes/clickhouse_lib:/var/lib/clickhouse
+      - .volumes/clickhouse_log:/var/log/clickhouse-server
+    networks:
+      - internal
+    healthcheck:
+      test: wget --no-verbose --tries=1 http://127.0.0.1:8123/ping || exit 1
+      interval: 10s
+      timeout: 10s
+      retries: 3
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.129.1
+    ports:
+      - 4317:4317   # OTLP gRPC receiver
+      - 4318:4318   # OTLP HTTP receiver
+    environment:
+      CLICKHOUSE_ENDPOINT: tcp://clickhouse:9000?dial_timeout=10s
+    command: ["--config=/etc/otel-collector-config.yml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yml
+    networks:
+      - internal
+    depends_on:
+      clickhouse:
+        condition: service_healthy

examples/custom_store.pyi

@@ -19,7 +19,7 @@ class MyCustomStore(ObjectStore):
     async def put_object(
         self, bucket_name: BucketName, key: Key, data: bytes, content_type: ContentType | None = None
     ) -> ObjectInfo: ...
-    async def delete_object(self, bucket_name: BucketName, key: Key) -> ObjectInfo: ...
+    async def delete_object(self, bucket_name: BucketName, key: Key) -> None: ...
     async def head_object(self, bucket_name: BucketName, key: Key) -> ObjectInfo: ...
     async def list_objects(
         self,

examples/gitlab.py

@@ -0,0 +1,16 @@
+import os
+
+from boxdrive import create_app
+from boxdrive.stores import GitlabStore
+
+repo_id = int(os.environ["REPO_ID"])
+branch = os.environ.get("BRANCH", "main")
+access_token = os.environ["ACCESS_TOKEN"]
+
+otel_exporter_http_endpoint = os.getenv("OTEL_EXPORTER_HTTP_ENDPOINT")
+
+store = GitlabStore(repo_id, branch, access_token=access_token)
+app = create_app(
+    store,
+    otel_exporter_http_endpoint=otel_exporter_http_endpoint,
+)

examples/inmemory.py

@@ -1,5 +1,12 @@
+import os
+
 from boxdrive import create_app
 from boxdrive.stores import InMemoryStore
 
+otel_exporter_http_endpoint = os.getenv("OTEL_EXPORTER_HTTP_ENDPOINT")
+
 store = InMemoryStore()
-app = create_app(store)
+app = create_app(
+    store,
+    otel_exporter_http_endpoint=otel_exporter_http_endpoint,
+)

flake.nix

@@ -34,6 +34,8 @@
             packages = with pkgs; [
               git
               uv
+
+              clickhouse
             ];
           };
         };

otel-collector-config.yaml

@@ -0,0 +1,55 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+    timeout: 5s
+    send_batch_size: 100000
+
+exporters:
+  clickhouse:
+    endpoint: ${env:CLICKHOUSE_ENDPOINT}
+    database: otel
+    async_insert: true
+    ttl: 72h
+    compress: lz4
+    create_schema: true
+    logs_table_name: otel_logs
+    traces_table_name: otel_traces
+    timeout: 5s
+    metrics_tables:
+      gauge:
+        name: "otel_metrics_gauge"
+      sum:
+        name: "otel_metrics_sum"
+      summary:
+        name: "otel_metrics_summary"
+      histogram:
+        name: "otel_metrics_histogram"
+      exponential_histogram:
+        name: "otel_metrics_exp_histogram"
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 300s
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [clickhouse]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [clickhouse]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [clickhouse]

pyproject.toml

@@ -1,17 +1,20 @@
 [project]
 name = "boxdrive"
-version = "0.0.2"
+version = "0.0.3"
 description = "S3-compatible API with Abstract Object Store in Python (FastAPI)."
 readme = "README.md"
 authors = [{ name = "Alex Severin", email = "severinalexeyv@gmail.com" }]
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 dependencies = [
     "fastapi[standard]>=0.115.14",
     "pydantic>=2.11.7",
     "pydantic-xml>=2.17.2",
     "opentelemetry-instrumentation-logging>=0.55b1",
     "opentelemetry-instrumentation-fastapi>=0.55b1",
     "opentelemetry-sdk>=1.34.1",
+    "aiorwlock>=1.5.0",
+    "opentelemetry-exporter-otlp>=1.35.0",
+    "opentelemetry-instrumentation-httpx>=0.56b0",
 ]
 
 [dependency-groups]