From 454d2920d5e3a4de5813ba8dfd68a0e684cbce0e Mon Sep 17 00:00:00 2001 From: alrocar Date: Wed, 18 Dec 2024 09:17:15 +0100 Subject: [PATCH 1/4] add vercel --- tinybird/datasources/vercel.datasource | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tinybird/datasources/vercel.datasource diff --git a/tinybird/datasources/vercel.datasource b/tinybird/datasources/vercel.datasource new file mode 100644 index 0000000..57de6fe --- /dev/null +++ b/tinybird/datasources/vercel.datasource @@ -0,0 +1,16 @@ +TOKEN "append" APPEND + +# Refer to https://vercel.com/docs/observability/webhooks-overview#events for the event types and payload schema + +TAGS "vercel" + +SCHEMA > + `type` LowCardinality(String) `json:$.type`, + `id` String `json:$.id`, + `date` DateTime64(3) `json:$.created_at` DEFAULT now(), + `region` LowCardinality(String) `json:$.region` DEFAULT '', + `payload` JSON `json:$.payload` DEFAULT '{}' + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "type, date" \ No newline at end of file From be9de149076654a0e51af14fda6ea86f778665d8 Mon Sep 17 00:00:00 2001 From: alrocar Date: Wed, 18 Dec 2024 10:57:50 +0100 Subject: [PATCH 2/4] add log drains --- tinybird/datasources/vercel_logs.datasource | 30 +++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tinybird/datasources/vercel_logs.datasource diff --git a/tinybird/datasources/vercel_logs.datasource b/tinybird/datasources/vercel_logs.datasource new file mode 100644 index 0000000..ec6fbe0 --- /dev/null +++ b/tinybird/datasources/vercel_logs.datasource @@ -0,0 +1,30 @@ +TOKEN "append" APPEND + +# Refer to https://vercel.com/docs/observability/log-drains/log-drains-reference for the logs schema + +TAGS "vercel" + +SCHEMA > + `id` String `json:$.id`, + `message` String `json:$.message`, + `date` DateTime64(3) `json:$.timestamp` DEFAULT now(), + `type` LowCardinality(String) `json:$.type`, + `request_id` Nullable(String) `json:$.requestId`, + `status_code` Nullable(UInt16) `json:$.statusCode`, + `project_id` String `json:$.projectId` DEFAULT '', + `project_name` Nullable(String) `json:$.projectName`, + `deployment_id` Nullable(String) `json:$.deploymentId`, + `source` LowCardinality(String) `json:$.source` DEFAULT '', + `host` Nullable(String) `json:$.host`, + `path` Nullable(String) `json:$.path`, + `proxy` JSON `json:$.proxy` DEFAULT '{}', + `level` LowCardinality(String) `json:$.level` DEFAULT '', + `execution_region` Nullable(String) `json:$.executionRegion`, + `environment` Nullable(String) `json:$.environment`, + `branch` Nullable(String) `json:$.branch`, + `build_id` Nullable(String) `json:$.buildId`, + `entrypoint` Nullable(String) `json:$.entrypoint` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "project_id, date" \ No newline at end of file From c7e688cfb2a3088c58818920248881ac14eeeb79 Mon Sep 17 00:00:00 2001 From: alrocar Date: Wed, 18 Dec 2024 11:14:16 +0100 Subject: [PATCH 3/4] fix schema --- tinybird/datasources/vercel_logs.datasource | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tinybird/datasources/vercel_logs.datasource b/tinybird/datasources/vercel_logs.datasource index ec6fbe0..c45694f 100644 --- a/tinybird/datasources/vercel_logs.datasource +++ b/tinybird/datasources/vercel_logs.datasource @@ -8,7 +8,7 @@ SCHEMA > `id` String `json:$.id`, `message` String `json:$.message`, `date` DateTime64(3) `json:$.timestamp` DEFAULT now(), - `type` LowCardinality(String) `json:$.type`, + `type` LowCardinality(String) `json:$.type` DEFAULT '', `request_id` Nullable(String) `json:$.requestId`, `status_code` Nullable(UInt16) `json:$.statusCode`, `project_id` String `json:$.projectId` DEFAULT '', @@ -17,7 +17,7 @@ SCHEMA > `source` LowCardinality(String) `json:$.source` DEFAULT '', `host` Nullable(String) `json:$.host`, `path` Nullable(String) `json:$.path`, - `proxy` JSON `json:$.proxy` DEFAULT '{}', + `proxy` String `json:$.proxy` DEFAULT '', `level` LowCardinality(String) `json:$.level` DEFAULT '', `execution_region` Nullable(String) `json:$.executionRegion`, `environment` Nullable(String) `json:$.environment`, From b7fb30acacb4602f21ba1e4a7f97b36c53e0ac4d Mon Sep 17 00:00:00 2001 From: alrocar Date: Wed, 18 Dec 2024 11:20:19 +0100 Subject: [PATCH 4/4] add column --- tinybird/datasources/vercel_logs.datasource | 1 + 1 file changed, 1 insertion(+) diff --git a/tinybird/datasources/vercel_logs.datasource b/tinybird/datasources/vercel_logs.datasource index c45694f..86f5785 100644 --- a/tinybird/datasources/vercel_logs.datasource +++ b/tinybird/datasources/vercel_logs.datasource @@ -17,6 +17,7 @@ SCHEMA > `source` LowCardinality(String) `json:$.source` DEFAULT '', `host` Nullable(String) `json:$.host`, `path` Nullable(String) `json:$.path`, + `destination` Nullable(String) `json:$.destination`, `proxy` String `json:$.proxy` DEFAULT '', `level` LowCardinality(String) `json:$.level` DEFAULT '', `execution_region` Nullable(String) `json:$.executionRegion`,