From ed020b136072c6b85dfe4aa6626b4e8cc5e5291e Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 3 Jan 2022 09:55:55 -0500 Subject: [PATCH 01/10] synthetic balances tables --- .../dbt/terra_dbt__synthetic_balances.sql | 36 +++++++++++ .../gold/terra__daily_synthetic_balances.sql | 62 +++++++++++++++++++ ...silver_terra__block_synthetic_balances.sql | 31 ++++++++++ 3 files changed, 129 insertions(+) create mode 100644 models/terra/dbt/terra_dbt__synthetic_balances.sql create mode 100644 models/terra/gold/terra__daily_synthetic_balances.sql create mode 100644 models/terra/silver/silver_terra__block_synthetic_balances.sql diff --git a/models/terra/dbt/terra_dbt__synthetic_balances.sql b/models/terra/dbt/terra_dbt__synthetic_balances.sql new file mode 100644 index 00000000..08a49b5b --- /dev/null +++ b/models/terra/dbt/terra_dbt__synthetic_balances.sql @@ -0,0 +1,36 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', chain_id, block_id, tx_id, msg_index)", + incremental_strategy = 'delete+insert', + tags = ['snowflake', 'terra_silver', 'terra_balances'] +) }} + +WITH base_tables AS ( + + SELECT + * + FROM + "FLIPSIDE_DEV_DB"."BRONZE"."PROD_TERRA_SINK_645110886" + WHERE + record_content:model:class = 'terra.balances.terra_synthetic_balances_model.Terra5SyntheticBalancesModel' + + {% if is_incremental() %} + AND (record_metadata :CreateTime :: INT / 1000) :: TIMESTAMP :: DATE >= (SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM{{ this }}) + {% endif %} + +) + +SELECT + (record_metadata :CreateTime :: INT / 1000) :: TIMESTAMP AS system_created_at, + t.value :block_id :: bigint AS block_id, + t.value :block_timestamp :: TIMESTAMP AS block_timestamp, + t.value :chain_id :: STRING AS chain_id, + t.value :function_name :: STRING AS function_name, + t.value :inputs :: STRING AS inputs, + t.value :project_id :: STRING AS project_id, + t.value :project_name :: STRING AS project_name, + t.value :value_numeric :: INTEGER AS value_numeric, + t.value :value_obj :: OBJECT AS value_obj +FROM + base_tables, + LATERAL FLATTEN(input => record_content :results) t \ No newline at end of file diff --git a/models/terra/gold/terra__daily_synthetic_balances.sql b/models/terra/gold/terra__daily_synthetic_balances.sql new file mode 100644 index 00000000..6c124143 --- /dev/null +++ b/models/terra/gold/terra__daily_synthetic_balances.sql @@ -0,0 +1,62 @@ +{{ config( + materialized = 'incremental', + sort = ['date', 'currency'], + unique_key = "CONCAT_WS('-', date, address, currency, balance_type)", + incremental_strategy = 'delete+insert', + cluster_by = ['date'], + tags = ['snowflake', 'terra', 'balances', 'terra_daily_synthetic_balances', 'address_labels'] +) }} + +WITH prices AS ( + + SELECT + p.symbol, + DATE_TRUNC('day', block_timestamp) AS DAY, + AVG(price_usd) AS price + FROM + {{ ref('terra__oracle_prices') }} p + GROUP BY + p.symbol, + DAY + +), + +rn as ( + +SELECT + block_timestamp, + address, + currency, + balance, + row_number() OVER (PARTITION BY date_trunc('day', block_timestamp), address, currency, chain_id ORDER BY block_timestamp DESC, block_id DESC) as rn +FROM {{ ref('silver_terra__block_synthetic_balances') }} + +) + +SELECT + date_trunc('day', b.block_timestamp) as date, + b.address, + address_labels.l1_label AS address_label_type, + address_labels.l2_label AS address_label_subtype, + address_labels.project_name AS address_label, + address_labels.address_name AS address_name, + balance, + balance * p.price AS balance_usd, + currency +FROM rn b + +LEFT OUTER JOIN prices p + ON p.symbol = currency + AND p.day = date_trunc('day', b.block_timestamp) + +LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS address_labels + ON b.address = address_labels.address + AND address_labels.blockchain = 'terra' + AND address_labels.creator = 'flipside' + +WHERE rn = 1 + AND balance > 0 + +{% if is_incremental() %} +AND date_trunc('day', b.block_timestamp) >= getdate() - INTERVAL '3 days' +{% endif %} \ No newline at end of file diff --git a/models/terra/silver/silver_terra__block_synthetic_balances.sql b/models/terra/silver/silver_terra__block_synthetic_balances.sql new file mode 100644 index 00000000..cd8dd0e8 --- /dev/null +++ b/models/terra/silver/silver_terra__block_synthetic_balances.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', date, address, currency, balance_type)", + incremental_strategy = 'delete+insert', + cluster_by = ['date'], + tags = ['snowflake', 'silver_terra', 'silver_terra__synthetic_balances'] +) }} + +WITH dbt_balances AS( + +SELECT + * +FROM + {{ ref('terra_dbt__synthetic_balances') }} +WHERE + 1 = 1 + +{% if is_incremental() %} +AND system_created_at :: DATE >= (SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }} AS balance + +) + +SELECT + block_id, + block_timestamp, + chain_id, + SUBSTRING(inputs,25,44) as address, + b.value:denom::string as currency, + b.value:amount as balance +FROM dbt_balances, +LATERAL FLATTEN(input => value_obj :balances) b \ No newline at end of file From 364566622a4bf13a401d2076982737360b29a8a9 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:09:23 -0500 Subject: [PATCH 02/10] incremental blob --- models/terra/dbt/terra_dbt__synthetic_balances.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/terra/dbt/terra_dbt__synthetic_balances.sql b/models/terra/dbt/terra_dbt__synthetic_balances.sql index 08a49b5b..847c267e 100644 --- a/models/terra/dbt/terra_dbt__synthetic_balances.sql +++ b/models/terra/dbt/terra_dbt__synthetic_balances.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = "CONCAT_WS('-', chain_id, block_id, tx_id, msg_index)", + unique_key = "CONCAT_WS('-', chain_id, block_id, inputs, value_obj)", incremental_strategy = 'delete+insert', tags = ['snowflake', 'terra_silver', 'terra_balances'] ) }} @@ -14,9 +14,9 @@ WITH base_tables AS ( WHERE record_content:model:class = 'terra.balances.terra_synthetic_balances_model.Terra5SyntheticBalancesModel' - {% if is_incremental() %} - AND (record_metadata :CreateTime :: INT / 1000) :: TIMESTAMP :: DATE >= (SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM{{ this }}) - {% endif %} +{% if is_incremental() %} +AND (record_metadata :CreateTime :: INT / 1000) :: TIMESTAMP :: DATE >= ( SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }}) +{% endif %} ) From e37b81c9359926b0129df6c606bb240b5298cae8 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:15:15 -0500 Subject: [PATCH 03/10] incremental load blob --- .../terra/silver/silver_terra__block_synthetic_balances.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/terra/silver/silver_terra__block_synthetic_balances.sql b/models/terra/silver/silver_terra__block_synthetic_balances.sql index cd8dd0e8..4755acfc 100644 --- a/models/terra/silver/silver_terra__block_synthetic_balances.sql +++ b/models/terra/silver/silver_terra__block_synthetic_balances.sql @@ -16,8 +16,8 @@ WHERE 1 = 1 {% if is_incremental() %} -AND system_created_at :: DATE >= (SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }} AS balance - +AND (record_metadata :CreateTime :: INT / 1000) :: TIMESTAMP :: DATE >= ( SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }}) +{% endif %} ) SELECT From e07d0ed60ebf80c254fd40b159f1c37723d6c733 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:16:09 -0500 Subject: [PATCH 04/10] more incremental load blobs --- models/terra/gold/terra__daily_synthetic_balances.sql | 4 ++++ .../terra/silver/silver_terra__block_synthetic_balances.sql | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/models/terra/gold/terra__daily_synthetic_balances.sql b/models/terra/gold/terra__daily_synthetic_balances.sql index 6c124143..38bce023 100644 --- a/models/terra/gold/terra__daily_synthetic_balances.sql +++ b/models/terra/gold/terra__daily_synthetic_balances.sql @@ -31,6 +31,10 @@ SELECT row_number() OVER (PARTITION BY date_trunc('day', block_timestamp), address, currency, chain_id ORDER BY block_timestamp DESC, block_id DESC) as rn FROM {{ ref('silver_terra__block_synthetic_balances') }} +{% if is_incremental() %} +AND block_timestamp :: DATE >= ( SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }}) +{% endif %} + ) SELECT diff --git a/models/terra/silver/silver_terra__block_synthetic_balances.sql b/models/terra/silver/silver_terra__block_synthetic_balances.sql index 4755acfc..50b2bb49 100644 --- a/models/terra/silver/silver_terra__block_synthetic_balances.sql +++ b/models/terra/silver/silver_terra__block_synthetic_balances.sql @@ -16,7 +16,7 @@ WHERE 1 = 1 {% if is_incremental() %} -AND (record_metadata :CreateTime :: INT / 1000) :: TIMESTAMP :: DATE >= ( SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }}) +AND block_timestamp :: DATE >= ( SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }}) {% endif %} ) From 05c9136123d161601c380d0ec4032b84c9345e5b Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:16:29 -0500 Subject: [PATCH 05/10] removed incremental --- models/terra/gold/terra__daily_synthetic_balances.sql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/models/terra/gold/terra__daily_synthetic_balances.sql b/models/terra/gold/terra__daily_synthetic_balances.sql index 38bce023..4df8fb37 100644 --- a/models/terra/gold/terra__daily_synthetic_balances.sql +++ b/models/terra/gold/terra__daily_synthetic_balances.sql @@ -59,8 +59,4 @@ LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS address_labels AND address_labels.creator = 'flipside' WHERE rn = 1 - AND balance > 0 - -{% if is_incremental() %} -AND date_trunc('day', b.block_timestamp) >= getdate() - INTERVAL '3 days' -{% endif %} \ No newline at end of file + AND balance > 0 \ No newline at end of file From 24d26f29e82301779174a53fac4bda25e5fd0dd8 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:17:18 -0500 Subject: [PATCH 06/10] using different incremental load blob --- models/terra/gold/terra__daily_synthetic_balances.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/terra/gold/terra__daily_synthetic_balances.sql b/models/terra/gold/terra__daily_synthetic_balances.sql index 4df8fb37..d6b5fba3 100644 --- a/models/terra/gold/terra__daily_synthetic_balances.sql +++ b/models/terra/gold/terra__daily_synthetic_balances.sql @@ -32,7 +32,7 @@ SELECT FROM {{ ref('silver_terra__block_synthetic_balances') }} {% if is_incremental() %} -AND block_timestamp :: DATE >= ( SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }}) +AND date_trunc('day', block_timestamp) >= getdate() - INTERVAL '3 days' {% endif %} ) From a4e73f5b6f3b70344b488a91a0aabb31243a2428 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:18:01 -0500 Subject: [PATCH 07/10] added where clause --- models/terra/gold/terra__daily_synthetic_balances.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/terra/gold/terra__daily_synthetic_balances.sql b/models/terra/gold/terra__daily_synthetic_balances.sql index d6b5fba3..baae7910 100644 --- a/models/terra/gold/terra__daily_synthetic_balances.sql +++ b/models/terra/gold/terra__daily_synthetic_balances.sql @@ -31,6 +31,8 @@ SELECT row_number() OVER (PARTITION BY date_trunc('day', block_timestamp), address, currency, chain_id ORDER BY block_timestamp DESC, block_id DESC) as rn FROM {{ ref('silver_terra__block_synthetic_balances') }} +WHERE 1=1 + {% if is_incremental() %} AND date_trunc('day', block_timestamp) >= getdate() - INTERVAL '3 days' {% endif %} From 9130e97f5f488c45f1c66c2f2e5666facc3fdf88 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:30:59 -0500 Subject: [PATCH 08/10] config changes --- models/terra/gold/terra__daily_synthetic_balances.sql | 4 ++-- .../silver/silver_terra__block_synthetic_balances.sql | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/models/terra/gold/terra__daily_synthetic_balances.sql b/models/terra/gold/terra__daily_synthetic_balances.sql index baae7910..5685f1c0 100644 --- a/models/terra/gold/terra__daily_synthetic_balances.sql +++ b/models/terra/gold/terra__daily_synthetic_balances.sql @@ -1,9 +1,9 @@ {{ config( materialized = 'incremental', sort = ['date', 'currency'], - unique_key = "CONCAT_WS('-', date, address, currency, balance_type)", + unique_key = "CONCAT_WS('-', block_timestamp, address, currency)", incremental_strategy = 'delete+insert', - cluster_by = ['date'], + cluster_by = ['block_timestamp::DATE'], tags = ['snowflake', 'terra', 'balances', 'terra_daily_synthetic_balances', 'address_labels'] ) }} diff --git a/models/terra/silver/silver_terra__block_synthetic_balances.sql b/models/terra/silver/silver_terra__block_synthetic_balances.sql index 50b2bb49..ffad5072 100644 --- a/models/terra/silver/silver_terra__block_synthetic_balances.sql +++ b/models/terra/silver/silver_terra__block_synthetic_balances.sql @@ -1,8 +1,8 @@ {{ config( materialized = 'incremental', - unique_key = "CONCAT_WS('-', date, address, currency, balance_type)", + unique_key = "CONCAT_WS('-', block_timestamp, address, currency)", incremental_strategy = 'delete+insert', - cluster_by = ['date'], + cluster_by = ['block_timestamp::DATE'], tags = ['snowflake', 'silver_terra', 'silver_terra__synthetic_balances'] ) }} @@ -16,8 +16,9 @@ WHERE 1 = 1 {% if is_incremental() %} -AND block_timestamp :: DATE >= ( SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }}) +AND system_created_at :: DATE >= ( SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }}) {% endif %} + ) SELECT From 491ef49cd6e8b0af82911bb7592f0ced217659c1 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:36:15 -0500 Subject: [PATCH 09/10] more config --- models/terra/gold/terra__daily_synthetic_balances.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/terra/gold/terra__daily_synthetic_balances.sql b/models/terra/gold/terra__daily_synthetic_balances.sql index 5685f1c0..d8c84b76 100644 --- a/models/terra/gold/terra__daily_synthetic_balances.sql +++ b/models/terra/gold/terra__daily_synthetic_balances.sql @@ -1,9 +1,9 @@ {{ config( materialized = 'incremental', sort = ['date', 'currency'], - unique_key = "CONCAT_WS('-', block_timestamp, address, currency)", + unique_key = "CONCAT_WS('-', date, address, currency)", incremental_strategy = 'delete+insert', - cluster_by = ['block_timestamp::DATE'], + cluster_by = ['date'], tags = ['snowflake', 'terra', 'balances', 'terra_daily_synthetic_balances', 'address_labels'] ) }} From 28ad254236fe13d006fec932be460811bb46d42e Mon Sep 17 00:00:00 2001 From: amasucci13 Date: Wed, 5 Jan 2022 14:26:37 -0800 Subject: [PATCH 10/10] Adjusted tests, added is_native column --- .../dbt/terra_dbt__synthetic_balances.sql | 7 +- models/terra/gold/terra__daily_balances.sql | 45 ++++++++++++- models/terra/gold/terra__daily_balances.yml | 27 +++++--- .../gold/terra__daily_synthetic_balances.sql | 64 ------------------- ...silver_terra__block_synthetic_balances.sql | 35 +++++----- ...silver_terra__block_synthetic_balances.yml | 48 ++++++++++++++ .../silver/silver_terra__daily_balances.sql | 1 + 7 files changed, 133 insertions(+), 94 deletions(-) delete mode 100644 models/terra/gold/terra__daily_synthetic_balances.sql create mode 100644 models/terra/silver/silver_terra__block_synthetic_balances.yml diff --git a/models/terra/dbt/terra_dbt__synthetic_balances.sql b/models/terra/dbt/terra_dbt__synthetic_balances.sql index 847c267e..cd26c304 100644 --- a/models/terra/dbt/terra_dbt__synthetic_balances.sql +++ b/models/terra/dbt/terra_dbt__synthetic_balances.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = "CONCAT_WS('-', chain_id, block_id, inputs, value_obj)", + unique_key = "CONCAT_WS('-', block_timestamp, chain_id, block_id, inputs)", incremental_strategy = 'delete+insert', tags = ['snowflake', 'terra_silver', 'terra_balances'] ) }} @@ -10,7 +10,10 @@ WITH base_tables AS ( SELECT * FROM - "FLIPSIDE_DEV_DB"."BRONZE"."PROD_TERRA_SINK_645110886" + {{ source( + 'bronze', + 'prod_terra_sink_645110886' + ) }} WHERE record_content:model:class = 'terra.balances.terra_synthetic_balances_model.Terra5SyntheticBalancesModel' diff --git a/models/terra/gold/terra__daily_balances.sql b/models/terra/gold/terra__daily_balances.sql index c2f8f3ae..3d1ce204 100644 --- a/models/terra/gold/terra__daily_balances.sql +++ b/models/terra/gold/terra__daily_balances.sql @@ -11,17 +11,20 @@ WITH prices AS ( SELECT p.symbol, + l.address_name, DATE_TRUNC( 'day', block_timestamp ) AS DAY, AVG(price_usd) AS price FROM - {{ ref('terra__oracle_prices') }} - p + {{ ref('terra__oracle_prices') }} p + LEFT JOIN {{ ref('terra__labels') }} l + on p.symbol = l.address GROUP BY p.symbol, - DAY + DAY, + l.address_name ) SELECT @@ -34,6 +37,7 @@ SELECT balance, balance * p.price AS balance_usd, b.balance_type, + b.is_native, currency FROM {{ ref('silver_terra__daily_balances') }} b @@ -52,3 +56,38 @@ WHERE {% if is_incremental() %} AND DATE >= getdate() - INTERVAL '3 days' {% endif %} + +UNION + +SELECT + date_trunc('day', b.block_timestamp) as DATE, + b.address, + address_labels.l1_label AS address_label_type, + address_labels.l2_label AS address_label_subtype, + address_labels.project_name AS address_label, + address_labels.address_name AS address_name, + balance, + balance * p.price AS balance_usd, + b.balance_type, + b.is_native, + currency +FROM {{ ref('silver_terra__block_synthetic_balances') }} b + +LEFT OUTER JOIN prices p + ON p.symbol = currency + AND p.day = date_trunc('day', b.block_timestamp) + +LEFT OUTER JOIN prices i + ON i.address_name = currency + AND i.day = date_trunc('day', b.block_timestamp) + +LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS address_labels + ON b.address = address_labels.address + AND address_labels.blockchain = 'terra' + AND address_labels.creator = 'flipside' + +WHERE balance > 0 + +{% if is_incremental() %} +AND date_trunc('day', block_timestamp) >= getdate() - INTERVAL '3 days' +{% endif %} \ No newline at end of file diff --git a/models/terra/gold/terra__daily_balances.yml b/models/terra/gold/terra__daily_balances.yml index 741a8008..d8988a4b 100644 --- a/models/terra/gold/terra__daily_balances.yml +++ b/models/terra/gold/terra__daily_balances.yml @@ -8,12 +8,15 @@ models: - DATE - CURRENCY - BALANCE_TYPE + where: DATE > CURRENT_DATE - 30 columns: - name: ADDRESS tests: - - not_null + - not_null: + where: DATE > CURRENT_DATE - 30 - dbt_expectations.expect_column_values_to_match_regex: regex: terra[0-9a-z]{39,39} + where: DATE > CURRENT_DATE - 30 # - name: ADDRESS_LABEL # tests: # - not_null @@ -28,20 +31,28 @@ models: # - not_null - name: BALANCE tests: - - not_null + - not_null: + where: DATE > CURRENT_DATE - 30 - name: BALANCE_TYPE tests: - - not_null + - not_null: + where: DATE > CURRENT_DATE - 30 - name: BALANCE_USD tests: - - not_null: - where: DATE > '2021-05-15' + #- not_null: + # where: DATE > '2021-05-15' - name: CURRENCY tests: - - not_null + - not_null: + where: DATE > CURRENT_DATE - 30 + - name: IS_NATIVE + tests: + - not_null: + where: DATE > CURRENT_DATE - 30 - name: DATE tests: - - not_null + - not_null: + where: DATE > CURRENT_DATE - 30 - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 2 + interval: 3 diff --git a/models/terra/gold/terra__daily_synthetic_balances.sql b/models/terra/gold/terra__daily_synthetic_balances.sql deleted file mode 100644 index d8c84b76..00000000 --- a/models/terra/gold/terra__daily_synthetic_balances.sql +++ /dev/null @@ -1,64 +0,0 @@ -{{ config( - materialized = 'incremental', - sort = ['date', 'currency'], - unique_key = "CONCAT_WS('-', date, address, currency)", - incremental_strategy = 'delete+insert', - cluster_by = ['date'], - tags = ['snowflake', 'terra', 'balances', 'terra_daily_synthetic_balances', 'address_labels'] -) }} - -WITH prices AS ( - - SELECT - p.symbol, - DATE_TRUNC('day', block_timestamp) AS DAY, - AVG(price_usd) AS price - FROM - {{ ref('terra__oracle_prices') }} p - GROUP BY - p.symbol, - DAY - -), - -rn as ( - -SELECT - block_timestamp, - address, - currency, - balance, - row_number() OVER (PARTITION BY date_trunc('day', block_timestamp), address, currency, chain_id ORDER BY block_timestamp DESC, block_id DESC) as rn -FROM {{ ref('silver_terra__block_synthetic_balances') }} - -WHERE 1=1 - -{% if is_incremental() %} -AND date_trunc('day', block_timestamp) >= getdate() - INTERVAL '3 days' -{% endif %} - -) - -SELECT - date_trunc('day', b.block_timestamp) as date, - b.address, - address_labels.l1_label AS address_label_type, - address_labels.l2_label AS address_label_subtype, - address_labels.project_name AS address_label, - address_labels.address_name AS address_name, - balance, - balance * p.price AS balance_usd, - currency -FROM rn b - -LEFT OUTER JOIN prices p - ON p.symbol = currency - AND p.day = date_trunc('day', b.block_timestamp) - -LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS address_labels - ON b.address = address_labels.address - AND address_labels.blockchain = 'terra' - AND address_labels.creator = 'flipside' - -WHERE rn = 1 - AND balance > 0 \ No newline at end of file diff --git a/models/terra/silver/silver_terra__block_synthetic_balances.sql b/models/terra/silver/silver_terra__block_synthetic_balances.sql index ffad5072..a9f7290c 100644 --- a/models/terra/silver/silver_terra__block_synthetic_balances.sql +++ b/models/terra/silver/silver_terra__block_synthetic_balances.sql @@ -6,27 +6,28 @@ tags = ['snowflake', 'silver_terra', 'silver_terra__synthetic_balances'] ) }} -WITH dbt_balances AS( -SELECT - * -FROM - {{ ref('terra_dbt__synthetic_balances') }} -WHERE - 1 = 1 - -{% if is_incremental() %} -AND system_created_at :: DATE >= ( SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }}) -{% endif %} - -) +with dedupe as ( -SELECT +SELECT + system_created_at, block_id, block_timestamp, chain_id, SUBSTRING(inputs,25,44) as address, b.value:denom::string as currency, - b.value:amount as balance -FROM dbt_balances, -LATERAL FLATTEN(input => value_obj :balances) b \ No newline at end of file + 'liquid' as balance_type, + false as is_native, + b.value:amount::FLOAT as balance, + row_number() OVER (PARTITION BY date_trunc('day', block_timestamp), address, currency, chain_id ORDER BY block_timestamp DESC, block_id DESC) as rn +FROM {{ ref('terra_dbt__synthetic_balances') }}, +LATERAL FLATTEN(input => value_obj :balances) b +WHERE 1 = 1 +) + +SELECT * FROM dedupe +WHERE rn = 1 + +{% if is_incremental() %} +AND system_created_at :: DATE >= ( SELECT DATEADD('day', -1, MAX(system_created_at :: DATE)) FROM {{ this }}) +{% endif %} \ No newline at end of file diff --git a/models/terra/silver/silver_terra__block_synthetic_balances.yml b/models/terra/silver/silver_terra__block_synthetic_balances.yml new file mode 100644 index 00000000..444be938 --- /dev/null +++ b/models/terra/silver/silver_terra__block_synthetic_balances.yml @@ -0,0 +1,48 @@ +version: 2 +models: + - name: silver_terra__block_synthetic_balances + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP + - ADDRESS + - CURRENCY + where: BLOCK_TIMESTAMP > CURRENT_DATE - 30 AND BALANCE > 0 + columns: + - name: BLOCK_ID + tests: + - not_null: + where: BLOCK_TIMESTAMP > CURRENT_DATE - 30 + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: BLOCK_TIMESTAMP > CURRENT_DATE - 30 + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - name: CHAIN_ID + tests: + - not_null: + where: BLOCK_TIMESTAMP > CURRENT_DATE - 30 + - name: ADDRESS + tests: + - not_null: + where: BLOCK_TIMESTAMP > CURRENT_DATE - 30 + - dbt_expectations.expect_column_values_to_match_regex: + regex: terra[0-9a-z]{39,39} + - name: CURRENCY + tests: + - not_null: + where: BLOCK_TIMESTAMP > CURRENT_DATE - 30 + - name: BALANCE_TYPE + tests: + - not_null: + where: BLOCK_TIMESTAMP > CURRENT_DATE - 30 + - name: IS_NATIVE + tests: + - not_null: + where: BLOCK_TIMESTAMP > CURRENT_DATE - 30 + - name: BALANCE + tests: + - not_null: + where: BLOCK_TIMESTAMP > CURRENT_DATE - 30 \ No newline at end of file diff --git a/models/terra/silver/silver_terra__daily_balances.sql b/models/terra/silver/silver_terra__daily_balances.sql index 4a740938..b1a71e6a 100644 --- a/models/terra/silver/silver_terra__daily_balances.sql +++ b/models/terra/silver/silver_terra__daily_balances.sql @@ -96,6 +96,7 @@ SELECT currency, balance_type, blockchain, + TRUE as is_native, LAST_VALUE( balance ignore nulls ) over(