From c4e7a7e52699fe390d8baf31345db290ceb2838a Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Tue, 18 Jan 2022 19:50:02 -0800 Subject: [PATCH 01/45] - fist pass at market stats table --- ...osschain__f_market_stats_usd_agg_daily.sql | 109 ++++++++++++++++++ ...osschain__f_market_stats_usd_agg_daily.yml | 50 ++++++++ ...sschain__f_market_stats_usd_agg_hourly.sql | 109 ++++++++++++++++++ ...sschain__f_market_stats_usd_agg_hourly.yml | 50 ++++++++ .../silver_crosschain__f_market_stats_usd.sql | 62 ++++++++++ .../silver_crosschain__f_market_stats_usd.yml | 50 ++++++++ 6 files changed, 430 insertions(+) create mode 100644 models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql create mode 100644 models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml create mode 100644 models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql create mode 100644 models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml create mode 100644 models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql create mode 100644 models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql new file mode 100644 index 00000000..4735a591 --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql @@ -0,0 +1,109 @@ +{{ config( + materialized = "incremental", + unique_key = "_unique_key", + incremental_strategy = "merge", + tags = ["snowflake", "market", "stats", "gold", "fact", "f_stats_usd", "daily"] +) }} + +SELECT + asset_id, + provider, + NAME, + symbol, + DATE_TRUNC( + DAY, + recorded_at + ) AS recorded_at, + COUNT(*) AS n, + AVG(circulating_supply) AS avg_circulating_supply, + AVG(market_cap) AS avg_market_cap, + AVG(max_supply) AS avg_max_supply, + AVG(price) AS avg_price, + AVG(total_supply) AS avg_total_supply, + AVG(volume_24h) AS avg_volume_24h, + MAX(circulating_supply) AS max_circulating_supply, + MAX(market_cap) AS max_market_cap, + MAX(max_supply) AS max_max_supply, + MAX(price) AS max_price, + MAX(total_supply) AS max_total_supply, + MAX(volume_24h) AS max_volume_24h, + MEDIAN(circulating_supply) AS median_circulating_supply, + MEDIAN(market_cap) AS median_market_cap, + MEDIAN(max_supply) AS median_max_supply, + MEDIAN(price) AS median_price, + MEDIAN(total_supply) AS median_total_supply, + MEDIAN(volume_24h) AS median_volume_24h, + MIN(circulating_supply) AS min_circulating_supply, + MIN(market_cap) AS min_market_cap, + ARRAY_AGG( + OBJECT_CONSTRUCT( + 'circulating_supply', + circulating_supply, + 'market_cap', + market_cap, + 'max_supply', + max_supply, + 'price', + price, + 'total_supply', + total_supply, + 'volume_24h', + volume_24h, + 'recorded_at', + recorded_at + ) + ) within GROUP ( + ORDER BY + recorded_at + ) AS DATA, + DATA [0] :market_cap AS first_market_cap, + DATA [ARRAY_SIZE(data)-1] :market_cap AS last_market_cap, + MIN(max_supply) AS min_max_supply, + DATA [0] :max_supply AS first_max_supply, + DATA [ARRAY_SIZE(data)-1] :max_supply AS last_max_supply, + MIN(price) AS min_price, + DATA [0] :price AS first_price, + DATA [ARRAY_SIZE(data)-1] :price AS last_price, + MIN(total_supply) AS min_total_supply, + DATA [0] :total_supply AS first_total_supply, + DATA [ARRAY_SIZE(data)-1] :total_supply AS last_total_supply, + MIN(volume_24h) AS min_volume_24h, + DATA [0] :volume_24h AS first_volume_24h, + DATA [ARRAY_SIZE(data)-1] :volume_24h AS last_volume_24h, + SYSDATE() AS _updated_timestamp, + concat_ws( + '-', + asset_id, + NAME, + symbol, + provider, + DATE_TRUNC( + DAY, + recorded_at + ) + ) AS _unique_key +FROM + {{ ref( + "silver_crosschain__f_market_stats_usd" + ) }} + +{% if is_incremental() %} +WHERE + recorded_at >= DATE_TRUNC( + DAY,( + SELECT + MAX(recorded_at) + FROM + {{ this }} + ) + ) - INTERVAL '2 DAYS' +{% endif %} +GROUP BY + asset_id, + provider, + NAME, + symbol, + DATE_TRUNC( + DAY, + recorded_at + ) diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml new file mode 100644 index 00000000..fef2d560 --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml @@ -0,0 +1,50 @@ +version: 2 +models: + - name: crosschain__f_market_stats_usd_agg_daily + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - NAME + - SYMBOL + - PROVIDER + - RECORDED_AT + columns: + - name: ASSET_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: NAME + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: RECORDED_AT + tests: + - not_null + - name: CIRCULATING_SUPPLY + tests: + - not_null + - name: MARKET_CAP + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: MAX_SUPPLY + tests: + - not_null + - name: TOTAL_SUPPLY + tests: + - not_null + - name: VOLUME_24H + tests: + - not_null + - name: _UPDATED + tests: + - not_null + - name: _UNIQUE_KEY + tests: + - not_null diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql new file mode 100644 index 00000000..5e2e72c8 --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql @@ -0,0 +1,109 @@ +{{ config( + materialized = "incremental", + unique_key = "_unique_key", + incremental_strategy = "merge", + tags = ["snowflake", "market", "stats", "gold", "fact", "f_stats_usd", "hourly"] +) }} + +SELECT + asset_id, + provider, + NAME, + symbol, + DATE_TRUNC( + HOUR, + recorded_at + ) AS recorded_at, + COUNT(*) AS n, + AVG(circulating_supply) AS avg_circulating_supply, + AVG(market_cap) AS avg_market_cap, + AVG(max_supply) AS avg_max_supply, + AVG(price) AS avg_price, + AVG(total_supply) AS avg_total_supply, + AVG(volume_24h) AS avg_volume_24h, + MAX(circulating_supply) AS max_circulating_supply, + MAX(market_cap) AS max_market_cap, + MAX(max_supply) AS max_max_supply, + MAX(price) AS max_price, + MAX(total_supply) AS max_total_supply, + MAX(volume_24h) AS max_volume_24h, + MEDIAN(circulating_supply) AS median_circulating_supply, + MEDIAN(market_cap) AS median_market_cap, + MEDIAN(max_supply) AS median_max_supply, + MEDIAN(price) AS median_price, + MEDIAN(total_supply) AS median_total_supply, + MEDIAN(volume_24h) AS median_volume_24h, + MIN(circulating_supply) AS min_circulating_supply, + MIN(market_cap) AS min_market_cap, + ARRAY_AGG( + OBJECT_CONSTRUCT( + 'circulating_supply', + circulating_supply, + 'market_cap', + market_cap, + 'max_supply', + max_supply, + 'price', + price, + 'total_supply', + total_supply, + 'volume_24h', + volume_24h, + 'recorded_at', + recorded_at + ) + ) within GROUP ( + ORDER BY + recorded_at + ) AS DATA, + DATA [0] :market_cap AS first_market_cap, + DATA [ARRAY_SIZE(data)-1] :market_cap AS last_market_cap, + MIN(max_supply) AS min_max_supply, + DATA [0] :max_supply AS first_max_supply, + DATA [ARRAY_SIZE(data)-1] :max_supply AS last_max_supply, + MIN(price) AS min_price, + DATA [0] :price AS first_price, + DATA [ARRAY_SIZE(data)-1] :price AS last_price, + MIN(total_supply) AS min_total_supply, + DATA [0] :total_supply AS first_total_supply, + DATA [ARRAY_SIZE(data)-1] :total_supply AS last_total_supply, + MIN(volume_24h) AS min_volume_24h, + DATA [0] :volume_24h AS first_volume_24h, + DATA [ARRAY_SIZE(data)-1] :volume_24h AS last_volume_24h, + SYSDATE() AS _updated_timestamp, + concat_ws( + '-', + asset_id, + NAME, + symbol, + provider, + DATE_TRUNC( + HOUR, + recorded_at + ) + ) AS _unique_key +FROM + {{ ref( + "silver_crosschain__f_market_stats_usd" + ) }} + +{% if is_incremental() %} +WHERE + recorded_at >= DATE_TRUNC( + HOUR,( + SELECT + MAX(recorded_at) + FROM + {{ this }} + ) + ) - INTERVAL '1 hour' +{% endif %} +GROUP BY + asset_id, + provider, + NAME, + symbol, + DATE_TRUNC( + HOUR, + recorded_at + ) diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml new file mode 100644 index 00000000..bd5e5f12 --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml @@ -0,0 +1,50 @@ +version: 2 +models: + - name: crosschain__f_market_stats_usd_agg_hourly + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - NAME + - SYMBOL + - PROVIDER + - RECORDED_AT + columns: + - name: ASSET_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: NAME + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: RECORDED_AT + tests: + - not_null + - name: CIRCULATING_SUPPLY + tests: + - not_null + - name: MARKET_CAP + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: MAX_SUPPLY + tests: + - not_null + - name: TOTAL_SUPPLY + tests: + - not_null + - name: VOLUME_24H + tests: + - not_null + - name: _UPDATED + tests: + - not_null + - name: _UNIQUE_KEY + tests: + - not_null diff --git a/models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql new file mode 100644 index 00000000..09ba42a1 --- /dev/null +++ b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql @@ -0,0 +1,62 @@ +{{ config( + materialized = "incremental", + cluster_by = "recorded_at::DATE", + unique_key = "_unique_key", + incremental_strategy = "merge", + tags = ["snowflake", "market", "stats", "gold", "fact", "f_stats_usd"] +) }} + +SELECT + DISTINCT asset_id, + provider, + NAME, + symbol, + recorded_at, + circulating_supply, + market_cap, + price, + max_supply, + total_supply, + volume_24h, + SYSDATE() AS _updated, + concat_ws( + '-', + NAME, + symbol, + provider, + to_varchar( + recorded_at, + 'YYYY-MM-DD HH24:MI:SS.FF9' + ) + ) AS _unique_key +FROM + {{ source( + "shared", + "prices_v2" + ) }} +WHERE + NAME IS NOT NULL + AND symbol IS NOT NULL + AND provider IS NOT NULL + AND recorded_at IS NOT NULL + +{% if is_incremental() %} +AND recorded_at >= ( + SELECT + MAX(recorded_at) + FROM + {{ this }} +) - INTERVAL '30 DAYS' +{% endif %} + +qualify ROW_NUMBER() over ( + PARTITION BY NAME, + symbol, + provider, + recorded_at + ORDER BY + to_varchar( + recorded_at, + 'YYYY-MM-DD HH24:MI:SS.FF9' + ) +) = 1 diff --git a/models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml new file mode 100644 index 00000000..ab3f8d73 --- /dev/null +++ b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml @@ -0,0 +1,50 @@ +version: 2 +models: + - name: silver_crosschain__f_market_stats_usd + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - NAME + - SYMBOL + - PROVIDER + - RECORDED_AT + columns: + - name: ASSET_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: NAME + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: RECORDED_AT + tests: + - not_null + - name: CIRCULATING_SUPPLY + tests: + - not_null + - name: MARKET_CAP + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: MAX_SUPPLY + tests: + - not_null + - name: TOTAL_SUPPLY + tests: + - not_null + - name: VOLUME_24H + tests: + - not_null + - name: _UPDATED + tests: + - not_null + - name: _UNIQUE_KEY + tests: + - not_null From 8bbbf1b77a5a97a47f273d430fdcadae97087a0a Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Wed, 19 Jan 2022 10:04:18 -0800 Subject: [PATCH 02/45] - sorted columns --- .../crosschain__f_market_stats_usd_agg_daily.sql | 16 ++++++++-------- ...crosschain__f_market_stats_usd_agg_hourly.sql | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql index 4735a591..16a0339b 100644 --- a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql @@ -35,6 +35,10 @@ SELECT MEDIAN(volume_24h) AS median_volume_24h, MIN(circulating_supply) AS min_circulating_supply, MIN(market_cap) AS min_market_cap, + MIN(max_supply) AS min_max_supply, + MIN(price) AS min_price, + MIN(total_supply) AS min_total_supply, + MIN(volume_24h) AS min_volume_24h, ARRAY_AGG( OBJECT_CONSTRUCT( 'circulating_supply', @@ -57,18 +61,14 @@ SELECT recorded_at ) AS DATA, DATA [0] :market_cap AS first_market_cap, - DATA [ARRAY_SIZE(data)-1] :market_cap AS last_market_cap, - MIN(max_supply) AS min_max_supply, DATA [0] :max_supply AS first_max_supply, - DATA [ARRAY_SIZE(data)-1] :max_supply AS last_max_supply, - MIN(price) AS min_price, DATA [0] :price AS first_price, - DATA [ARRAY_SIZE(data)-1] :price AS last_price, - MIN(total_supply) AS min_total_supply, DATA [0] :total_supply AS first_total_supply, - DATA [ARRAY_SIZE(data)-1] :total_supply AS last_total_supply, - MIN(volume_24h) AS min_volume_24h, DATA [0] :volume_24h AS first_volume_24h, + DATA [ARRAY_SIZE(data)-1] :market_cap AS last_market_cap, + DATA [ARRAY_SIZE(data)-1] :max_supply AS last_max_supply, + DATA [ARRAY_SIZE(data)-1] :price AS last_price, + DATA [ARRAY_SIZE(data)-1] :total_supply AS last_total_supply, DATA [ARRAY_SIZE(data)-1] :volume_24h AS last_volume_24h, SYSDATE() AS _updated_timestamp, concat_ws( diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql index 5e2e72c8..ebaa6259 100644 --- a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql @@ -35,6 +35,10 @@ SELECT MEDIAN(volume_24h) AS median_volume_24h, MIN(circulating_supply) AS min_circulating_supply, MIN(market_cap) AS min_market_cap, + MIN(max_supply) AS min_max_supply, + MIN(price) AS min_price, + MIN(total_supply) AS min_total_supply, + MIN(volume_24h) AS min_volume_24h, ARRAY_AGG( OBJECT_CONSTRUCT( 'circulating_supply', @@ -57,18 +61,14 @@ SELECT recorded_at ) AS DATA, DATA [0] :market_cap AS first_market_cap, - DATA [ARRAY_SIZE(data)-1] :market_cap AS last_market_cap, - MIN(max_supply) AS min_max_supply, DATA [0] :max_supply AS first_max_supply, - DATA [ARRAY_SIZE(data)-1] :max_supply AS last_max_supply, - MIN(price) AS min_price, DATA [0] :price AS first_price, - DATA [ARRAY_SIZE(data)-1] :price AS last_price, - MIN(total_supply) AS min_total_supply, DATA [0] :total_supply AS first_total_supply, - DATA [ARRAY_SIZE(data)-1] :total_supply AS last_total_supply, - MIN(volume_24h) AS min_volume_24h, DATA [0] :volume_24h AS first_volume_24h, + DATA [ARRAY_SIZE(data)-1] :market_cap AS last_market_cap, + DATA [ARRAY_SIZE(data)-1] :max_supply AS last_max_supply, + DATA [ARRAY_SIZE(data)-1] :price AS last_price, + DATA [ARRAY_SIZE(data)-1] :total_supply AS last_total_supply, DATA [ARRAY_SIZE(data)-1] :volume_24h AS last_volume_24h, SYSDATE() AS _updated_timestamp, concat_ws( From 368ae3d5bfd07f2e22da18105222ac522ea7ded5 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Thu, 20 Jan 2022 13:53:58 -0800 Subject: [PATCH 03/45] test --- models/algorand/gold/algorand__key_registration_transaction.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/algorand/gold/algorand__key_registration_transaction.sql b/models/algorand/gold/algorand__key_registration_transaction.sql index 94b46343..3f2cce51 100644 --- a/models/algorand/gold/algorand__key_registration_transaction.sql +++ b/models/algorand/gold/algorand__key_registration_transaction.sql @@ -17,7 +17,7 @@ SELECT vote_first, vote_last, vote_keydilution, - tx_type, + tx_typxxe, tx_type_name, genisis_hash, tx_message, From 23cfde147f137667e81e0a3f901d585d15e8ab05 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Fri, 21 Jan 2022 10:24:52 -0800 Subject: [PATCH 04/45] undo test --- models/algorand/gold/algorand__key_registration_transaction.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/algorand/gold/algorand__key_registration_transaction.sql b/models/algorand/gold/algorand__key_registration_transaction.sql index 3f2cce51..94b46343 100644 --- a/models/algorand/gold/algorand__key_registration_transaction.sql +++ b/models/algorand/gold/algorand__key_registration_transaction.sql @@ -17,7 +17,7 @@ SELECT vote_first, vote_last, vote_keydilution, - tx_typxxe, + tx_type, tx_type_name, genisis_hash, tx_message, From af429d041907f680beb07e0c38dd65a7dd18ea28 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Thu, 27 Jan 2022 13:09:46 -0800 Subject: [PATCH 05/45] - wip --- .../{ => gold}/crosschain__address_labels.sql | 0 .../{ => gold}/crosschain__address_labels.yml | 0 .../gold/crosschain__f_market_stats_usd.sql | 23 +++++++++++ .../gold/crosschain__f_market_stats_usd.yml | 38 +++++++++++++++++++ ...osschain__f_market_stats_usd_agg_daily.sql | 8 ++-- ...osschain__f_market_stats_usd_agg_daily.yml | 12 ------ ...sschain__f_market_stats_usd_agg_hourly.sql | 8 ++-- ...sschain__f_market_stats_usd_agg_hourly.yml | 12 ------ .../silver_crosschain__f_market_stats_usd.sql | 5 ++- .../silver_crosschain__f_market_stats_usd.yml | 12 ------ 10 files changed, 71 insertions(+), 47 deletions(-) rename models/crosschain/{ => gold}/crosschain__address_labels.sql (100%) rename models/crosschain/{ => gold}/crosschain__address_labels.yml (100%) create mode 100644 models/crosschain/gold/crosschain__f_market_stats_usd.sql create mode 100644 models/crosschain/gold/crosschain__f_market_stats_usd.yml diff --git a/models/crosschain/crosschain__address_labels.sql b/models/crosschain/gold/crosschain__address_labels.sql similarity index 100% rename from models/crosschain/crosschain__address_labels.sql rename to models/crosschain/gold/crosschain__address_labels.sql diff --git a/models/crosschain/crosschain__address_labels.yml b/models/crosschain/gold/crosschain__address_labels.yml similarity index 100% rename from models/crosschain/crosschain__address_labels.yml rename to models/crosschain/gold/crosschain__address_labels.yml diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd.sql b/models/crosschain/gold/crosschain__f_market_stats_usd.sql new file mode 100644 index 00000000..454de506 --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = "view", + unique_key = "_unique_key", + tags = ["crosschain", "gold", "fact", "f_stats_usd"] +) }} + +SELECT + asset_id, + provider, + platform, + recorded_at, + circulating_supply, + market_cap, + price, + max_supply, + total_supply, + volume_24h, + _updated, + _unique_key +FROM + {{ ref( + "silver_crosschain__f_market_stats_usd" + ) }} diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd.yml b/models/crosschain/gold/crosschain__f_market_stats_usd.yml new file mode 100644 index 00000000..8fe57ee2 --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: crosschain__f_market_stats_usd + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - NAME + - SYMBOL + - PROVIDER + - RECORDED_AT + columns: + - name: ASSET_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: NAME + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: RECORDED_AT + tests: + - not_null + - name: CIRCULATING_SUPPLY + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: _UPDATED + tests: + - not_null + - name: _UNIQUE_KEY + tests: + - not_null diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql index 16a0339b..c4c92d4b 100644 --- a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql @@ -2,14 +2,13 @@ materialized = "incremental", unique_key = "_unique_key", incremental_strategy = "merge", - tags = ["snowflake", "market", "stats", "gold", "fact", "f_stats_usd", "daily"] + tags = ["crosschain", "gold", "fact", "f_stats_usd", "aggregate", "daily"] ) }} SELECT asset_id, provider, - NAME, - symbol, + platform, DATE_TRUNC( DAY, recorded_at @@ -101,8 +100,7 @@ WHERE GROUP BY asset_id, provider, - NAME, - symbol, + platform, DATE_TRUNC( DAY, recorded_at diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml index fef2d560..37bbd301 100644 --- a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml @@ -27,21 +27,9 @@ models: - name: CIRCULATING_SUPPLY tests: - not_null - - name: MARKET_CAP - tests: - - not_null - name: PRICE tests: - not_null - - name: MAX_SUPPLY - tests: - - not_null - - name: TOTAL_SUPPLY - tests: - - not_null - - name: VOLUME_24H - tests: - - not_null - name: _UPDATED tests: - not_null diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql index ebaa6259..898df4ab 100644 --- a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql @@ -2,14 +2,13 @@ materialized = "incremental", unique_key = "_unique_key", incremental_strategy = "merge", - tags = ["snowflake", "market", "stats", "gold", "fact", "f_stats_usd", "hourly"] + tags = ["crosschain", "gold", "fact", "f_stats_usd", "aggregate", "daily"] ) }} SELECT asset_id, provider, - NAME, - symbol, + platform, DATE_TRUNC( HOUR, recorded_at @@ -101,8 +100,7 @@ WHERE GROUP BY asset_id, provider, - NAME, - symbol, + platform, DATE_TRUNC( HOUR, recorded_at diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml index bd5e5f12..53a88a99 100644 --- a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml @@ -27,21 +27,9 @@ models: - name: CIRCULATING_SUPPLY tests: - not_null - - name: MARKET_CAP - tests: - - not_null - name: PRICE tests: - not_null - - name: MAX_SUPPLY - tests: - - not_null - - name: TOTAL_SUPPLY - tests: - - not_null - - name: VOLUME_24H - tests: - - not_null - name: _UPDATED tests: - not_null diff --git a/models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql index 09ba42a1..570704c6 100644 --- a/models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql +++ b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql @@ -7,7 +7,10 @@ ) }} SELECT - DISTINCT asset_id, + COALESCE( + asset_id, + '#UNKNOWN' + ) AS asset_id, provider, NAME, symbol, diff --git a/models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml index ab3f8d73..a7a26b12 100644 --- a/models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml +++ b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml @@ -27,21 +27,9 @@ models: - name: CIRCULATING_SUPPLY tests: - not_null - - name: MARKET_CAP - tests: - - not_null - name: PRICE tests: - not_null - - name: MAX_SUPPLY - tests: - - not_null - - name: TOTAL_SUPPLY - tests: - - not_null - - name: VOLUME_24H - tests: - - not_null - name: _UPDATED tests: - not_null From 752bc792530725e19a7f2572275f2d199299d3b5 Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Thu, 23 Dec 2021 12:44:17 -0600 Subject: [PATCH 06/45] merged in updates in main From a696c67283fdcaa4d732fda6f95575f812f7c2a9 Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Mon, 10 Jan 2022 09:35:29 -0600 Subject: [PATCH 07/45] Current Solana Tables, Monday jan 10 --- models/solana/gold/solana__swaps.sql | 133 ++++++++++++++++++ models/solana/gold/solana__swaps.yml | 54 +++++++ models/solana/gold/solana__transactions.sql | 6 +- models/solana/gold/solana__transfers.sql | 61 ++++++++ models/solana/gold/solana__transfers.yml | 47 +++++++ models/solana/gold/solana__votes.sql | 25 ++++ models/solana/gold/solana__votes.yml | 41 ++++++ .../solana/silver/silver_solana__events.sql | 58 ++++++++ .../solana/silver/silver_solana__events.yml | 45 ++++++ 9 files changed, 467 insertions(+), 3 deletions(-) create mode 100644 models/solana/gold/solana__swaps.sql create mode 100644 models/solana/gold/solana__swaps.yml create mode 100644 models/solana/gold/solana__transfers.sql create mode 100644 models/solana/gold/solana__transfers.yml create mode 100644 models/solana/gold/solana__votes.sql create mode 100644 models/solana/gold/solana__votes.yml create mode 100644 models/solana/silver/silver_solana__events.sql create mode 100644 models/solana/silver/silver_solana__events.yml diff --git a/models/solana/gold/solana__swaps.sql b/models/solana/gold/solana__swaps.sql new file mode 100644 index 00000000..7dc04acc --- /dev/null +++ b/models/solana/gold/solana__swaps.sql @@ -0,0 +1,133 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', block_id, tx_id)", + incremental_strategy = 'delete+insert', + tags = ['snowflake', 'solana', 'gold_solana', 'solana_events', 'solana_swaps'] +) }} + +WITH base_table AS ( + SELECT + block_timestamp, + block_id, + blockchain, + recent_blockhash, + tx_id, + event_info[0]:parsed:info:source :: STRING AS SENDER, + COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + ) AS TOKEN_SENT, + CASE WHEN COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + ) = '48F1neXh5bGgKr8G6CM6tFZkaC51UgtVb5pqGLC27Doi' + OR COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + )= '2Zkvs84qBpMvWFSh9MFek2ZPAqAEjfufrs7Sbu42PbzBRTSv5Qx1bTWTGsbaSsuAPe4yx6ku3KmwVaGRbdANytF6' + OR COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + ) = 'DD6oh3HRCvMzqHkGeUW3za4pLgWNPJdV6aNYW3gVjXXi' + OR COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + ) = 'DQyrAcCrDXQ7NeoqGgDCZwBvWDcYmFCjSb9JtteuvPpz' + OR COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + ) = '9wArqjNDHm43RJGSBSXuHygqJzEaW1xYA3XmmpU671kC' + OR COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + ) = '5sHXydo4z6p2x7Jqfx8T4FnPRM9S2y8wpgNG1pAZf9Cg' + OR COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + ) = 'ArwS8uBqFTKNgLAy7r3poSCujCazc4GiqBZguUzQPrKj' + OR COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + ) = '4aiKnDHFmNnLsopVsDyRBh8sbVohZYgdGzh3P9orpqNB' + THEN event_meta[0]:parsed:info:amount/POW(10,9) :: INTEGER + ELSE event_meta[0]:parsed:info:amount/POW(10,6) :: INTEGER END AS AMOUNT_SENT, + event_info[1]:parsed:info:mint :: STRING AS TOKEN_RECEIVED, + CASE WHEN event_info[1]:parsed:info:mint :: STRING = 'mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So' OR event_info[1]:parsed:info:mint :: STRING = 'StepAscQoEioFxxWGnh2sLBDFp9d8rvKz2Yp39iDpyT' OR event_info[1]:parsed:info:mint :: STRING = 'So11111111111111111111111111111111111111112' THEN COALESCE( + event_meta[2]:parsed:info:amount/POW(10,9) :: INTEGER, + event_meta[1]:parsed:info:amount/POW(10,9) :: INTEGER + ) + ELSE COALESCE( + event_meta[2]:parsed:info:amount/POW(10,6) :: INTEGER, + event_meta[1]:parsed:info:amount/POW(10,6) :: INTEGER + ) END AS AMOUNT_RECEIVED, + succeeded, + ingested_at + + FROM {{ ref('silver_solana__events') }} + + WHERE event_info[1]:program = 'spl-token' + AND event_info[1]:programId = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' + AND event_meta[0]:parsed:info:amount/POW(10,6) :: INTEGER IS NOT NULL -- IF THESE ARE NULL, THEN NFT MINT + AND event_meta[1]:parsed:info:amount/POW(10,6) :: INTEGER IS NOT NULL + + {% if is_incremental() %} + AND ingested_at >= ( + SELECT + MAX( + ingested_at + ) + FROM + {{ this }} + ) + {% endif %} +), + +token_symbols AS ( + SELECT + m.asset_id, + p.name, + p.symbol, + m.token_address +FROM {{ source('shared', 'prices_v2')}} p +JOIN {{ source('shared', 'market_asset_metadata') }} m ON m.asset_id = p.asset_id +WHERE m.token_address IS NOT NULL +{% if is_incremental() %} + AND ingested_at >= ( + SELECT + MAX( + ingested_at + ) + FROM + {{ this }} + ) + {% endif %} + +ORDER BY p.recorded_at DESC +) + +SELECT + block_timestamp, + block_id, + blockchain, + recent_blockhash, + tx_id, + sender, + token_sent, + t.symbol AS token_sent_symbol, + amount_sent, + token_received, + f.symbol AS token_received_symbol, + amount_received, + succeeded, + ingested_at +FROM base_table b + +LEFT OUTER JOIN token_symbols t +ON token_received = t.token_address + +LEFT OUTER JOIN token_symbols f +ON token_received = f.token_address + +qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +ORDER BY + ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/gold/solana__swaps.yml b/models/solana/gold/solana__swaps.yml new file mode 100644 index 00000000..da080ef2 --- /dev/null +++ b/models/solana/gold/solana__swaps.yml @@ -0,0 +1,54 @@ +version: 2 +models: + - name: solana__swaps + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - TX_ID + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: RECENT_BLOCKHASH + tests: + - not_null + - name: TX_ID + tests: + - not_null + - name: SENDER + tests: + - not_null + - name: TOKEN_SENT + tests: + - not_null + - name: TOKEN_SENT_SYMBOL + tests: + - not_null + - name: AMOUNT_SENT + tests: + - not_null + - name: TOKEN_RECEIVED + tests: + - not_null + - name: TOKEN_RECEIVED_SYMBOL + tests: + - not_null + - name: AMOUNT_RECEIVED + tests: + - not_null + - name: SUCCEEDED + tests: + - not_null + - name: INGESTED_AT + tests: + - not_null \ No newline at end of file diff --git a/models/solana/gold/solana__transactions.sql b/models/solana/gold/solana__transactions.sql index 8a6b99b3..68dd5de6 100644 --- a/models/solana/gold/solana__transactions.sql +++ b/models/solana/gold/solana__transactions.sql @@ -31,10 +31,10 @@ FROM {{ ref('silver_solana__transactions') }} LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS contract_labels -ON program_id = contract_labels.address AND contract_labels.blockchain = 'solana' +ON program_id COLLATE 'en-ci' = contract_labels.address AND contract_labels.blockchain = 'solana' LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS to_labels -ON tx_to_address = to_labels.address AND to_labels.blockchain = 'solana' +ON tx_to_address COLLATE 'en-ci' = to_labels.address AND to_labels.blockchain = 'solana' LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS from_labels -ON tx_from_address = from_labels.address AND from_labels.blockchain = 'solana' \ No newline at end of file +ON tx_from_address COLLATE 'en-ci' = from_labels.address AND from_labels.blockchain = 'solana' \ No newline at end of file diff --git a/models/solana/gold/solana__transfers.sql b/models/solana/gold/solana__transfers.sql new file mode 100644 index 00000000..9d83b5ae --- /dev/null +++ b/models/solana/gold/solana__transfers.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'view', + tags = ['snowflake', 'solana', 'gold_solana', 'solana_transfers', 'solana_user_transfers'] +) }} + +WITH base_table AS ( + SELECT + block_timestamp, + block_id, + blockchain, + recent_blockhash, + tx_id, + ei.value:parsed:info:authority :: STRING AS tx_from_address, + ei.value:parsed:info:destination :: STRING AS tx_to_address, + ei.value:parsed:info:amount/POW(10,9) :: INTEGER AS solana_amount, + succeeded, + ei.value:programId :: STRING AS program_id, + ingested_at + FROM {{ ref('silver_solana__events') }}, + + LATERAL FLATTEN ( + input => event_info ) ei + + WHERE ei.value:parsed:type :: STRING = 'transfer' + AND ei.value:program :: STRING = 'spl-token' + +) + +SELECT + block_timestamp, + block_id, + e.blockchain, + recent_blockhash, + tx_id, + e.tx_from_address, + from_labels.label_type AS tx_from_label, + from_labels.label_subtype AS tx_from_label_subtype, + from_labels.program_name as tx_from_program_name, + from_labels.project_name as tx_from_address_name, + e.tx_to_address, + to_labels.label_type AS tx_to_label, + to_labels.label_subtype AS tx_to_label_subtype, + to_labels.program_name as tx_to_program_name, + to_labels.project_name as tx_to_address_name, + solana_amount, + succeeded, + e.program_id, + contract_labels.project_name AS program_name, + ingested_at, + CASE WHEN program_id <> 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' THEN TRUE ELSE FALSE END AS transfer_tx_flag +FROM base_table e + +LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS contract_labels +ON e.program_id COLLATE 'en-ci' = contract_labels.address AND contract_labels.blockchain = 'solana' + +LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS to_labels +ON e.tx_to_address COLLATE 'en-ci' = to_labels.address AND to_labels.blockchain = 'solana' + +LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS from_labels +ON e.tx_from_address COLLATE 'en-ci' = from_labels.address AND from_labels.blockchain = 'solana' + diff --git a/models/solana/gold/solana__transfers.yml b/models/solana/gold/solana__transfers.yml new file mode 100644 index 00000000..9bb7cb49 --- /dev/null +++ b/models/solana/gold/solana__transfers.yml @@ -0,0 +1,47 @@ +version: 2 +models: + - name: solana__transfers + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - TX_ID + - TX_TO_ADDRESS + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_ID + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: RECENT_BLOCKHASH + tests: + - not_null + - name: TX_ID + tests: + - not_null + - name: TX_FROM_ADDRESS + tests: + - not_null: + where: TRANSFER_TX_FLAG = TRUE + - name: TX_TO_ADDRESS + tests: + - not_null + - name: SOLANA_AMOUNT + tests: + - not_null + - name: SUCCEEDED + tests: + - not_null + - name: PROGRAM_ID + tests: + - not_null + - name: INGESTED_AT + tests: + - not_null \ No newline at end of file diff --git a/models/solana/gold/solana__votes.sql b/models/solana/gold/solana__votes.sql new file mode 100644 index 00000000..f0418923 --- /dev/null +++ b/models/solana/gold/solana__votes.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view', + tags = ['snowflake', 'solana', 'gold_solana', 'solana_events', 'solana_votes'] +) }} + +SELECT + block_timestamp, + block_id, + e.blockchain, + recent_blockhash, + tx_id, + succeeded, + event_info:program :: STRING AS program_type, + event_info:programId :: STRING AS program_id, + contract_labels.project_name :: STRING AS program_name, + event_info:parsed:info:voteAccount :: STRING AS vote_account, + event_info:parsed:info:voteAuthority :: STRING AS vote_authority, + ingested_at, + transfer_tx_flag +FROM {{ ref('silver_solana__events') }} e + +LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS contract_labels +ON event_info:programId :: STRING COLLATE 'en-ci' = contract_labels.address AND contract_labels.blockchain = 'solana' + +WHERE event_info:program = 'vote' \ No newline at end of file diff --git a/models/solana/gold/solana__votes.yml b/models/solana/gold/solana__votes.yml new file mode 100644 index 00000000..2c8a4529 --- /dev/null +++ b/models/solana/gold/solana__votes.yml @@ -0,0 +1,41 @@ +version: 2 +models: + - name: solana__votes + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - TX_ID + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_ID + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: RECENT_BLOCKHASH + tests: + - not_null + - name: TX_ID + tests: + - not_null + - name: SUCCEEDED + tests: + - not_null + - name: PROGRAM_TYPE + tests: + - not_null: + where: TRANSFER_TX_FLAG = TRUE + - name: PROGRAM_ID + tests: + - not_null: + where: TRANSFER_TX_FLAG = TRUE + - name: INGESTED_AT + tests: + - not_null \ No newline at end of file diff --git a/models/solana/silver/silver_solana__events.sql b/models/solana/silver/silver_solana__events.sql new file mode 100644 index 00000000..50b7f3c7 --- /dev/null +++ b/models/solana/silver/silver_solana__events.sql @@ -0,0 +1,58 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', block_id, tx_id)", + incremental_strategy = 'delete+insert', + tags = ['snowflake', 'solana', 'silver_solana', 'solana_events'] +) }} + +WITH base_table as ( + SELECT + block_timestamp :: TIMESTAMP AS block_timestamp, + offset_id :: INTEGER AS block_id, + chain_id :: STRING AS blockchain, + tx :transaction:message:recentBlockhash :: STRING AS recent_blockhash, + tx_id :: STRING AS tx_id, + tx :meta:preTokenBalances[0]:owner :: STRING AS tx_from_address, + tx :meta:postTokenBalances[0]:owner :: STRING AS tx_to_address, + CASE WHEN tx :meta:status:Err IS NULL THEN TRUE ELSE FALSE END AS succeeded, + tx :meta:innerInstructions[0]:instructions :: ARRAY AS event_meta, + tx :transaction:message:instructions :: ARRAY AS event_info, + ingested_at :: TIMESTAMP AS ingested_at, + CASE WHEN len(tx :meta:postTokenBalances[0]) > 0 AND len(tx :meta:preTokenBalances[0]) > 0 AND SUCCEEDED = TRUE THEN TRUE ELSE FALSE END AS transfer_tx_flag + + FROM {{ ref('bronze_solana__transactions') }} + + WHERE + 1 = 1 + AND tx :transaction:message:instructions[0]:parsed:type :: STRING IS NOT NULL + + {% if is_incremental() %} + AND ingested_at >= ( + SELECT + MAX( + ingested_at + ) + FROM + {{ this }} + ) + {% endif %} +) + +SELECT + block_timestamp, + block_id, + blockchain, + recent_blockhash, + tx_id, + tx_from_address, + tx_to_address, + succeeded, + event_meta, + event_info, + ingested_at, + transfer_tx_flag +FROM base_table + +qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +ORDER BY + ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/silver/silver_solana__events.yml b/models/solana/silver/silver_solana__events.yml new file mode 100644 index 00000000..b89d6eaf --- /dev/null +++ b/models/solana/silver/silver_solana__events.yml @@ -0,0 +1,45 @@ +version: 2 +models: + - name: silver_solana__events + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - TX_ID + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_ID + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: RECENT_BLOCKHASH + tests: + - not_null + - name: TX_ID + tests: + - not_null + - name: TX_FROM_ADDRESS + tests: + - not_null: + where: TRANSFER_TX_FLAG = TRUE + - name: TX_TO_ADDRESS + tests: + - not_null: + where: TRANSFER_TX_FLAG = TRUE + - name: SUCCEEDED + tests: + - not_null + - name: EVENT_INFO + tests: + - not_null: + where: TRANSFER_TX_FLAG = TRUE + - name: INGESTED_AT + tests: + - not_null \ No newline at end of file From bfd717fa283ef42fe9db65cf1fa131fd2c5b5b95 Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Thu, 13 Jan 2022 16:54:56 -0600 Subject: [PATCH 08/45] Solana silver and gold tables --- models/solana/gold/solana__blocks.yml | 2 +- models/solana/gold/solana__events.sql | 25 ++ models/solana/gold/solana__events.yml | 45 ++++ models/solana/gold/solana__swaps.sql | 119 +-------- models/solana/gold/solana__swaps.yml | 8 +- models/solana/gold/solana__transactions.sql | 20 +- models/solana/gold/solana__transactions.yml | 2 +- models/solana/gold/solana__transfers.sql | 66 ++--- models/solana/gold/solana__transfers.yml | 3 +- models/solana/gold/solana__votes.sql | 20 +- models/solana/gold/solana__votes.yml | 2 +- .../solana/silver/silver_solana__blocks.sql | 1 + .../solana/silver/silver_solana__blocks.yml | 2 +- .../silver/silver_solana__contract_names.sql | 57 ----- .../silver/silver_solana__contract_names.yml | 36 --- .../solana/silver/silver_solana__events.sql | 8 +- .../solana/silver/silver_solana__events.yml | 2 +- models/solana/silver/silver_solana__swaps.sql | 230 ++++++++++++++++++ models/solana/silver/silver_solana__swaps.yml | 48 ++++ .../silver/silver_solana__transactions.sql | 2 + .../silver/silver_solana__transactions.yml | 2 +- models/solana/silver/silver_solana__votes.sql | 60 +++++ models/solana/silver/silver_solana__votes.yml | 42 ++++ 23 files changed, 501 insertions(+), 301 deletions(-) create mode 100644 models/solana/gold/solana__events.sql create mode 100644 models/solana/gold/solana__events.yml delete mode 100644 models/solana/silver/silver_solana__contract_names.sql delete mode 100644 models/solana/silver/silver_solana__contract_names.yml create mode 100644 models/solana/silver/silver_solana__swaps.sql create mode 100644 models/solana/silver/silver_solana__swaps.yml create mode 100644 models/solana/silver/silver_solana__votes.sql create mode 100644 models/solana/silver/silver_solana__votes.yml diff --git a/models/solana/gold/solana__blocks.yml b/models/solana/gold/solana__blocks.yml index eb70d822..2e019105 100644 --- a/models/solana/gold/solana__blocks.yml +++ b/models/solana/gold/solana__blocks.yml @@ -14,7 +14,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 1 + interval: 3 - name: NETWORK tests: - not_null diff --git a/models/solana/gold/solana__events.sql b/models/solana/gold/solana__events.sql new file mode 100644 index 00000000..b0877fa7 --- /dev/null +++ b/models/solana/gold/solana__events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view', + tags = ['snowflake', 'solana', 'gold_solana', 'solana_events'] +) }} + +SELECT + block_timestamp, + block_id, + blockchain, + recent_blockhash, + tx_id, + tx_from_address, + tx_to_address, + succeeded, + event_meta, + postTokenBalances, + event_info, + ingested_at, + transfer_tx_flag + +FROM {{ ref('silver_solana__events') }} + +qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +ORDER BY + ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/gold/solana__events.yml b/models/solana/gold/solana__events.yml new file mode 100644 index 00000000..b17dbd7a --- /dev/null +++ b/models/solana/gold/solana__events.yml @@ -0,0 +1,45 @@ +version: 2 +models: + - name: solana__events + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - TX_ID + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - name: BLOCK_ID + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: RECENT_BLOCKHASH + tests: + - not_null + - name: TX_ID + tests: + - not_null + - name: TX_FROM_ADDRESS + tests: + - not_null: + where: TRANSFER_TX_FLAG = TRUE + - name: TX_TO_ADDRESS + tests: + - not_null: + where: TRANSFER_TX_FLAG = TRUE + - name: SUCCEEDED + tests: + - not_null + - name: EVENT_INFO + tests: + - not_null: + where: TRANSFER_TX_FLAG = TRUE + - name: INGESTED_AT + tests: + - not_null \ No newline at end of file diff --git a/models/solana/gold/solana__swaps.sql b/models/solana/gold/solana__swaps.sql index 7dc04acc..ef6cf009 100644 --- a/models/solana/gold/solana__swaps.sql +++ b/models/solana/gold/solana__swaps.sql @@ -1,110 +1,8 @@ {{ config( - materialized = 'incremental', - unique_key = "CONCAT_WS('-', block_id, tx_id)", - incremental_strategy = 'delete+insert', + materialized = 'view', tags = ['snowflake', 'solana', 'gold_solana', 'solana_events', 'solana_swaps'] ) }} -WITH base_table AS ( - SELECT - block_timestamp, - block_id, - blockchain, - recent_blockhash, - tx_id, - event_info[0]:parsed:info:source :: STRING AS SENDER, - COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - ) AS TOKEN_SENT, - CASE WHEN COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - ) = '48F1neXh5bGgKr8G6CM6tFZkaC51UgtVb5pqGLC27Doi' - OR COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - )= '2Zkvs84qBpMvWFSh9MFek2ZPAqAEjfufrs7Sbu42PbzBRTSv5Qx1bTWTGsbaSsuAPe4yx6ku3KmwVaGRbdANytF6' - OR COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - ) = 'DD6oh3HRCvMzqHkGeUW3za4pLgWNPJdV6aNYW3gVjXXi' - OR COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - ) = 'DQyrAcCrDXQ7NeoqGgDCZwBvWDcYmFCjSb9JtteuvPpz' - OR COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - ) = '9wArqjNDHm43RJGSBSXuHygqJzEaW1xYA3XmmpU671kC' - OR COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - ) = '5sHXydo4z6p2x7Jqfx8T4FnPRM9S2y8wpgNG1pAZf9Cg' - OR COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - ) = 'ArwS8uBqFTKNgLAy7r3poSCujCazc4GiqBZguUzQPrKj' - OR COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - ) = '4aiKnDHFmNnLsopVsDyRBh8sbVohZYgdGzh3P9orpqNB' - THEN event_meta[0]:parsed:info:amount/POW(10,9) :: INTEGER - ELSE event_meta[0]:parsed:info:amount/POW(10,6) :: INTEGER END AS AMOUNT_SENT, - event_info[1]:parsed:info:mint :: STRING AS TOKEN_RECEIVED, - CASE WHEN event_info[1]:parsed:info:mint :: STRING = 'mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So' OR event_info[1]:parsed:info:mint :: STRING = 'StepAscQoEioFxxWGnh2sLBDFp9d8rvKz2Yp39iDpyT' OR event_info[1]:parsed:info:mint :: STRING = 'So11111111111111111111111111111111111111112' THEN COALESCE( - event_meta[2]:parsed:info:amount/POW(10,9) :: INTEGER, - event_meta[1]:parsed:info:amount/POW(10,9) :: INTEGER - ) - ELSE COALESCE( - event_meta[2]:parsed:info:amount/POW(10,6) :: INTEGER, - event_meta[1]:parsed:info:amount/POW(10,6) :: INTEGER - ) END AS AMOUNT_RECEIVED, - succeeded, - ingested_at - - FROM {{ ref('silver_solana__events') }} - - WHERE event_info[1]:program = 'spl-token' - AND event_info[1]:programId = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' - AND event_meta[0]:parsed:info:amount/POW(10,6) :: INTEGER IS NOT NULL -- IF THESE ARE NULL, THEN NFT MINT - AND event_meta[1]:parsed:info:amount/POW(10,6) :: INTEGER IS NOT NULL - - {% if is_incremental() %} - AND ingested_at >= ( - SELECT - MAX( - ingested_at - ) - FROM - {{ this }} - ) - {% endif %} -), - -token_symbols AS ( - SELECT - m.asset_id, - p.name, - p.symbol, - m.token_address -FROM {{ source('shared', 'prices_v2')}} p -JOIN {{ source('shared', 'market_asset_metadata') }} m ON m.asset_id = p.asset_id -WHERE m.token_address IS NOT NULL -{% if is_incremental() %} - AND ingested_at >= ( - SELECT - MAX( - ingested_at - ) - FROM - {{ this }} - ) - {% endif %} - -ORDER BY p.recorded_at DESC -) - SELECT block_timestamp, block_id, @@ -112,21 +10,14 @@ SELECT recent_blockhash, tx_id, sender, - token_sent, - t.symbol AS token_sent_symbol, + token_sent, amount_sent, - token_received, - f.symbol AS token_received_symbol, + token_received, amount_received, succeeded, ingested_at -FROM base_table b - -LEFT OUTER JOIN token_symbols t -ON token_received = t.token_address - -LEFT OUTER JOIN token_symbols f -ON token_received = f.token_address + +FROM {{ ref('silver_solana__swaps') }} qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id ORDER BY diff --git a/models/solana/gold/solana__swaps.yml b/models/solana/gold/solana__swaps.yml index da080ef2..70241ac7 100644 --- a/models/solana/gold/solana__swaps.yml +++ b/models/solana/gold/solana__swaps.yml @@ -12,7 +12,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 2 + interval: 3 - name: BLOCK_ID tests: - not_null @@ -31,18 +31,12 @@ models: - name: TOKEN_SENT tests: - not_null - - name: TOKEN_SENT_SYMBOL - tests: - - not_null - name: AMOUNT_SENT tests: - not_null - name: TOKEN_RECEIVED tests: - not_null - - name: TOKEN_RECEIVED_SYMBOL - tests: - - not_null - name: AMOUNT_RECEIVED tests: - not_null diff --git a/models/solana/gold/solana__transactions.sql b/models/solana/gold/solana__transactions.sql index 68dd5de6..524c0111 100644 --- a/models/solana/gold/solana__transactions.sql +++ b/models/solana/gold/solana__transactions.sql @@ -12,29 +12,15 @@ SELECT pre_mint, post_mint, tx_from_address, - from_labels.label_type AS tx_from_label, - from_labels.label_subtype AS tx_from_label_subtype, - from_labels.program_name as tx_from_program_name, - from_labels.project_name as tx_from_address_name, tx_to_address, - to_labels.label_type AS tx_to_label, - to_labels.label_subtype AS tx_to_label_subtype, - to_labels.program_name as tx_to_program_name, - to_labels.project_name as tx_to_address_name, fee, succeeded, program_id, - contract_labels.project_name AS program_name, ingested_at, transfer_tx_flag FROM {{ ref('silver_solana__transactions') }} -LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS contract_labels -ON program_id COLLATE 'en-ci' = contract_labels.address AND contract_labels.blockchain = 'solana' - -LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS to_labels -ON tx_to_address COLLATE 'en-ci' = to_labels.address AND to_labels.blockchain = 'solana' - -LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS from_labels -ON tx_from_address COLLATE 'en-ci' = from_labels.address AND from_labels.blockchain = 'solana' \ No newline at end of file +qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +ORDER BY + ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/gold/solana__transactions.yml b/models/solana/gold/solana__transactions.yml index 16608409..cb68905c 100644 --- a/models/solana/gold/solana__transactions.yml +++ b/models/solana/gold/solana__transactions.yml @@ -12,7 +12,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 1 + interval: 3 - name: BLOCK_ID tests: - not_null diff --git a/models/solana/gold/solana__transfers.sql b/models/solana/gold/solana__transfers.sql index 9d83b5ae..60b146da 100644 --- a/models/solana/gold/solana__transfers.sql +++ b/models/solana/gold/solana__transfers.sql @@ -3,59 +3,27 @@ tags = ['snowflake', 'solana', 'gold_solana', 'solana_transfers', 'solana_user_transfers'] ) }} -WITH base_table AS ( - SELECT - block_timestamp, - block_id, - blockchain, - recent_blockhash, - tx_id, - ei.value:parsed:info:authority :: STRING AS tx_from_address, - ei.value:parsed:info:destination :: STRING AS tx_to_address, - ei.value:parsed:info:amount/POW(10,9) :: INTEGER AS solana_amount, - succeeded, - ei.value:programId :: STRING AS program_id, - ingested_at - FROM {{ ref('silver_solana__events') }}, - - LATERAL FLATTEN ( - input => event_info ) ei - - WHERE ei.value:parsed:type :: STRING = 'transfer' - AND ei.value:program :: STRING = 'spl-token' - -) - -SELECT +SELECT block_timestamp, - block_id, - e.blockchain, + block_id, + blockchain, recent_blockhash, tx_id, - e.tx_from_address, - from_labels.label_type AS tx_from_label, - from_labels.label_subtype AS tx_from_label_subtype, - from_labels.program_name as tx_from_program_name, - from_labels.project_name as tx_from_address_name, - e.tx_to_address, - to_labels.label_type AS tx_to_label, - to_labels.label_subtype AS tx_to_label_subtype, - to_labels.program_name as tx_to_program_name, - to_labels.project_name as tx_to_address_name, - solana_amount, + ei.value:parsed:info:authority :: STRING AS tx_from_address, + ei.value:parsed:info:destination :: STRING AS tx_to_address, + ei.value:parsed:info:amount/POW(10,9) :: INTEGER AS solana_amount, succeeded, - e.program_id, - contract_labels.project_name AS program_name, - ingested_at, - CASE WHEN program_id <> 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' THEN TRUE ELSE FALSE END AS transfer_tx_flag -FROM base_table e - -LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS contract_labels -ON e.program_id COLLATE 'en-ci' = contract_labels.address AND contract_labels.blockchain = 'solana' + ei.value:programId :: STRING AS program_id, + ingested_at, + CASE WHEN program_id <> 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' THEN TRUE ELSE FALSE END AS transfer_tx_flag +FROM {{ ref('silver_solana__events') }}, -LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS to_labels -ON e.tx_to_address COLLATE 'en-ci' = to_labels.address AND to_labels.blockchain = 'solana' +LATERAL FLATTEN ( + input => event_info ) ei -LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS from_labels -ON e.tx_from_address COLLATE 'en-ci' = from_labels.address AND from_labels.blockchain = 'solana' +WHERE ei.value:parsed:type :: STRING = 'transfer' +AND ei.value:program :: STRING = 'spl-token' +qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +ORDER BY + ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/gold/solana__transfers.yml b/models/solana/gold/solana__transfers.yml index 9bb7cb49..e5bb3d57 100644 --- a/models/solana/gold/solana__transfers.yml +++ b/models/solana/gold/solana__transfers.yml @@ -6,14 +6,13 @@ models: combination_of_columns: - BLOCK_ID - TX_ID - - TX_TO_ADDRESS columns: - name: BLOCK_TIMESTAMP tests: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 1 + interval: 3 - name: BLOCK_ID tests: - not_null diff --git a/models/solana/gold/solana__votes.sql b/models/solana/gold/solana__votes.sql index f0418923..adb8f397 100644 --- a/models/solana/gold/solana__votes.sql +++ b/models/solana/gold/solana__votes.sql @@ -6,20 +6,18 @@ SELECT block_timestamp, block_id, - e.blockchain, + blockchain, recent_blockhash, tx_id, succeeded, - event_info:program :: STRING AS program_type, - event_info:programId :: STRING AS program_id, - contract_labels.project_name :: STRING AS program_name, - event_info:parsed:info:voteAccount :: STRING AS vote_account, - event_info:parsed:info:voteAuthority :: STRING AS vote_authority, + program_type, + program_id, + vote_account, + vote_authority, ingested_at, transfer_tx_flag -FROM {{ ref('silver_solana__events') }} e +FROM {{ ref('silver_solana__votes') }} -LEFT OUTER JOIN {{ ref('silver_solana__contract_names') }} AS contract_labels -ON event_info:programId :: STRING COLLATE 'en-ci' = contract_labels.address AND contract_labels.blockchain = 'solana' - -WHERE event_info:program = 'vote' \ No newline at end of file +qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +ORDER BY + ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/gold/solana__votes.yml b/models/solana/gold/solana__votes.yml index 2c8a4529..9638dd68 100644 --- a/models/solana/gold/solana__votes.yml +++ b/models/solana/gold/solana__votes.yml @@ -12,7 +12,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 1 + interval: 3 - name: BLOCK_ID tests: - not_null diff --git a/models/solana/silver/silver_solana__blocks.sql b/models/solana/silver/silver_solana__blocks.sql index 8b1672c3..c59f7fab 100644 --- a/models/solana/silver/silver_solana__blocks.sql +++ b/models/solana/silver/silver_solana__blocks.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "block_id", incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], tags = ['snowflake', 'solana', 'silver_solana', 'solana_blocks'] ) }} diff --git a/models/solana/silver/silver_solana__blocks.yml b/models/solana/silver/silver_solana__blocks.yml index b4ab9dc0..679ea33a 100644 --- a/models/solana/silver/silver_solana__blocks.yml +++ b/models/solana/silver/silver_solana__blocks.yml @@ -14,7 +14,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 1 + interval: 3 - name: NETWORK tests: - not_null diff --git a/models/solana/silver/silver_solana__contract_names.sql b/models/solana/silver/silver_solana__contract_names.sql deleted file mode 100644 index 24e9ec97..00000000 --- a/models/solana/silver/silver_solana__contract_names.sql +++ /dev/null @@ -1,57 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "CONCAT_WS('-', blockchain, address, creator)", - incremental_strategy = 'delete+insert', - tags = ['snowflake', 'solana', 'silver_solana', 'solana_contract_names'] -) }} - -WITH base_tables AS ( - - SELECT - *, - split(substr(record_metadata:key::string, 2, len(record_metadata:key::string)-2),'-')[1]::string as blockchain, - to_timestamp(split(substr(record_metadata:key::string, 2, len(record_metadata:key::string)-2),'-')[2]::int) as insert_date - - FROM - {{ source( - 'bronze', - 'prod_address_label_sink_291098491' - ) }} - WHERE - array_size(split(substr(record_metadata:key::string, 2, len(record_metadata:key::string)-2),'-')) = 3 - AND split(substr(record_metadata:key::string, 2, len(record_metadata:key::string)-2),'-')[0] = 'labels' - AND split(substr(record_metadata:key::string, 2, len(record_metadata:key::string)-2),'-')[1]::string = 'solana' - -{% if is_incremental() %} -AND ( - record_metadata :CreateTime :: INT / 1000 -) :: TIMESTAMP :: DATE >= ( - SELECT - DATEADD('day', -1, MAX(system_created_at :: DATE)) - FROM - {{ this }} -) -{% endif %} -) - -SELECT - ( - record_metadata :CreateTime :: INT / 1000 - ) :: TIMESTAMP AS system_created_at, - insert_date, - blockchain, - LOWER(t.value :address :: STRING) AS address, - t.value :creator :: STRING AS creator, - t.value :l1_label :: STRING AS label_type, - t.value :l2_label :: STRING AS label_subtype, - t.value :address_name :: STRING AS program_name, - t.value :project_name :: STRING AS project_name -FROM - base_tables, - LATERAL FLATTEN( - input => record_content - ) t - - qualify(ROW_NUMBER() over(PARTITION BY blockchain, address, creator -ORDER BY - system_created_at DESC)) = 1 diff --git a/models/solana/silver/silver_solana__contract_names.yml b/models/solana/silver/silver_solana__contract_names.yml deleted file mode 100644 index de595042..00000000 --- a/models/solana/silver/silver_solana__contract_names.yml +++ /dev/null @@ -1,36 +0,0 @@ -version: 2 -models: - - name: silver_solana__contract_names - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - BLOCKCHAIN - - CREATOR - - ADDRESS - columns: - - name: BLOCKCHAIN - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_set: - value_set: ['solana'] - - name: CREATOR - tests: - - not_null - - name: INSERT_DATE - tests: - - not_null - - name: LABEL_TYPE - tests: - - not_null - - name: LABEL_SUBTYPE - tests: - - not_null - - name: PROJECT_NAME - tests: - - not_null - - name: PROGRAM_NAME - tests: - - not_null - - name: ADDRESS - tests: - - not_null diff --git a/models/solana/silver/silver_solana__events.sql b/models/solana/silver/silver_solana__events.sql index 50b7f3c7..5c04ca08 100644 --- a/models/solana/silver/silver_solana__events.sql +++ b/models/solana/silver/silver_solana__events.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "CONCAT_WS('-', block_id, tx_id)", incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], tags = ['snowflake', 'solana', 'silver_solana', 'solana_events'] ) }} @@ -15,7 +16,8 @@ WITH base_table as ( tx :meta:preTokenBalances[0]:owner :: STRING AS tx_from_address, tx :meta:postTokenBalances[0]:owner :: STRING AS tx_to_address, CASE WHEN tx :meta:status:Err IS NULL THEN TRUE ELSE FALSE END AS succeeded, - tx :meta:innerInstructions[0]:instructions :: ARRAY AS event_meta, + tx :meta:innerInstructions[0]:instructions :: ARRAY AS event_meta, + tx :meta:postTokenBalances :: ARRAY AS postTokenBalances, tx :transaction:message:instructions :: ARRAY AS event_info, ingested_at :: TIMESTAMP AS ingested_at, CASE WHEN len(tx :meta:postTokenBalances[0]) > 0 AND len(tx :meta:preTokenBalances[0]) > 0 AND SUCCEEDED = TRUE THEN TRUE ELSE FALSE END AS transfer_tx_flag @@ -25,6 +27,7 @@ WITH base_table as ( WHERE 1 = 1 AND tx :transaction:message:instructions[0]:parsed:type :: STRING IS NOT NULL + AND tx :transaction:message:instructions[0]:programId :: STRING <> 'Vote111111111111111111111111111111111111111' {% if is_incremental() %} AND ingested_at >= ( @@ -47,7 +50,8 @@ SELECT tx_from_address, tx_to_address, succeeded, - event_meta, + event_meta, + postTokenBalances, event_info, ingested_at, transfer_tx_flag diff --git a/models/solana/silver/silver_solana__events.yml b/models/solana/silver/silver_solana__events.yml index b89d6eaf..97aca407 100644 --- a/models/solana/silver/silver_solana__events.yml +++ b/models/solana/silver/silver_solana__events.yml @@ -12,7 +12,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 1 + interval: 3 - name: BLOCK_ID tests: - not_null diff --git a/models/solana/silver/silver_solana__swaps.sql b/models/solana/silver/silver_solana__swaps.sql new file mode 100644 index 00000000..d5973228 --- /dev/null +++ b/models/solana/silver/silver_solana__swaps.sql @@ -0,0 +1,230 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', block_id, tx_id)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], + tags = ['snowflake', 'solana', 'silver_solana', 'solana_events', 'solana_swaps'] +) }} + +WITH base_table AS ( + SELECT + block_timestamp, + block_id, + blockchain, + recent_blockhash, + tx_id, + event_info[0]:parsed:info:source :: STRING AS sender, + + CASE WHEN event_info[2]:programId :: STRING IN ('9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin', '675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8') -- SERUM DEX || Raydium Liquidity Pool V4 + THEN event_info[1]:parsed:info:mint :: STRING + WHEN event_info[3]:programId :: STRING = 'SSwpkEEcbUqx4vtoEByFjSkhKdCT862DNVb52nZg1UZ' -- SABER STABLE SWAP + THEN event_meta[1]:parsed:info:destination :: STRING + ELSE COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + ) END AS token_sent_account, + CASE WHEN event_info[3]:programId :: STRING IN ('SSwpkEEcbUqx4vtoEByFjSkhKdCT862DNVb52nZg1UZ', 'SSwpMgqNDsyV7mAgN9ady4bDVu5ySjmmXejXvy2vLt1') -- SABER STABLE SWAP || STEP FINANCE + THEN postTokenBalances[1]:mint :: STRING + WHEN event_info[4]:programId :: STRING = '82yxjeMsvaURa4MbZZ7WZZHfobirZYkH1zF8fmeGtyaQ' -- ORCA AQUAFARM + THEN postTokenBalances[3]:mint :: STRING + WHEN event_info[2]:programId :: STRING IN ('9W959DqEETiGZocYWCQPaJ6sBmUzgfxXfqGeTEdp3aQP', '9W959DqEETiGZocYWCQPaJ6sBmUzgfxXfqGeTEdp3aQP')-- ORCA TOKEN SWAP + THEN postTokenBalances[0]:mint :: STRING + WHEN event_info[2]:programId :: STRING IN ('2nAAsYdXF3eTQzaeUQS3fr4o782dDg8L28mX39Wr5j8N', '31ARfyxZg6fr1J9hVs1NqBWkdeYeCKipuY71bMovNpy9') -- UNKNOWN PROGRAM + THEN postTokenBalances[1]:mint :: STRING + ELSE postTokenBalances[0]:mint :: STRING END AS t0_sent, + + CASE WHEN event_info[2]:programId :: STRING = 'LendZqTs7gn5CTSJU1jWKhKuVpjJGom45nnwPb2AMTi' + THEN event_meta[1]:parsed:info:amount :: INTEGER + ELSE event_meta[0]:parsed:info:amount :: INTEGER END AS amount_sent, + + CASE WHEN event_info[2]:programId :: STRING IN ('9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin', '675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8') -- SERUM DEX || Raydium Liquidity Pool V4 + THEN COALESCE( + event_meta[0]:parsed:info:destination :: STRING, + event_meta[1]:parsed:info:destination :: STRING + ) + WHEN event_info[3]:programId :: STRING IN ('9W959DqEETiGZocYWCQPaJ6sBmUzgfxXfqGeTEdp3aQP' , 'SWiMDJYFUGj6cPrQ6QYYYWZtvXQdRChSVAygDZDsCHC')-- ETH WORMHOLE + THEN postTokenBalances[1]:mint :: STRING + WHEN event_info[4]:programId :: STRING = 'MERLuDFBMmsHnsBPZw2sDQZHvXFMwp8EdjudcU2HKky' -- Mercurial Stable Swap + THEN postTokenBalances[3]:mint :: STRING + WHEN event_info[3]:programId :: STRING = 'MERLuDFBMmsHnsBPZw2sDQZHvXFMwp8EdjudcU2HKky' -- Mercurial Stable Swap + THEN postTokenBalances[2]:mint :: STRING + WHEN event_info[4]:programId :: STRING = '82yxjeMsvaURa4MbZZ7WZZHfobirZYkH1zF8fmeGtyaQ' -- ORCA AQUAFARM + THEN postTokenBalances[0]:mint :: STRING + WHEN event_info[2]:programId :: STRING IN ('9W959DqEETiGZocYWCQPaJ6sBmUzgfxXfqGeTEdp3aQP','SSwpMgqNDsyV7mAgN9ady4bDVu5ySjmmXejXvy2vLt1', 'DmzAmomATKpNp2rCBfYLS7CSwQqeQTsgRYJA1oSSAJaP', '31ARfyxZg6fr1J9hVs1NqBWkdeYeCKipuY71bMovNpy9', 'Bt2WPMmbwHPk36i4CRucNDyLcmoGdC7xEdrVuxgJaNE6', 'PSwapMdSai8tjrEXcxFeQth87xC4rRsa4VA5mhGhXkP', 'SWiMDJYFUGj6cPrQ6QYYYWZtvXQdRChSVAygDZDsCHC') -- ORCA TOKEN SWAP V2 || STEP FINANCE || UNKNOWN PROGRAM + THEN postTokenBalances[1]:mint :: STRING + WHEN event_info[3]:programId :: STRING IN ('DjVE6JNiYqPL2QXyCUUh8rNjHrbz9hXHNYt99MQ59qw1', '2rcQtPTFZd3UWzEWwa86TtYn3ePi4U2EDaZxf3BKSvfd') -- ORCA TOKEN SWAP V1 || UNKNOWN + THEN postTokenBalances[1]:mint :: STRING + WHEN event_info[2]:programId :: STRING = '2nAAsYdXF3eTQzaeUQS3fr4o782dDg8L28mX39Wr5j8N' -- UNKNOWN PROGRAM + THEN postTokenBalances[0]:mint :: STRING + WHEN event_info[2]:programId :: STRING = 'SSwpkEEcbUqx4vtoEByFjSkhKdCT862DNVb52nZg1UZ' + THEN COALESCE ( + event_info[1]:parsed:info:mint :: STRING, + postTokenBalances[3]:mint :: STRING + ) + WHEN event_info[2]:programId :: STRING = 'LendZqTs7gn5CTSJU1jWKhKuVpjJGom45nnwPb2AMTi' + THEN postTokenBalances[4]:mint :: STRING + ELSE event_info[1]:parsed:info:mint :: STRING END AS token_received, + + CASE WHEN event_info[2]:programId :: STRING IN ('SSwpMgqNDsyV7mAgN9ady4bDVu5ySjmmXejXvy2vLt1', 'PSwapMdSai8tjrEXcxFeQth87xC4rRsa4VA5mhGhXkP') + THEN event_meta[1]:parsed:info:amount :: INTEGER + WHEN event_info[4]:programId :: STRING = 'MERLuDFBMmsHnsBPZw2sDQZHvXFMwp8EdjudcU2HKky' -- Mercurial Stable Swap + THEN event_info[3]:parsed:info:amount :: INTEGER + WHEN event_info[3]:programId :: STRING = 'SWiMDJYFUGj6cPrQ6QYYYWZtvXQdRChSVAygDZDsCHC' + THEN event_meta[1]:parsed:info:amount :: INTEGER + WHEN event_meta[2]:parsed:type :: STRING <> 'burn' + THEN COALESCE( + event_meta[2]:parsed:info:amount :: INTEGER, + event_meta[1]:parsed:info:amount :: INTEGER + ) + ELSE event_meta[1]:parsed:info:amount :: INTEGER + END AS amount_received, + + succeeded, + ingested_at + +FROM "FLIPSIDE_DEV_DB"."SILVER_SOLANA"."EVENTS" + +WHERE event_info[1]:program = 'spl-token' +AND event_info[1]:programId = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' +AND event_meta[0]:parsed:info:amount :: INTEGER IS NOT NULL -- IF THESE ARE NULL, THEN NFT MINT +AND event_meta[1]:parsed:info:amount :: INTEGER IS NOT NULL +AND token_received IS NOT NULL + +{% if is_incremental() %} + AND ingested_at >= ( + SELECT + MAX( + ingested_at + ) + FROM + {{ this }} + ) +{% endif %} +), + +token_account AS ( + SELECT + tx:transaction:message:instructions[1]:program :: STRING as program, + tx:transaction:message:instructions[1]:parsed:info:wallet :: STRING as owner_address, + tx:transaction:message:instructions[1]:parsed:info:account :: STRING as associated_token_address, + tx:transaction:message:instructions[1]:parsed:info:mint :: STRING as token_address, + ingested_at + FROM {{ ref('bronze_solana__transactions') }} + WHERE program = 'spl-associated-token-account' + AND + 1 = 1 + {% if is_incremental() %} + AND ingested_at >= ( + SELECT + MAX( + ingested_at + ) + FROM + {{ this }} + ) +{% endif %} +), + +token_address_table AS ( + SELECT + block_timestamp, + block_id, + blockchain, + recent_blockhash, + tx_id, + sender, + token_sent_account, + t0_sent, + t.token_address AS token_sent2, + amount_sent, + token_received, + amount_received, + succeeded, + b.ingested_at + + FROM base_table b + + LEFT OUTER JOIN token_account t + ON token_sent_account = t.associated_token_address +) + +SELECT + block_timestamp, + block_id, + blockchain, + recent_blockhash, + tx_id, + sender, + + COALESCE ( + token_sent2, + t0_sent, + token_sent_account) AS token_sent, + + CASE WHEN token_sent_account IN ('48F1neXh5bGgKr8G6CM6tFZkaC51UgtVb5pqGLC27Doi', + '2Zkvs84qBpMvWFSh9MFek2ZPAqAEjfufrs7Sbu42PbzBRTSv5Qx1bTWTGsbaSsuAPe4yx6ku3KmwVaGRbdANytF6', + 'DD6oh3HRCvMzqHkGeUW3za4pLgWNPJdV6aNYW3gVjXXi', + 'DQyrAcCrDXQ7NeoqGgDCZwBvWDcYmFCjSb9JtteuvPpz', + '9wArqjNDHm43RJGSBSXuHygqJzEaW1xYA3XmmpU671kC', + '5sHXydo4z6p2x7Jqfx8T4FnPRM9S2y8wpgNG1pAZf9Cg', + 'ArwS8uBqFTKNgLAy7r3poSCujCazc4GiqBZguUzQPrKj', + '4aiKnDHFmNnLsopVsDyRBh8sbVohZYgdGzh3P9orpqNB', + '2YuVnXkSmW714ziXkfPfmvAnniBc1tHiAB7gYgRpx43a', + 'mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So', + 'StepAscQoEioFxxWGnh2sLBDFp9d8rvKz2Yp39iDpyT', + 'So11111111111111111111111111111111111111112', + 'DubwWZNWiNGMMeeQHPnMATNj77YZPZSAz2WVR5WjLJqz', + 'H7Qc9APCWWGDVxGD5fJHmLTmdEgT9GFatAKFNg6sHh8A', + '7dHbWXmci3dT8UFYWYZweBLXgycu7Y3iL6trKn1Y7ARj') + THEN amount_sent/POW(10,9) :: INTEGER + WHEN token_sent_account = '7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs' + THEN amount_sent/POW(10,8) :: INTEGER + WHEN token_sent IN ('48F1neXh5bGgKr8G6CM6tFZkaC51UgtVb5pqGLC27Doi', + '2Zkvs84qBpMvWFSh9MFek2ZPAqAEjfufrs7Sbu42PbzBRTSv5Qx1bTWTGsbaSsuAPe4yx6ku3KmwVaGRbdANytF6', + 'DD6oh3HRCvMzqHkGeUW3za4pLgWNPJdV6aNYW3gVjXXi', + 'DQyrAcCrDXQ7NeoqGgDCZwBvWDcYmFCjSb9JtteuvPpz', + '9wArqjNDHm43RJGSBSXuHygqJzEaW1xYA3XmmpU671kC', + '5sHXydo4z6p2x7Jqfx8T4FnPRM9S2y8wpgNG1pAZf9Cg', + 'ArwS8uBqFTKNgLAy7r3poSCujCazc4GiqBZguUzQPrKj', + '4aiKnDHFmNnLsopVsDyRBh8sbVohZYgdGzh3P9orpqNB', + '2YuVnXkSmW714ziXkfPfmvAnniBc1tHiAB7gYgRpx43a', + 'mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So', + 'StepAscQoEioFxxWGnh2sLBDFp9d8rvKz2Yp39iDpyT', + 'So11111111111111111111111111111111111111112', + 'DubwWZNWiNGMMeeQHPnMATNj77YZPZSAz2WVR5WjLJqz', + 'H7Qc9APCWWGDVxGD5fJHmLTmdEgT9GFatAKFNg6sHh8A', + '7dHbWXmci3dT8UFYWYZweBLXgycu7Y3iL6trKn1Y7ARj') + THEN amount_sent/POW(10,9) :: INTEGER + WHEN token_sent IN ('7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs', '9TE7ebz1dsFo1uQ2T4oYAKSm39Y6fWuHrd6Uk6XaiD16') + THEN amount_sent/POW(10,8) :: INTEGER + ELSE amount_sent/POW(10,6) :: INTEGER END AS amount_sent, + + token_received, + + CASE WHEN token_received IN ('48F1neXh5bGgKr8G6CM6tFZkaC51UgtVb5pqGLC27Doi', + '2Zkvs84qBpMvWFSh9MFek2ZPAqAEjfufrs7Sbu42PbzBRTSv5Qx1bTWTGsbaSsuAPe4yx6ku3KmwVaGRbdANytF6', + 'DD6oh3HRCvMzqHkGeUW3za4pLgWNPJdV6aNYW3gVjXXi', + 'DQyrAcCrDXQ7NeoqGgDCZwBvWDcYmFCjSb9JtteuvPpz', + '9wArqjNDHm43RJGSBSXuHygqJzEaW1xYA3XmmpU671kC', + '5sHXydo4z6p2x7Jqfx8T4FnPRM9S2y8wpgNG1pAZf9Cg', + 'ArwS8uBqFTKNgLAy7r3poSCujCazc4GiqBZguUzQPrKj', + '4aiKnDHFmNnLsopVsDyRBh8sbVohZYgdGzh3P9orpqNB', + '2YuVnXkSmW714ziXkfPfmvAnniBc1tHiAB7gYgRpx43a', + 'mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So', + 'StepAscQoEioFxxWGnh2sLBDFp9d8rvKz2Yp39iDpyT', + 'So11111111111111111111111111111111111111112', + 'AGFEad2et2ZJif9jaGpdMixQqvW5i81aBdvKe7PHNfz3', + 'DubwWZNWiNGMMeeQHPnMATNj77YZPZSAz2WVR5WjLJqz') + THEN amount_received/POW(10,9) :: INTEGER + WHEN token_received IN ('7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs', '9TE7ebz1dsFo1uQ2T4oYAKSm39Y6fWuHrd6Uk6XaiD16') + THEN amount_received/POW(10,8) :: INTEGER + ELSE amount_received/POW(10,6) :: INTEGER END AS amount_received, + + succeeded, + ingested_at + +FROM token_address_table + +qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +ORDER BY + ingested_at DESC)) = 1 diff --git a/models/solana/silver/silver_solana__swaps.yml b/models/solana/silver/silver_solana__swaps.yml new file mode 100644 index 00000000..1b57d2df --- /dev/null +++ b/models/solana/silver/silver_solana__swaps.yml @@ -0,0 +1,48 @@ +version: 2 +models: + - name: silver_solana__swaps + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - TX_ID + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - name: BLOCK_ID + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: RECENT_BLOCKHASH + tests: + - not_null + - name: TX_ID + tests: + - not_null + - name: SENDER + tests: + - not_null + - name: TOKEN_SENT + tests: + - not_null + - name: AMOUNT_SENT + tests: + - not_null + - name: TOKEN_RECEIVED + tests: + - not_null + - name: AMOUNT_RECEIVED + tests: + - not_null + - name: SUCCEEDED + tests: + - not_null + - name: INGESTED_AT + tests: + - not_null \ No newline at end of file diff --git a/models/solana/silver/silver_solana__transactions.sql b/models/solana/silver/silver_solana__transactions.sql index 38e7634d..2ec7cadc 100644 --- a/models/solana/silver/silver_solana__transactions.sql +++ b/models/solana/silver/silver_solana__transactions.sql @@ -2,6 +2,7 @@ materialized = 'incremental', unique_key = "CONCAT_WS('-', block_id, tx_id)", incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], tags = ['snowflake', 'solana', 'silver_solana', 'solana_transactions'] ) }} @@ -25,6 +26,7 @@ FROM {{ ref('bronze_solana__transactions') }} WHERE 1 = 1 AND program_id <> 'Vote111111111111111111111111111111111111111' +AND tx :meta:preTokenBalances[0]:owner :: STRING IS NOT NULL OR tx :meta:postTokenBalances[0]:owner :: STRING IS NOT NULL {% if is_incremental() %} AND ingested_at >= ( diff --git a/models/solana/silver/silver_solana__transactions.yml b/models/solana/silver/silver_solana__transactions.yml index ad7e929f..b2c0e7df 100644 --- a/models/solana/silver/silver_solana__transactions.yml +++ b/models/solana/silver/silver_solana__transactions.yml @@ -12,7 +12,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 1 + interval: 3 - name: BLOCK_ID tests: - not_null diff --git a/models/solana/silver/silver_solana__votes.sql b/models/solana/silver/silver_solana__votes.sql new file mode 100644 index 00000000..fc4ed059 --- /dev/null +++ b/models/solana/silver/silver_solana__votes.sql @@ -0,0 +1,60 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', block_id, tx_id)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], + tags = ['snowflake', 'solana', 'silver_solana', 'solana_votes'] +) }} + +WITH base_table as ( + SELECT + block_timestamp :: TIMESTAMP AS block_timestamp, + offset_id :: INTEGER AS block_id, + chain_id :: STRING AS blockchain, + tx :transaction:message:recentBlockhash :: STRING AS recent_blockhash, + tx_id :: STRING AS tx_id, + CASE WHEN tx :meta:status:Err IS NULL THEN TRUE ELSE FALSE END AS succeeded, + tx :transaction:message:instructions[0]:program :: STRING AS program_type, + tx :transaction:message:instructions[0]:programId :: STRING AS program_id, + tx :transaction:message:instructions[0]:parsed:info:voteAccount :: STRING AS vote_account, + tx :transaction:message:instructions[0]:parsed:info:voteAuthority :: STRING AS vote_authority, + ingested_at :: TIMESTAMP AS ingested_at, + CASE WHEN len(tx :meta:postTokenBalances[0]) > 0 AND len(tx :meta:preTokenBalances[0]) > 0 AND SUCCEEDED = TRUE THEN TRUE ELSE FALSE END AS transfer_tx_flag + + FROM {{ ref('bronze_solana__transactions') }} + + WHERE + 1 = 1 + AND tx :transaction:message:instructions[0]:parsed:type :: STRING IS NOT NULL + AND tx :transaction:message:instructions[0]:programId :: STRING = 'Vote111111111111111111111111111111111111111' + + {% if is_incremental() %} + AND ingested_at >= ( + SELECT + MAX( + ingested_at + ) + FROM + {{ this }} + ) + {% endif %} +) + +SELECT + block_timestamp, + block_id, + blockchain, + recent_blockhash, + tx_id, + succeeded, + program_type, + program_id, + vote_account, + vote_authority, + ingested_at, + transfer_tx_flag +FROM base_table + +qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +ORDER BY + ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/silver/silver_solana__votes.yml b/models/solana/silver/silver_solana__votes.yml new file mode 100644 index 00000000..4551ae59 --- /dev/null +++ b/models/solana/silver/silver_solana__votes.yml @@ -0,0 +1,42 @@ +version: 2 +models: + - name: silver_solana__votes + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - TX_ID + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - name: BLOCK_ID + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: RECENT_BLOCKHASH + tests: + - not_null + - name: TX_ID + tests: + - not_null + - name: SUCCEEDED + tests: + - not_null + - name: PROGRAM_TYPE + tests: + - not_null + - name: PROGRAM_ID + tests: + - not_null + - name: VOTE_ACCOUNT + tests: + - not_null + - name: INGESTED_AT + tests: + - not_null \ No newline at end of file From 3478a60c2a686692d85cacc122d23c50efd23b31 Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Thu, 13 Jan 2022 17:40:07 -0600 Subject: [PATCH 09/45] Removed tables that do not have to go out ASAP --- models/solana/gold/solana__events.sql | 25 -------------- models/solana/gold/solana__events.yml | 45 ------------------------- models/solana/gold/solana__swaps.sql | 24 -------------- models/solana/gold/solana__swaps.yml | 48 --------------------------- models/solana/gold/solana__votes.sql | 23 ------------- models/solana/gold/solana__votes.yml | 41 ----------------------- models/sources.yml | 5 --- 7 files changed, 211 deletions(-) delete mode 100644 models/solana/gold/solana__events.sql delete mode 100644 models/solana/gold/solana__events.yml delete mode 100644 models/solana/gold/solana__swaps.sql delete mode 100644 models/solana/gold/solana__swaps.yml delete mode 100644 models/solana/gold/solana__votes.sql delete mode 100644 models/solana/gold/solana__votes.yml diff --git a/models/solana/gold/solana__events.sql b/models/solana/gold/solana__events.sql deleted file mode 100644 index b0877fa7..00000000 --- a/models/solana/gold/solana__events.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['snowflake', 'solana', 'gold_solana', 'solana_events'] -) }} - -SELECT - block_timestamp, - block_id, - blockchain, - recent_blockhash, - tx_id, - tx_from_address, - tx_to_address, - succeeded, - event_meta, - postTokenBalances, - event_info, - ingested_at, - transfer_tx_flag - -FROM {{ ref('silver_solana__events') }} - -qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id -ORDER BY - ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/gold/solana__events.yml b/models/solana/gold/solana__events.yml deleted file mode 100644 index b17dbd7a..00000000 --- a/models/solana/gold/solana__events.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: 2 -models: - - name: solana__events - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - BLOCK_ID - - TX_ID - columns: - - name: BLOCK_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 3 - - name: BLOCK_ID - tests: - - not_null - - name: BLOCKCHAIN - tests: - - not_null - - name: RECENT_BLOCKHASH - tests: - - not_null - - name: TX_ID - tests: - - not_null - - name: TX_FROM_ADDRESS - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: TX_TO_ADDRESS - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: SUCCEEDED - tests: - - not_null - - name: EVENT_INFO - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: INGESTED_AT - tests: - - not_null \ No newline at end of file diff --git a/models/solana/gold/solana__swaps.sql b/models/solana/gold/solana__swaps.sql deleted file mode 100644 index ef6cf009..00000000 --- a/models/solana/gold/solana__swaps.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['snowflake', 'solana', 'gold_solana', 'solana_events', 'solana_swaps'] -) }} - -SELECT - block_timestamp, - block_id, - blockchain, - recent_blockhash, - tx_id, - sender, - token_sent, - amount_sent, - token_received, - amount_received, - succeeded, - ingested_at - -FROM {{ ref('silver_solana__swaps') }} - -qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id -ORDER BY - ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/gold/solana__swaps.yml b/models/solana/gold/solana__swaps.yml deleted file mode 100644 index 70241ac7..00000000 --- a/models/solana/gold/solana__swaps.yml +++ /dev/null @@ -1,48 +0,0 @@ -version: 2 -models: - - name: solana__swaps - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - BLOCK_ID - - TX_ID - columns: - - name: BLOCK_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 3 - - name: BLOCK_ID - tests: - - not_null - - name: BLOCKCHAIN - tests: - - not_null - - name: RECENT_BLOCKHASH - tests: - - not_null - - name: TX_ID - tests: - - not_null - - name: SENDER - tests: - - not_null - - name: TOKEN_SENT - tests: - - not_null - - name: AMOUNT_SENT - tests: - - not_null - - name: TOKEN_RECEIVED - tests: - - not_null - - name: AMOUNT_RECEIVED - tests: - - not_null - - name: SUCCEEDED - tests: - - not_null - - name: INGESTED_AT - tests: - - not_null \ No newline at end of file diff --git a/models/solana/gold/solana__votes.sql b/models/solana/gold/solana__votes.sql deleted file mode 100644 index adb8f397..00000000 --- a/models/solana/gold/solana__votes.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['snowflake', 'solana', 'gold_solana', 'solana_events', 'solana_votes'] -) }} - -SELECT - block_timestamp, - block_id, - blockchain, - recent_blockhash, - tx_id, - succeeded, - program_type, - program_id, - vote_account, - vote_authority, - ingested_at, - transfer_tx_flag -FROM {{ ref('silver_solana__votes') }} - -qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id -ORDER BY - ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/gold/solana__votes.yml b/models/solana/gold/solana__votes.yml deleted file mode 100644 index 9638dd68..00000000 --- a/models/solana/gold/solana__votes.yml +++ /dev/null @@ -1,41 +0,0 @@ -version: 2 -models: - - name: solana__votes - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - BLOCK_ID - - TX_ID - columns: - - name: BLOCK_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 3 - - name: BLOCK_ID - tests: - - not_null - - name: BLOCKCHAIN - tests: - - not_null - - name: RECENT_BLOCKHASH - tests: - - not_null - - name: TX_ID - tests: - - not_null - - name: SUCCEEDED - tests: - - not_null - - name: PROGRAM_TYPE - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: PROGRAM_ID - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: INGESTED_AT - tests: - - not_null \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 2ee8ece5..d9d8defe 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -136,11 +136,6 @@ sources: schema: silver_near tables: - name: daily_balances - - name: bronze_solana - schema: bronze_solana - tables: - - name: solana_blocks - - name: solana_txs - name: algorand schema: BRONZE_ALGORAND_2_6_4_PUBLIC tables: From f6f95aaae5513943884e5114835c2f7dfdbed61c Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Wed, 12 Jan 2022 16:04:28 -0800 Subject: [PATCH 10/45] add backfilled contracts to silver table --- .gitignore | 2 ++ .../ethereum/silver/silver_ethereum__contracts.sql | 12 ++++++++++++ .../ethereum/silver/silver_ethereum__contracts.yml | 11 +++++++++++ models/sources.yml | 1 + models/terra/silver/silver_terra__tax_rate.yml | 3 --- 5 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 models/ethereum/silver/silver_ethereum__contracts.yml diff --git a/.gitignore b/.gitignore index 3d9733bc..fec6df8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ target/ dbt_modules/ +# newer versions of dbt use this directory instead of dbt_modules for test dependencies +dbt_packages/ logs/ .venv/ diff --git a/models/ethereum/silver/silver_ethereum__contracts.sql b/models/ethereum/silver/silver_ethereum__contracts.sql index 1abad441..6e0d91de 100644 --- a/models/ethereum/silver/silver_ethereum__contracts.sql +++ b/models/ethereum/silver/silver_ethereum__contracts.sql @@ -29,7 +29,19 @@ SELECT name FROM {{ source('ethereum','ethereum_contracts') }} + +union + +select + '2000-01-01'::timestamp as system_created_at, + contract_address as address, + to_object(parse_json(contract_meta)) as meta, + name +from {{ source('ethereum','ethereum_contracts_backfill') }} +where check_json(contract_meta) is null ) +where address is not null qualify(ROW_NUMBER() over(PARTITION BY address ORDER BY system_created_at DESC)) = 1 + diff --git a/models/ethereum/silver/silver_ethereum__contracts.yml b/models/ethereum/silver/silver_ethereum__contracts.yml new file mode 100644 index 00000000..bf0fd0df --- /dev/null +++ b/models/ethereum/silver/silver_ethereum__contracts.yml @@ -0,0 +1,11 @@ +version: 2 +models: + - name: silver_ethereum__contracts + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ADDRESS + columns: + - name: ADDRESS + tests: + - not_null diff --git a/models/sources.yml b/models/sources.yml index d9d8defe..e99a2eee 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -26,6 +26,7 @@ sources: tables: - name: ethereum_balances - name: ethereum_contracts + - name: ethereum_contracts_backfill - name: daily_ethereum_token_balances - name: ethereum_address_labels - name: ethereum_address_labels_updates diff --git a/models/terra/silver/silver_terra__tax_rate.yml b/models/terra/silver/silver_terra__tax_rate.yml index e08aed53..2d2c042c 100644 --- a/models/terra/silver/silver_terra__tax_rate.yml +++ b/models/terra/silver/silver_terra__tax_rate.yml @@ -16,6 +16,3 @@ models: - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day interval: 1 - - name: CHAIN_ID - tests: - - not_null From ad9041a5b6918b9e776efb99f17fff3ce5231c93 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 12 Jan 2022 10:22:44 -0500 Subject: [PATCH 11/45] updating innertx test --- models/algorand/silver/silver_algorand__inner_txids.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/algorand/silver/silver_algorand__inner_txids.yml b/models/algorand/silver/silver_algorand__inner_txids.yml index d0846543..e879bed0 100644 --- a/models/algorand/silver/silver_algorand__inner_txids.yml +++ b/models/algorand/silver/silver_algorand__inner_txids.yml @@ -4,8 +4,8 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - BLOCK_ID - - INTRA + - inner_round + - inner_intra columns: - name: txn_round tests: From 6d38f79091006002467bc6cf7df48313589eb9fd Mon Sep 17 00:00:00 2001 From: JAMES MISSION Date: Thu, 13 Jan 2022 08:54:23 -0800 Subject: [PATCH 12/45] add coalesce for contract address and contract meta fields --- models/ethereum/ethereum_dbt__contracts.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ethereum/ethereum_dbt__contracts.sql b/models/ethereum/ethereum_dbt__contracts.sql index 4c1b03d5..f48af8d3 100644 --- a/models/ethereum/ethereum_dbt__contracts.sql +++ b/models/ethereum/ethereum_dbt__contracts.sql @@ -34,8 +34,8 @@ SELECT ( record_metadata :CreateTime :: INT / 1000 ) :: TIMESTAMP AS system_created_at, - t.value :address :: STRING AS address, - t.value :meta :: OBJECT AS meta, + coalesce(t.value :contract_address :: STRING, t.value :address :: STRING) AS address, + coalesce(t.value :contract_meta :: OBJECT, t.value :meta :: OBJECT) AS meta, t.value :name :: STRING AS name FROM base_tables, From 9ead2a186152cc99f07785ec06e79610ce9056df Mon Sep 17 00:00:00 2001 From: austinFlipside Date: Thu, 13 Jan 2022 18:11:19 -0500 Subject: [PATCH 13/45] decimal updates --- models/ethereum/ethereum__udm_events.sql | 33 ++++++++- .../silver/silver_ethereum__events.sql | 70 ++++++++++++++----- 2 files changed, 82 insertions(+), 21 deletions(-) diff --git a/models/ethereum/ethereum__udm_events.sql b/models/ethereum/ethereum__udm_events.sql index af6899f3..17bdbd71 100644 --- a/models/ethereum/ethereum__udm_events.sql +++ b/models/ethereum/ethereum__udm_events.sql @@ -22,6 +22,23 @@ WITH token_prices AS ( {% if is_incremental() %} AND HOUR >= getdate() - INTERVAL '2 days' {% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY HOUR, token_address +ORDER BY + symbol DESC) = 1) +), +full_decimals AS ( + SELECT + LOWER(address) AS contract_address, + meta :name :: STRING AS NAME, + meta :symbol :: STRING AS symbol, + meta :decimals :: INT AS decimals + FROM + {{ ref('silver_ethereum__contracts') }} + WHERE + meta :decimals NOT LIKE '%00%' qualify(ROW_NUMBER() over(PARTITION BY contract_address, NAME, symbol + ORDER BY + decimals DESC) = 1) --need the %00% filter to exclude messy data ), events AS ( SELECT @@ -169,10 +186,16 @@ token_transfers AS ( 'erc20_transfer' AS event_type, event_id, e.contract_address, - p.symbol AS symbol, + COALESCE( + fde.symbol, + p.symbol + ) AS symbol, token_value AS amount, CASE - WHEN p.decimals IS NOT NULL THEN amount * p.price + WHEN COALESCE( + fde.decimals, + p.decimals + ) IS NOT NULL THEN amount * p.price ELSE NULL END AS amount_usd FROM @@ -187,6 +210,12 @@ token_transfers AS ( 'hour', e.block_timestamp ) = p.hour + LEFT OUTER JOIN full_decimals fde + ON LOWER( + e.contract_address + ) = LOWER( + fde.contract_address + ) WHERE event_name = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' ), diff --git a/models/ethereum/silver/silver_ethereum__events.sql b/models/ethereum/silver/silver_ethereum__events.sql index fc6e64f3..2b9762af 100644 --- a/models/ethereum/silver/silver_ethereum__events.sql +++ b/models/ethereum/silver/silver_ethereum__events.sql @@ -6,24 +6,44 @@ tags = ['snowflake', 'ethereum', 'silver_ethereum','silver_ethereum__events'] ) }} +WITH decimals AS ( + + SELECT + LOWER(address) AS contract_address, + meta :name :: STRING AS NAME, + meta :symbol :: STRING AS symbol, + meta :decimals :: INT AS decimals + FROM + {{ ref('silver_ethereum__contracts') }} + WHERE + meta :decimals NOT LIKE '%00%' qualify(ROW_NUMBER() over(PARTITION BY contract_address, NAME, symbol --need the %00% filter to exclude messy data + ORDER BY + decimals DESC) = 1) +) SELECT system_created_at, - block_id, - block_timestamp, - tx_hash, - input_method, - "from", - "to", - name, - symbol, - contract_address, - eth_value, - fee, - log_index, - log_method, - token_value, - coalesce("from", '') as from_uk, - coalesce("to", '') as to_uk + block_id, + block_timestamp, + tx_hash, + input_method, + "from", + "to", + NAME, + symbol, + contract_address, + eth_value, + fee, + log_index, + log_method, + token_value, + COALESCE( + "from", + '' + ) AS from_uk, + COALESCE( + "to", + '' + ) AS to_uk FROM ( SELECT @@ -36,22 +56,34 @@ FROM "to", e.name AS NAME, e.symbol AS symbol, - contract_address, + e.contract_address, eth_value, fee, log_index, log_method, CASE WHEN LOWER(log_method) = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' - AND de.decimals IS NOT NULL THEN token_value / pow( - 10, + AND COALESCE( + fde.decimals, de.decimals + ) IS NOT NULL THEN token_value / pow( + 10, + COALESCE( + fde.decimals, + de.decimals + ) ) ELSE token_value END AS token_value FROM {{ ref('ethereum_dbt__events') }} e + LEFT OUTER JOIN decimals fde + ON LOWER( + fde.contract_address + ) = LOWER( + e.contract_address + ) LEFT OUTER JOIN {{ source( 'ethereum', 'ethereum_contract_decimal_adjustments' From dac66c296aea298abdb306eb17fbe6adf8e5c160 Mon Sep 17 00:00:00 2001 From: yulike Date: Thu, 13 Jan 2022 20:19:44 -0500 Subject: [PATCH 14/45] updated to add the 2 collateral tokens --- models/terra/anchor/anchor__liquidations.sql | 75 +++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/models/terra/anchor/anchor__liquidations.sql b/models/terra/anchor/anchor__liquidations.sql index 63d24e75..469a3258 100644 --- a/models/terra/anchor/anchor__liquidations.sql +++ b/models/terra/anchor/anchor__liquidations.sql @@ -67,7 +67,7 @@ AND block_timestamp :: DATE >= ( ) {% endif %} ), -events AS ( +events_first_token AS ( SELECT tx_id, COALESCE(event_attributes :liquidator :: STRING, event_attributes :"0_liquidator" :: STRING) AS liquidator, @@ -88,7 +88,7 @@ events AS ( 6 ) ELSE - event_attributes :"2_repay_amount" / pow( + event_attributes :"0_repay_amount" / pow( 10, 6 ) @@ -139,6 +139,75 @@ AND block_timestamp :: DATE >= ( ) {% endif %} ), +events_second_tokens AS ( + SELECT + tx_id, + COALESCE(event_attributes :liquidator :: STRING, event_attributes :"1_liquidator" :: STRING) AS liquidator, + COALESCE(event_attributes :collateral_amount / pow( + 10, + 6 + ), event_attributes :"1_collateral_amount" / pow( + 10, + 6 + )) AS liquidated_amount, + liquidated_amount * l.price AS liquidated_amount_usd, + COALESCE(event_attributes :collateral_token :: STRING, event_attributes :"1_collateral_token" :: STRING) AS liquidated_currency, + event_attributes :"1_repay_amount" / pow( + 10, + 6 + ) AS repay_amount, + event_attributes :"1_borrower" :: STRING AS borrower, + repay_amount * r.price AS repay_amount_usd, + COALESCE(event_attributes :stable_denom :: STRING, event_attributes :"1_stable_denom" :: STRING) AS repay_currency, + COALESCE(event_attributes :bid_fee / pow( + 10, + 6 + ), event_attributes :"1_bid_fee" / pow( + 10, + 6 + )) AS bid_fee + FROM + {{ ref('silver_terra__msg_events') }} + LEFT OUTER JOIN prices l + ON DATE_TRUNC( + 'hour', + block_timestamp + ) = l.hour + AND COALESCE(event_attributes :collateral_token :: STRING, event_attributes :"0_collateral_token" :: STRING) = l.currency + LEFT OUTER JOIN prices r + ON DATE_TRUNC( + 'hour', + block_timestamp + ) = r.hour + AND COALESCE(event_attributes :stable_denom :: STRING, event_attributes :"0_stable_denom" :: STRING) = r.currency + WHERE + event_type = 'from_contract' + AND tx_id IN( + SELECT + tx_id + FROM + msgs + ) + AND tx_status = 'SUCCEEDED' + AND event_attributes :"0_action" :: STRING = 'liquidate_collateral' + AND event_attributes :"2_repay_amount" IS NOT NULL + +{% if is_incremental() %} +AND block_timestamp :: DATE >= ( + SELECT + MAX( + block_timestamp :: DATE + ) + FROM + {{ ref('silver_terra__msgs') }} +) +{% endif %} +), +events AS ( + SELECT * FROM events_first_token + UNION ALL + SELECT * FROM events_second_tokens +), single_payment_tbl AS ( SELECT DISTINCT blockchain, @@ -242,3 +311,5 @@ multiple_repay_tbl AS ( SELECT * FROM single_payment_tbl UNION ALL SELECT * FROM multiple_repay_tbl + + From 8ed1b0f1c2713ef3516afb46c224ffc0441337be Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 17 Jan 2022 15:25:26 -0500 Subject: [PATCH 15/45] Added MINE to oracle prices table --- models/terra/gold/terra__oracle_prices.sql | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/models/terra/gold/terra__oracle_prices.sql b/models/terra/gold/terra__oracle_prices.sql index 53727bf2..ed487842 100644 --- a/models/terra/gold/terra__oracle_prices.sql +++ b/models/terra/gold/terra__oracle_prices.sql @@ -85,6 +85,26 @@ massets AS( AND m.block_timestamp >= getdate() - INTERVAL '1 days' {% endif %} +), + +polymine AS ( + +SELECT + 'terra' as blockchain, + DATE_TRUNC('minute', recorded_at) AS block_timestamp, + 'terra1kcthelkax4j9x8d3ny6sdag0qmxxynl3qtcrpy' AS currency, + symbol, + AVG(p.price) AS price, + 'coinmarketcap' as source +FROM {{ source('shared','prices_v2') }} p +WHERE asset_id IN('pylon-protocol', + '10767') +GROUP BY 1, + 2, + 3, + 4, + 6 + ) SELECT @@ -199,3 +219,18 @@ WHERE AND tx_id IN( SELECT tx_id FROM {{ ref('silver_terra__msgs') }} WHERE msg_value :contract :: STRING = 'terra1cgg6yef7qcdm070qftghfulaxmllgmvk77nc7t' AND msg_value :execute_msg :feed_price IS NOT NULL) + +UNION + +SELECT + p.blockchain, + p.block_timestamp, + p.currency, + p.symbol, + l.price/ p.price as luna_exchange_rate, + p.price AS price_usd, + p.source +FROM polymine p + +LEFT OUTER JOIN prices l + ON date_trunc('hour', p.block_timestamp) = l.block_timestamp From 61857fa8b1764c8fc5935de81b36f486a9b9bbc6 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 17 Jan 2022 16:22:09 -0500 Subject: [PATCH 16/45] incremental load --- models/terra/gold/terra__oracle_prices.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/models/terra/gold/terra__oracle_prices.sql b/models/terra/gold/terra__oracle_prices.sql index ed487842..25157559 100644 --- a/models/terra/gold/terra__oracle_prices.sql +++ b/models/terra/gold/terra__oracle_prices.sql @@ -99,6 +99,11 @@ SELECT FROM {{ source('shared','prices_v2') }} p WHERE asset_id IN('pylon-protocol', '10767') + +{% if is_incremental() %} +AND recorded_at >= getdate() - INTERVAL '1 days' +{% endif %} + GROUP BY 1, 2, 3, From 85f33c58a0ae517c445f2d78bfb8b577b96d5d18 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Thu, 6 Jan 2022 16:52:50 -0500 Subject: [PATCH 17/45] format --- models/terra/gold/terra__transfers.sql | 220 ++++++++++++------------- 1 file changed, 107 insertions(+), 113 deletions(-) diff --git a/models/terra/gold/terra__transfers.sql b/models/terra/gold/terra__transfers.sql index a7d97013..7965bf32 100644 --- a/models/terra/gold/terra__transfers.sql +++ b/models/terra/gold/terra__transfers.sql @@ -9,127 +9,120 @@ WITH prices AS ( - SELECT - DATE_TRUNC( - 'hour', - block_timestamp - ) AS HOUR, +SELECT + DATE_TRUNC('hour', block_timestamp) AS HOUR, currency, symbol, AVG(price_usd) AS price_usd - FROM - {{ ref('terra__oracle_prices') }} + FROM {{ ref('terra__oracle_prices') }} GROUP BY 1, 2, 3 + ), + symbol AS ( - SELECT - currency, - symbol - FROM - {{ ref('terra__oracle_prices') }} - WHERE - block_timestamp >= CURRENT_DATE - 2 - GROUP BY - 1, - 2 + +SELECT + currency, + symbol +FROM {{ ref('terra__oracle_prices') }} +WHERE block_timestamp >= CURRENT_DATE - 2 +GROUP BY 1, + 2 + ), + inputs AS( - SELECT - blockchain, - chain_id, - tx_status, - block_id, - block_timestamp, - tx_id, - msg_type, - A.value :address :: STRING AS event_from, - A.value :coins [0] :amount / pow( - 10, - 6 - ) AS event_amount, - A.value :coins [0] :denom :: STRING AS event_currency, - A.index AS input_index - FROM - {{ ref('silver_terra__msgs') }}, - LATERAL FLATTEN( - input => msg_value :inputs - ) A - WHERE - msg_module = 'bank' - AND msg_type = 'bank/MsgMultiSend' - AND tx_status = 'SUCCEEDED' + +SELECT + blockchain, + chain_id, + tx_status, + block_id, + block_timestamp, + tx_id, + msg_type, + A.value :address :: STRING AS event_from, + A.value :coins [0] :amount / pow(10,6) AS event_amount, + A.value :coins [0] :denom :: STRING AS event_currency, + A.index AS input_index +FROM {{ ref('silver_terra__msgs') }}, +LATERAL FLATTEN(input => msg_value :inputs) A +WHERE msg_module = 'bank' + AND msg_type = 'bank/MsgMultiSend' + AND tx_status = 'SUCCEEDED' {% if is_incremental() %} AND block_timestamp >= getdate() - INTERVAL '1 days' {% endif %} + ), + outputs AS( - SELECT - tx_id, - A.value :address :: STRING AS event_to, - A.index AS output_index - FROM - {{ ref('silver_terra__msgs') }}, - LATERAL FLATTEN( - input => msg_value :outputs - ) A - WHERE - msg_module = 'bank' - AND msg_type = 'bank/MsgMultiSend' - AND tx_status = 'SUCCEEDED' + +SELECT + tx_id, + A.value :address :: STRING AS event_to, + A.index AS output_index +FROM {{ ref('silver_terra__msgs') }}, +LATERAL FLATTEN(input => msg_value :outputs) A +WHERE msg_module = 'bank' + AND msg_type = 'bank/MsgMultiSend' + AND tx_status = 'SUCCEEDED' {% if is_incremental() %} AND block_timestamp >= getdate() - INTERVAL '1 days' {% endif %} + ), + transfers AS( - SELECT - blockchain, - chain_id, - tx_status, - block_id, - block_timestamp, - i.tx_id, - msg_type, - event_from, - event_to, - event_amount, - event_currency - FROM - inputs i - JOIN outputs o - ON i.tx_id = o.tx_id - AND i.input_index = o.output_index - UNION - SELECT - blockchain, - chain_id, - tx_status, - block_id, - block_timestamp, - tx_id, - msg_type, - msg_value :from_address :: STRING AS event_from, - msg_value :to_address :: STRING AS event_to, - msg_value :amount [0] :amount / pow( - 10, - 6 - ) AS event_amount, - msg_value :amount [0] :denom :: STRING AS event_currency - FROM - {{ ref('silver_terra__msgs') }} - WHERE - msg_module = 'bank' - AND msg_type = 'bank/MsgSend' - AND tx_status = 'SUCCEEDED' + +SELECT + blockchain, + chain_id, + tx_status, + block_id, + block_timestamp, + i.tx_id, + msg_type, + event_from, + event_to, + event_amount, + event_currency +FROM inputs i + +JOIN outputs o + ON i.tx_id = o.tx_id + AND i.input_index = o.output_index + +UNION + +SELECT + blockchain, + chain_id, + tx_status, + block_id, + block_timestamp, + tx_id, + msg_type, + msg_value :from_address :: STRING AS event_from, + msg_value :to_address :: STRING AS event_to, + msg_value :amount [0] :amount / pow(10,6) AS event_amount, + msg_value :amount [0] :denom :: STRING AS event_currency +FROM {{ ref('silver_terra__msgs') }} +WHERE msg_module = 'bank' + AND msg_type = 'bank/MsgSend' + AND tx_status = 'SUCCEEDED' {% if is_incremental() %} AND block_timestamp >= getdate() - INTERVAL '1 days' {% endif %} + UNION + SELECT blockchain, chain_id, @@ -140,21 +133,18 @@ SELECT msg_type, msg_value :sender :: STRING AS event_from, msg_value :execute_msg :transfer :recipient :: STRING AS event_to, - msg_value :execute_msg :transfer :amount / pow( - 10, - 6 - ) AS event_amount, + msg_value :execute_msg :transfer :amount / pow(10,6) AS event_amount, msg_value :contract :: STRING AS event_currency -FROM - {{ ref('silver_terra__msgs') }} -WHERE - msg_value :execute_msg :transfer IS NOT NULL +FROM {{ ref('silver_terra__msgs') }} +WHERE msg_value :execute_msg :transfer IS NOT NULL AND tx_status = 'SUCCEEDED' {% if is_incremental() %} AND block_timestamp >= getdate() - INTERVAL '1 days' {% endif %} + ) + SELECT t.blockchain, t.chain_id, @@ -176,17 +166,21 @@ SELECT t.event_amount, t.event_amount * price_usd AS event_amount_usd, coalesce(s.symbol, t.event_currency) AS event_currency -FROM - transfers t - LEFT OUTER JOIN prices o - ON DATE_TRUNC( - 'hour', - t.block_timestamp - ) = o.hour +FROM transfers t + +LEFT OUTER JOIN prices o + ON DATE_TRUNC('hour', t.block_timestamp) = o.hour AND t.event_currency = o.currency - LEFT OUTER JOIN symbol s + +LEFT OUTER JOIN symbol s ON t.event_currency = s.currency - LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS from_labels - ON event_from = from_labels.address AND from_labels.blockchain = 'terra' AND from_labels.creator = 'flipside' - LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS to_labels - ON event_to = to_labels.address AND to_labels.blockchain = 'terra' AND to_labels.creator = 'flipside' + +LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS from_labels + ON event_from = from_labels.address + AND from_labels.blockchain = 'terra' + AND from_labels.creator = 'flipside' + +LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS to_labels + ON event_to = to_labels.address + AND to_labels.blockchain = 'terra' + AND to_labels.creator = 'flipside' From 2e2792796559ee76c5318112c54a04092ddeb3f4 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 17 Jan 2022 12:42:13 -0500 Subject: [PATCH 18/45] prod ready --- models/terra/gold/terra__transfers.sql | 100 +++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/models/terra/gold/terra__transfers.sql b/models/terra/gold/terra__transfers.sql index 7965bf32..11354b4e 100644 --- a/models/terra/gold/terra__transfers.sql +++ b/models/terra/gold/terra__transfers.sql @@ -78,6 +78,74 @@ AND block_timestamp >= getdate() - INTERVAL '1 days' ), +wormhole_native AS( + +SELECT + r.blockchain, + r.chain_id, + r.tx_status, + r.block_id, + r.block_timestamp, + r.tx_id, + r.msg_type, + COALESCE(REGEXP_SUBSTR(key,'^[0-9]+')::STRING,'0') AS index, + MAX(CASE WHEN key REGEXP '.*sender' THEN value::STRING ELSE NULL::STRING END) AS event_from, + MAX(CASE WHEN key REGEXP '.*recipient' THEN value::STRING ELSE NULL::STRING END) AS event_to, + MAX(CASE WHEN key REGEXP '.*amount' THEN value[0]:denom::STRING ELSE NULL::STRING END) AS event_currency, + MAX(CASE WHEN key REGEXP '.*amount' THEN value[0]:amount::NUMERIC ELSE NULL::NUMERIC END/POWER(10,6)) AS event_amount +FROM {{ ref('silver_terra__msg_events') }} r, + LATERAL FLATTEN(input => r.event_attributes) a +WHERE tx_id IN(SELECT tx_id + FROM {{ ref('silver_terra__msg_events') }} + WHERE event_type = 'from_contract' + AND event_attributes:action::string = 'complete_transfer_terra_native' + AND event_attributes:contract_address::string = 'terra10nmmwe8r3g99a9newtqa7a75xfgs2e8z87r2sf' + AND tx_status = 'SUCCEEDED') +AND r.event_type = 'transfer' +AND tx_status = 'SUCCEEDED' + +{% if is_incremental() %} +AND r.block_timestamp >= getdate() - INTERVAL '1 days' +{% endif %} + +GROUP BY 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8 + +), + +wormhole_wrapped AS( + +SELECT + blockchain, + chain_id, + tx_status, + block_id, + block_timestamp, + tx_id, + msg_type, + msg_index, + event_attributes:"0_contract_address"::STRING as event_from, + event_attributes:recipient::STRING as event_to, + event_attributes:contract::STRING as event_currency, + event_attributes:"0_amount"::NUMERIC / POW(10,6) as event_amount +FROM {{ ref('silver_terra__msg_events') }} +WHERE event_type = 'from_contract' +AND event_attributes:"0_action"::string = 'complete_transfer_wrapped' +AND event_attributes:"0_contract_address"::string = 'terra10nmmwe8r3g99a9newtqa7a75xfgs2e8z87r2sf' +AND tx_status = 'SUCCEEDED' + +{% if is_incremental() %} +AND block_timestamp >= getdate() - INTERVAL '1 days' +{% endif %} + +), + transfers AS( SELECT @@ -143,6 +211,38 @@ WHERE msg_value :execute_msg :transfer IS NOT NULL AND block_timestamp >= getdate() - INTERVAL '1 days' {% endif %} +UNION + +SELECT + blockchain, + chain_id, + tx_status, + block_id, + block_timestamp, + tx_id, + msg_type, + event_from, + event_to, + event_amount, + event_currency +FROM wormhole_native + +UNION + +SELECT + blockchain, + chain_id, + tx_status, + block_id, + block_timestamp, + tx_id, + msg_type, + event_from, + event_to, + event_amount, + event_currency +FROM wormhole_wrapped + ) SELECT From c7ce0e4fd1ec161c06ba192932f66a82bf3c3754 Mon Sep 17 00:00:00 2001 From: austinFlipside Date: Tue, 18 Jan 2022 19:31:14 -0500 Subject: [PATCH 19/45] exclude nulls --- .../silver/silver_ethereum__contracts.sql | 70 ++++++++++--------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/models/ethereum/silver/silver_ethereum__contracts.sql b/models/ethereum/silver/silver_ethereum__contracts.sql index 6e0d91de..76b93f84 100644 --- a/models/ethereum/silver/silver_ethereum__contracts.sql +++ b/models/ethereum/silver/silver_ethereum__contracts.sql @@ -6,42 +6,48 @@ tags = ['snowflake', 'ethereum', 'silver_ethereum','silver_ethereum__contracts'] ) }} -select system_created_at, - address, - meta, - name -from -( SELECT system_created_at, address, meta, - name + NAME FROM - {{ ref('ethereum_dbt__contracts') }} - -union - -SELECT - '2000-01-01'::timestamp as system_created_at, - address, - meta, - name -FROM - {{ source('ethereum','ethereum_contracts') }} - -union - -select - '2000-01-01'::timestamp as system_created_at, - contract_address as address, - to_object(parse_json(contract_meta)) as meta, - name -from {{ source('ethereum','ethereum_contracts_backfill') }} -where check_json(contract_meta) is null -) -where address is not null -qualify(ROW_NUMBER() over(PARTITION BY address + ( + SELECT + system_created_at, + address, + meta, + NAME + FROM + {{ ref('ethereum_dbt__contracts') }} + WHERE + meta IS NOT NULL + UNION + SELECT + '2000-01-01' :: TIMESTAMP AS system_created_at, + address, + meta, + NAME + FROM + {{ source( + 'ethereum', + 'ethereum_contracts' + ) }} + UNION + SELECT + '2000-01-01' :: TIMESTAMP AS system_created_at, + contract_address AS address, + TO_OBJECT(PARSE_JSON(contract_meta)) AS meta, + NAME + FROM + {{ source( + 'ethereum', + 'ethereum_contracts_backfill' + ) }} + WHERE + CHECK_JSON(contract_meta) IS NULL + ) +WHERE + address IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY address ORDER BY system_created_at DESC)) = 1 - From 4d84cfc820e6824ea139d4be13ec46d57b2b9f83 Mon Sep 17 00:00:00 2001 From: Jessica Huhnke <91915469+jhuhnke@users.noreply.github.com> Date: Wed, 19 Jan 2022 13:22:16 -0600 Subject: [PATCH 20/45] An 598 sol missing txs (#256) --- models/solana/silver/silver_solana__swaps.sql | 1 + .../silver/silver_solana__transactions.sql | 27 +++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/models/solana/silver/silver_solana__swaps.sql b/models/solana/silver/silver_solana__swaps.sql index d5973228..d4ba567f 100644 --- a/models/solana/silver/silver_solana__swaps.sql +++ b/models/solana/silver/silver_solana__swaps.sql @@ -23,6 +23,7 @@ WITH base_table AS ( event_meta[0]:parsed:info:destination :: STRING, event_meta[1]:parsed:info:destination :: STRING ) END AS token_sent_account, + CASE WHEN event_info[3]:programId :: STRING IN ('SSwpkEEcbUqx4vtoEByFjSkhKdCT862DNVb52nZg1UZ', 'SSwpMgqNDsyV7mAgN9ady4bDVu5ySjmmXejXvy2vLt1') -- SABER STABLE SWAP || STEP FINANCE THEN postTokenBalances[1]:mint :: STRING WHEN event_info[4]:programId :: STRING = '82yxjeMsvaURa4MbZZ7WZZHfobirZYkH1zF8fmeGtyaQ' -- ORCA AQUAFARM diff --git a/models/solana/silver/silver_solana__transactions.sql b/models/solana/silver/silver_solana__transactions.sql index 2ec7cadc..b3821817 100644 --- a/models/solana/silver/silver_solana__transactions.sql +++ b/models/solana/silver/silver_solana__transactions.sql @@ -14,19 +14,30 @@ WITH base_table as ( tx_id :: STRING AS tx_id, tx :meta:preTokenBalances[0]:mint :: STRING as pre_mint, tx :meta:postTokenBalances[0]:mint :: STRING as post_mint, - tx :meta:preTokenBalances[0]:owner :: STRING AS tx_from_address, - tx :meta:postTokenBalances[0]:owner :: STRING AS tx_to_address, - tx :meta:fee :: INTEGER AS fee, -- This is in lamports right now - CASE WHEN tx :meta:status:Err IS NULL THEN TRUE ELSE FALSE END AS succeeded, - --tx :meta:status:Err :: ARRAY AS error, -- Need some sort of coalesce statement here + COALESCE( + tx :meta:preTokenBalances[0]:owner :: STRING, + tx: transaction:message:instructions[0]:parsed:info:source :: STRING + ) AS tx_from_address, + COALESCE ( + tx :meta:postTokenBalances[0]:owner :: STRING, + tx: transaction:message:instructions[0]:parsed:info:destination :: STRING + ) AS tx_to_address, + tx :meta:fee :: INTEGER AS fee, + CASE WHEN tx :meta:status:Err IS NULL THEN TRUE ELSE FALSE END AS succeeded, tx :transaction:message:instructions[0]:programId :: STRING AS program_id, ingested_at :: TIMESTAMP AS ingested_at, CASE WHEN len(tx :meta:postTokenBalances[0]) > 0 AND len(tx :meta:preTokenBalances[0]) > 0 THEN TRUE ELSE FALSE END AS transfer_tx_flag FROM {{ ref('bronze_solana__transactions') }} WHERE - 1 = 1 -AND program_id <> 'Vote111111111111111111111111111111111111111' -AND tx :meta:preTokenBalances[0]:owner :: STRING IS NOT NULL OR tx :meta:postTokenBalances[0]:owner :: STRING IS NOT NULL + program_id <> 'Vote111111111111111111111111111111111111111' +AND COALESCE( + tx :meta:preTokenBalances[0]:owner :: STRING, + tx: transaction:message:instructions[0]:parsed:info:source :: STRING + ) IS NOT NULL +AND COALESCE ( + tx :meta:postTokenBalances[0]:owner :: STRING, + tx: transaction:message:instructions[0]:parsed:info:destination :: STRING + ) IS NOT NULL {% if is_incremental() %} AND ingested_at >= ( From 5e9e278f60f5705186855bd72565e15612623db9 Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Wed, 19 Jan 2022 14:47:02 -0600 Subject: [PATCH 21/45] Emergency typo fix - was pulling in wrong tx_to_address --- models/solana/silver/silver_solana__transactions.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/models/solana/silver/silver_solana__transactions.sql b/models/solana/silver/silver_solana__transactions.sql index b3821817..ab29b84b 100644 --- a/models/solana/silver/silver_solana__transactions.sql +++ b/models/solana/silver/silver_solana__transactions.sql @@ -19,7 +19,7 @@ WITH base_table as ( tx: transaction:message:instructions[0]:parsed:info:source :: STRING ) AS tx_from_address, COALESCE ( - tx :meta:postTokenBalances[0]:owner :: STRING, + tx :meta:postTokenBalances[1]:owner :: STRING, tx: transaction:message:instructions[0]:parsed:info:destination :: STRING ) AS tx_to_address, tx :meta:fee :: INTEGER AS fee, @@ -35,7 +35,7 @@ AND COALESCE( tx: transaction:message:instructions[0]:parsed:info:source :: STRING ) IS NOT NULL AND COALESCE ( - tx :meta:postTokenBalances[0]:owner :: STRING, + tx :meta:postTokenBalances[1]:owner :: STRING, tx: transaction:message:instructions[0]:parsed:info:destination :: STRING ) IS NOT NULL @@ -50,7 +50,6 @@ AND ingested_at >= ( ) {% endif %} ) - SELECT block_timestamp, From 0d3fae7c6414b61fe7ca419a4f4c8bc6eb557b43 Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Wed, 19 Jan 2022 16:29:34 -0600 Subject: [PATCH 22/45] Solana gold view optimizations --- models/solana/gold/solana__transactions.sql | 7 +------ models/solana/gold/solana__transfers.sql | 4 ---- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/models/solana/gold/solana__transactions.sql b/models/solana/gold/solana__transactions.sql index 524c0111..3aa153b3 100644 --- a/models/solana/gold/solana__transactions.sql +++ b/models/solana/gold/solana__transactions.sql @@ -4,7 +4,6 @@ ) }} SELECT - DISTINCT block_timestamp, block_id, recent_blockhash, @@ -19,8 +18,4 @@ SELECT ingested_at, transfer_tx_flag FROM - {{ ref('silver_solana__transactions') }} - -qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id -ORDER BY - ingested_at DESC)) = 1 \ No newline at end of file + {{ ref('silver_solana__transactions') }} \ No newline at end of file diff --git a/models/solana/gold/solana__transfers.sql b/models/solana/gold/solana__transfers.sql index 60b146da..c18a3410 100644 --- a/models/solana/gold/solana__transfers.sql +++ b/models/solana/gold/solana__transfers.sql @@ -23,7 +23,3 @@ LATERAL FLATTEN ( WHERE ei.value:parsed:type :: STRING = 'transfer' AND ei.value:program :: STRING = 'spl-token' - -qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id -ORDER BY - ingested_at DESC)) = 1 \ No newline at end of file From e870dc2a66eccf8841ac6857eeda885741844a0f Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Tue, 18 Jan 2022 15:33:45 -0800 Subject: [PATCH 23/45] initial working version --- .../ethereum__token_prices_hourly.sql | 118 ++++++++++++++---- 1 file changed, 94 insertions(+), 24 deletions(-) diff --git a/models/ethereum/ethereum__token_prices_hourly.sql b/models/ethereum/ethereum__token_prices_hourly.sql index 30c17fb0..386922eb 100644 --- a/models/ethereum/ethereum__token_prices_hourly.sql +++ b/models/ethereum/ethereum__token_prices_hourly.sql @@ -8,29 +8,99 @@ ) }} +-- SELECT +-- p.symbol, +-- date_trunc('hour', recorded_at) as hour, +-- lower(a.token_address) as token_address, +-- d.decimals, +-- avg(price) as price +-- FROM +-- {{ source('shared', 'prices_v2')}} p +-- JOIN +-- {{ source('shared', 'market_asset_metadata')}} a +-- ON +-- p.asset_id = a.asset_id +-- LEFT OUTER JOIN +-- {{ source('ethereum', 'ethereum_contract_decimal_adjustments')}} d +-- ON +-- d.address = lower(a.token_address) +-- WHERE +-- (a.platform_id = '1027' OR a.asset_id = '1027' or a.platform_id = 'ethereum') +-- {% if is_incremental() %} +-- AND recorded_at >= getdate() - interval '45 days' +-- {% else %} +-- AND recorded_at >= '2020-05-05'::timestamp +-- {% endif %} + +-- GROUP BY 1,2,3,4 + +with hours as ( + select dateadd(hour, -seq4(), date_trunc('hour',current_timestamp())) as day_hour + from + -- get all hours since some date + {% if is_incremental() %} + table(generator(rowcount=>50*24)) + {% else %} + table(generator(rowcount=>(select max(days_since) * 24 from {{ ref('ethereum_dbt__days_since_for_prices_hourly') }}))) + {% endif %} +), +hourly_prices as ( + select + p.symbol, + date_trunc('hour', recorded_at) as hour, + lower(a.token_address) as token_address, + d.decimals, + avg(price) as price + from {{ source('shared', 'prices_v2') }} p + left outer join {{ source('shared', 'market_asset_metadata') }} a on p.asset_id = a.asset_id + left outer join {{ source('ethereum', 'ethereum_contract_decimal_adjustments') }} d on d.address = lower(a.token_address) + where (a.platform_id = '1027' or a.asset_id = '1027' or a.platform_id = 'ethereum') + {% if is_incremental() %} + and recorded_at >= current_date - 45 + {% else %} + and recorded_at >= '2018-02-21' -- first date with valid prices data for a symbol + {% endif %} + group by 1,2,3,4 +) +, symbols as ( + select distinct symbol, token_address + from {{ this }} +) +, hour_symbols_pair as ( + select * + from hours + cross join symbols + {% if is_incremental() %} + where day_hour >= current_date - 45 + {% else %} + where day_hour >= '2018-02-21' -- first date with valid prices data for a symbol + {% endif %} +) +, imputed as ( + select + h.day_hour + , h.symbol + , p.price as avg_price + , p.token_address + , p.decimals + , last_value(p.token_address) ignore nulls over (partition by h.symbol, h.token_address order by day_hour) as lag_token_address + , last_value(p.decimals) ignore nulls over (partition by h.symbol, h.token_address order by day_hour) as lag_decimals + , lag(p.price) ignore nulls over (partition by h.symbol, h.token_address order by day_hour) as imputed_price + from hour_symbols_pair h + left outer join hourly_prices p on p.hour = h.day_hour and p.symbol = h.symbol and p.token_address = h.token_address +) +select + p.day_hour as hour + , p.symbol + , case when token_address is not null then token_address + else lag_token_address end as token_address + , case when decimals is not null then decimals + else lag_decimals end as decimals + , case when avg_price is not null then avg_price + else imputed_price end as price + , case when avg_price is null then TRUE + else FALSE end as is_imputed +from imputed p +where price is not null -SELECT - p.symbol, - date_trunc('hour', recorded_at) as hour, - lower(a.token_address) as token_address, - d.decimals, - avg(price) as price -FROM - {{ source('shared', 'prices_v2')}} p -JOIN - {{ source('shared', 'market_asset_metadata')}} a -ON - p.asset_id = a.asset_id -LEFT OUTER JOIN - {{ source('ethereum', 'ethereum_contract_decimal_adjustments')}} d -ON - d.address = lower(a.token_address) -WHERE - (a.platform_id = '1027' OR a.asset_id = '1027' or a.platform_id = 'ethereum') - {% if is_incremental() %} - AND recorded_at >= getdate() - interval '45 days' - {% else %} - AND recorded_at >= '2020-05-05'::timestamp - {% endif %} -GROUP BY 1,2,3,4 From 9d9bc3e4c3319d9c12286f6d0746403218a1cbb0 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Tue, 18 Jan 2022 15:46:57 -0800 Subject: [PATCH 24/45] use the hours table instead of generating dynamically --- .../ethereum__token_prices_hourly.sql | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/models/ethereum/ethereum__token_prices_hourly.sql b/models/ethereum/ethereum__token_prices_hourly.sql index 386922eb..267a40e2 100644 --- a/models/ethereum/ethereum__token_prices_hourly.sql +++ b/models/ethereum/ethereum__token_prices_hourly.sql @@ -34,17 +34,7 @@ -- GROUP BY 1,2,3,4 -with hours as ( - select dateadd(hour, -seq4(), date_trunc('hour',current_timestamp())) as day_hour - from - -- get all hours since some date - {% if is_incremental() %} - table(generator(rowcount=>50*24)) - {% else %} - table(generator(rowcount=>(select max(days_since) * 24 from {{ ref('ethereum_dbt__days_since_for_prices_hourly') }}))) - {% endif %} -), -hourly_prices as ( +with hourly_prices as ( select p.symbol, date_trunc('hour', recorded_at) as hour, @@ -68,29 +58,29 @@ hourly_prices as ( ) , hour_symbols_pair as ( select * - from hours + from silver.hours cross join symbols {% if is_incremental() %} - where day_hour >= current_date - 45 + where hour >= current_date - 45 {% else %} - where day_hour >= '2018-02-21' -- first date with valid prices data for a symbol + where hour >= '2018-02-21' -- first date with valid prices data for a symbol {% endif %} ) , imputed as ( select - h.day_hour + h.hour , h.symbol , p.price as avg_price , p.token_address , p.decimals - , last_value(p.token_address) ignore nulls over (partition by h.symbol, h.token_address order by day_hour) as lag_token_address - , last_value(p.decimals) ignore nulls over (partition by h.symbol, h.token_address order by day_hour) as lag_decimals - , lag(p.price) ignore nulls over (partition by h.symbol, h.token_address order by day_hour) as imputed_price + , last_value(p.token_address) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as lag_token_address + , last_value(p.decimals) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as lag_decimals + , lag(p.price) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as imputed_price from hour_symbols_pair h - left outer join hourly_prices p on p.hour = h.day_hour and p.symbol = h.symbol and p.token_address = h.token_address + left outer join hourly_prices p on p.hour = h.hour and p.symbol = h.symbol and p.token_address = h.token_address ) select - p.day_hour as hour + p.hour as hour , p.symbol , case when token_address is not null then token_address else lag_token_address end as token_address From 0918e87c9109f5b00471d7e6d7ec6fc4687be71c Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Tue, 18 Jan 2022 16:05:53 -0800 Subject: [PATCH 25/45] adjust hardcoded start date for prices data --- .../ethereum__token_prices_hourly.sql | 41 ++++--------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/models/ethereum/ethereum__token_prices_hourly.sql b/models/ethereum/ethereum__token_prices_hourly.sql index 267a40e2..eae11f12 100644 --- a/models/ethereum/ethereum__token_prices_hourly.sql +++ b/models/ethereum/ethereum__token_prices_hourly.sql @@ -8,32 +8,6 @@ ) }} --- SELECT --- p.symbol, --- date_trunc('hour', recorded_at) as hour, --- lower(a.token_address) as token_address, --- d.decimals, --- avg(price) as price --- FROM --- {{ source('shared', 'prices_v2')}} p --- JOIN --- {{ source('shared', 'market_asset_metadata')}} a --- ON --- p.asset_id = a.asset_id --- LEFT OUTER JOIN --- {{ source('ethereum', 'ethereum_contract_decimal_adjustments')}} d --- ON --- d.address = lower(a.token_address) --- WHERE --- (a.platform_id = '1027' OR a.asset_id = '1027' or a.platform_id = 'ethereum') --- {% if is_incremental() %} --- AND recorded_at >= getdate() - interval '45 days' --- {% else %} --- AND recorded_at >= '2020-05-05'::timestamp --- {% endif %} - --- GROUP BY 1,2,3,4 - with hourly_prices as ( select p.symbol, @@ -48,7 +22,7 @@ with hourly_prices as ( {% if is_incremental() %} and recorded_at >= current_date - 45 {% else %} - and recorded_at >= '2018-02-21' -- first date with valid prices data for a symbol + and recorded_at >= '2020-05-05' -- first date with valid prices data {% endif %} group by 1,2,3,4 ) @@ -63,18 +37,18 @@ with hourly_prices as ( {% if is_incremental() %} where hour >= current_date - 45 {% else %} - where hour >= '2018-02-21' -- first date with valid prices data for a symbol + where hour >= '2020-05-05' -- first date with valid prices data {% endif %} ) , imputed as ( select h.hour , h.symbol - , p.price as avg_price - , p.token_address + , h.token_address , p.decimals - , last_value(p.token_address) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as lag_token_address - , last_value(p.decimals) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as lag_decimals + , p.price as avg_price + -- , last_value(p.token_address) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as lag_token_address + , lag(p.decimals) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as lag_decimals , lag(p.price) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as imputed_price from hour_symbols_pair h left outer join hourly_prices p on p.hour = h.hour and p.symbol = h.symbol and p.token_address = h.token_address @@ -82,8 +56,7 @@ with hourly_prices as ( select p.hour as hour , p.symbol - , case when token_address is not null then token_address - else lag_token_address end as token_address + , p.token_address , case when decimals is not null then decimals else lag_decimals end as decimals , case when avg_price is not null then avg_price From a44a0aac2c241b9b310848e006af8d75bccd1c60 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Tue, 18 Jan 2022 16:59:53 -0800 Subject: [PATCH 26/45] remove dependency on itself --- models/ethereum/ethereum__token_prices_hourly.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/models/ethereum/ethereum__token_prices_hourly.sql b/models/ethereum/ethereum__token_prices_hourly.sql index eae11f12..a3042285 100644 --- a/models/ethereum/ethereum__token_prices_hourly.sql +++ b/models/ethereum/ethereum__token_prices_hourly.sql @@ -27,8 +27,10 @@ with hourly_prices as ( group by 1,2,3,4 ) , symbols as ( - select distinct symbol, token_address - from {{ this }} + select distinct p.symbol, lower(a.token_address) as token_address + from {{ source('shared', 'prices_v2') }} p + left outer join {{ source('shared', 'market_asset_metadata') }} a on p.asset_id = a.asset_id + where (a.platform_id = '1027' or a.asset_id = '1027' or a.platform_id = 'ethereum') ) , hour_symbols_pair as ( select * From 1e001b919f8eeb95ebfd4d3c83a612927537f169 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Wed, 19 Jan 2022 14:10:10 -0800 Subject: [PATCH 27/45] use token address and hour for cross join --- .../ethereum__token_prices_hourly.sql | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/models/ethereum/ethereum__token_prices_hourly.sql b/models/ethereum/ethereum__token_prices_hourly.sql index a3042285..571a255b 100644 --- a/models/ethereum/ethereum__token_prices_hourly.sql +++ b/models/ethereum/ethereum__token_prices_hourly.sql @@ -26,16 +26,17 @@ with hourly_prices as ( {% endif %} group by 1,2,3,4 ) -, symbols as ( - select distinct p.symbol, lower(a.token_address) as token_address +, token_addresses as ( + select distinct lower(a.token_address) as token_address from {{ source('shared', 'prices_v2') }} p left outer join {{ source('shared', 'market_asset_metadata') }} a on p.asset_id = a.asset_id where (a.platform_id = '1027' or a.asset_id = '1027' or a.platform_id = 'ethereum') + and (token_address is not null or p.symbol = 'ETH') ) -, hour_symbols_pair as ( +, hour_token_addresses_pair as ( select * from silver.hours - cross join symbols + cross join token_addresses {% if is_incremental() %} where hour >= current_date - 45 {% else %} @@ -45,20 +46,24 @@ with hourly_prices as ( , imputed as ( select h.hour - , h.symbol + -- , h.symbol , h.token_address + , p.symbol , p.decimals , p.price as avg_price -- , last_value(p.token_address) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as lag_token_address - , lag(p.decimals) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as lag_decimals - , lag(p.price) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as imputed_price - from hour_symbols_pair h - left outer join hourly_prices p on p.hour = h.hour and p.symbol = h.symbol and p.token_address = h.token_address + , lag(p.symbol) ignore nulls over (partition by h.token_address order by h.hour) as lag_symbol + , lag(p.decimals) ignore nulls over (partition by h.token_address order by h.hour) as lag_decimals + , lag(p.price) ignore nulls over (partition by h.token_address order by h.hour) as imputed_price + from hour_token_addresses_pair h + left outer join hourly_prices p on p.hour = h.hour and p.token_address = h.token_address ) select p.hour as hour - , p.symbol + -- , p.symbol , p.token_address + , case when symbol is not null then symbol + else lag_symbol end as symbol , case when decimals is not null then decimals else lag_decimals end as decimals , case when avg_price is not null then avg_price From 314aefe2fd39e19f60b85f79d30a729b1f082380 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Wed, 19 Jan 2022 14:10:18 -0800 Subject: [PATCH 28/45] initial tests --- .../ethereum/ethereum__token_prices_hourly.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 models/ethereum/ethereum__token_prices_hourly.yml diff --git a/models/ethereum/ethereum__token_prices_hourly.yml b/models/ethereum/ethereum__token_prices_hourly.yml new file mode 100644 index 00000000..43190fa4 --- /dev/null +++ b/models/ethereum/ethereum__token_prices_hourly.yml @@ -0,0 +1,18 @@ +version: 2 +models: + - name: ethereum__token_prices_hourly + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - SYMBOL + - TOKEN_ADDRESS + columns: + - name: HOUR + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: PRICE + tests: + - not_null \ No newline at end of file From 1a00f7b8f3d1964cd05c13df0de86b461901bc47 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Wed, 19 Jan 2022 14:10:27 -0800 Subject: [PATCH 29/45] remove unused source --- models/sources.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/models/sources.yml b/models/sources.yml index e99a2eee..9f19697f 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -61,10 +61,6 @@ sources: - name: uniswapv3_pool_stats - name: uniswapv3_positions - name: uniswapv3_swaps - - name: ethereum_nondbt - schema: ethereum - tables: - - name: token_prices_hourly_v2 - name: gold schema: gold tables: From c2e20a67c75727d2c909c58b3e0f646415b98220 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Wed, 19 Jan 2022 14:15:20 -0800 Subject: [PATCH 30/45] remove commented out code --- models/ethereum/ethereum__token_prices_hourly.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/models/ethereum/ethereum__token_prices_hourly.sql b/models/ethereum/ethereum__token_prices_hourly.sql index 571a255b..3f5e10b9 100644 --- a/models/ethereum/ethereum__token_prices_hourly.sql +++ b/models/ethereum/ethereum__token_prices_hourly.sql @@ -46,12 +46,10 @@ with hourly_prices as ( , imputed as ( select h.hour - -- , h.symbol , h.token_address , p.symbol , p.decimals , p.price as avg_price - -- , last_value(p.token_address) ignore nulls over (partition by h.symbol, h.token_address order by h.hour) as lag_token_address , lag(p.symbol) ignore nulls over (partition by h.token_address order by h.hour) as lag_symbol , lag(p.decimals) ignore nulls over (partition by h.token_address order by h.hour) as lag_decimals , lag(p.price) ignore nulls over (partition by h.token_address order by h.hour) as imputed_price @@ -60,7 +58,6 @@ with hourly_prices as ( ) select p.hour as hour - -- , p.symbol , p.token_address , case when symbol is not null then symbol else lag_symbol end as symbol From 7f6aa35f30815f5184e3dbb05b0d1b43d2f399e6 Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Thu, 20 Jan 2022 10:58:47 -0600 Subject: [PATCH 31/45] fixes issue where contract name was pulling in an address --- models/ethereum/ethereum__events_emitted.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/ethereum/ethereum__events_emitted.sql b/models/ethereum/ethereum__events_emitted.sql index 7d984ea5..2d1e27f4 100644 --- a/models/ethereum/ethereum__events_emitted.sql +++ b/models/ethereum/ethereum__events_emitted.sql @@ -29,7 +29,7 @@ SELECT to_labels.project_name as tx_to_label, to_labels.address_name as tx_to_address_name, CONTRACT_ADDR AS contract_address, - COALESCE(contract_labels.address,CONTRACT_NAME) AS contract_name, + COALESCE(contract_labels.address_name, CONTRACT_NAME) AS contract_name, TX_SUCCEEDED AS tx_succeeded FROM {{ ref('silver_ethereum__events_emitted') }} b From d9a495e753e707cbee4dcb3e3472e25809cf30d8 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 20 Jan 2022 14:51:00 -0800 Subject: [PATCH 32/45] fix eth prices due to null address, change decimal lookup --- .../ethereum/ethereum__token_prices_hourly.sql | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/models/ethereum/ethereum__token_prices_hourly.sql b/models/ethereum/ethereum__token_prices_hourly.sql index 3f5e10b9..39e4825f 100644 --- a/models/ethereum/ethereum__token_prices_hourly.sql +++ b/models/ethereum/ethereum__token_prices_hourly.sql @@ -8,7 +8,16 @@ ) }} -with hourly_prices as ( +with full_decimals as ( + select + lower(address) as contract_address, + meta :decimals :: int as decimals + from + {{ ref('silver_ethereum__contracts') }} + where + meta :decimals not like '%00%' qualify(row_number() over(partition by contract_address order by decimals desc) = 1) --need the %00% filter to exclude messy data +), +hourly_prices as ( select p.symbol, date_trunc('hour', recorded_at) as hour, @@ -17,7 +26,7 @@ with hourly_prices as ( avg(price) as price from {{ source('shared', 'prices_v2') }} p left outer join {{ source('shared', 'market_asset_metadata') }} a on p.asset_id = a.asset_id - left outer join {{ source('ethereum', 'ethereum_contract_decimal_adjustments') }} d on d.address = lower(a.token_address) + left outer join full_decimals d on d.contract_address = lower(a.token_address) where (a.platform_id = '1027' or a.asset_id = '1027' or a.platform_id = 'ethereum') {% if is_incremental() %} and recorded_at >= current_date - 45 @@ -54,7 +63,8 @@ with hourly_prices as ( , lag(p.decimals) ignore nulls over (partition by h.token_address order by h.hour) as lag_decimals , lag(p.price) ignore nulls over (partition by h.token_address order by h.hour) as imputed_price from hour_token_addresses_pair h - left outer join hourly_prices p on p.hour = h.hour and p.token_address = h.token_address + left outer join hourly_prices p on p.hour = h.hour + and (p.token_address = h.token_address or (h.token_address is null and p.symbol = 'ETH')) ) select p.hour as hour @@ -69,5 +79,3 @@ select else FALSE end as is_imputed from imputed p where price is not null - - From adcca8ecbe3f73020d71a17a98058c6025430776 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 20 Jan 2022 15:13:18 -0800 Subject: [PATCH 33/45] cap the hour to the current sysdate hour --- models/ethereum/ethereum__token_prices_hourly.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ethereum/ethereum__token_prices_hourly.sql b/models/ethereum/ethereum__token_prices_hourly.sql index 39e4825f..76ae8707 100644 --- a/models/ethereum/ethereum__token_prices_hourly.sql +++ b/models/ethereum/ethereum__token_prices_hourly.sql @@ -47,9 +47,9 @@ hourly_prices as ( from silver.hours cross join token_addresses {% if is_incremental() %} - where hour >= current_date - 45 + where hour between current_date - 45 and date_trunc('hour',sysdate()) {% else %} - where hour >= '2020-05-05' -- first date with valid prices data + where hour between '2020-05-05' and date_trunc('hour',sysdate())-- first date with valid prices data {% endif %} ) , imputed as ( From da9880b2bc747cdefcb7cfd275dbe3ef60058bab Mon Sep 17 00:00:00 2001 From: Jessica Huhnke <91915469+jhuhnke@users.noreply.github.com> Date: Fri, 21 Jan 2022 13:31:45 -0600 Subject: [PATCH 34/45] Update to contract name and disabled tests (#265) --- models/polygon/polygon__events_emitted.sql | 2 +- models/polygon/polygon__events_emitted.yml | 27 ++++++++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/models/polygon/polygon__events_emitted.sql b/models/polygon/polygon__events_emitted.sql index bad02644..7dca3fd5 100644 --- a/models/polygon/polygon__events_emitted.sql +++ b/models/polygon/polygon__events_emitted.sql @@ -39,7 +39,7 @@ SELECT to_labels.project_name as tx_to_label, to_labels.address_name as tx_to_address_name, CONTRACT_ADDRESS AS contract_address, - COALESCE(contract_labels.address,CONTRACT_NAME) AS contract_name, + COALESCE(contract_labels.address_name, CONTRACT_NAME) AS contract_name, TX_SUCCEEDED AS tx_succeeded FROM {{ ref('silver_polygon__events_emitted')}} b diff --git a/models/polygon/polygon__events_emitted.yml b/models/polygon/polygon__events_emitted.yml index a71dc946..fd5ba3f4 100644 --- a/models/polygon/polygon__events_emitted.yml +++ b/models/polygon/polygon__events_emitted.yml @@ -24,7 +24,8 @@ models: regex: 0[xX][0-9a-fA-F]+ - name: CONTRACT_NAME tests: - - not_null + - not_null: + enabled: False # Crosschain labels will not be available for all addresses - name: EVENT_INDEX tests: - not_null @@ -44,16 +45,20 @@ models: regex: 0[xX][0-9a-fA-F]+ - name: TX_FROM_ADDRESS_NAME tests: - - not_null + - not_null: + enabled: False # Crosschain labels will not be available for all addresses - name: TX_FROM_LABEL tests: - - not_null + - not_null: + enabled: False # Crosschain labels will not be available for all addresses - name: TX_FROM_LABEL_SUBTYPE tests: - - not_null + - not_null: + enabled: False # Crosschain labels will not be available for all addresses - name: TX_FROM_LABEL_TYPE tests: - - not_null + - not_null: + enabled: False # Crosschain labels will not be available for all addresses - name: TX_ID tests: - not_null @@ -67,13 +72,17 @@ models: regex: 0[xX][0-9a-fA-F]+ - name: TX_TO_ADDRESS_NAME tests: - - not_null + - not_null: + enabled: False # Crosschain labels will not be available for all addresses - name: TX_TO_LABEL tests: - - not_null + - not_null: + enabled: False # Crosschain labels will not be available for all addresses - name: TX_TO_LABEL_SUBTYPE tests: - - not_null + - not_null: + enabled: False # Crosschain labels will not be available for all addresses - name: TX_TO_LABEL_TYPE tests: - - not_null + - not_null: + enabled: False # Crosschain labels will not be available for all addresses From 6356f1d1059b4d2e80068d5e656b7f5193537317 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 24 Jan 2022 19:34:45 -0500 Subject: [PATCH 35/45] Added missing currencies to oracle prices tables --- models/terra/gold/terra__oracle_prices.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/terra/gold/terra__oracle_prices.sql b/models/terra/gold/terra__oracle_prices.sql index 25157559..60b8b3dd 100644 --- a/models/terra/gold/terra__oracle_prices.sql +++ b/models/terra/gold/terra__oracle_prices.sql @@ -137,6 +137,8 @@ SELECT WHEN l.currency = 'uaud' THEN 'AUT' WHEN l.currency = 'uidr' THEN 'IDT' WHEN l.currency = 'uphp' THEN 'PHT' + WHEN l.currency = 'utwd' THEN 'TWT' + WHEN l.currency = 'umyr' THEN 'MYT' ELSE l.currency END AS symbol, exchange_rate AS luna_exchange_rate, From a96f0d167deb1652db010e9a97692bc1efab8865 Mon Sep 17 00:00:00 2001 From: jkallogjeri <41237295+jkallogjeri@users.noreply.github.com> Date: Mon, 24 Jan 2022 19:42:52 -0500 Subject: [PATCH 36/45] very annoying change --- models/terra/gold/terra__oracle_prices.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/terra/gold/terra__oracle_prices.sql b/models/terra/gold/terra__oracle_prices.sql index 60b8b3dd..6a68b66b 100644 --- a/models/terra/gold/terra__oracle_prices.sql +++ b/models/terra/gold/terra__oracle_prices.sql @@ -68,7 +68,7 @@ massets AS( m.block_timestamp, m.block_id, m.msg_value :execute_msg :feed_price :prices [0] [0] :: STRING AS currency, - p.address AS symbol, + p.address_name AS symbol, m.msg_value :execute_msg :feed_price :prices [0] [1] :: FLOAT AS price FROM {{ ref('silver_terra__msgs') }} m From 7fbf95053315d16205aa02b330dc0c2e7a683423 Mon Sep 17 00:00:00 2001 From: Nick Payiatis <89624589+nickpayiatis@users.noreply.github.com> Date: Tue, 25 Jan 2022 20:53:55 -0500 Subject: [PATCH 37/45] An 578/upgrading schema to 2.8 (#254) --- ...algorand__application_call_transaction.sql | 2 +- ...algorand__application_call_transaction.yml | 2 +- ...orand__asset_configuration_transaction.sql | 2 +- ...orand__asset_configuration_transaction.yml | 2 +- .../algorand__asset_freeze_transaction.sql | 2 +- .../algorand__asset_freeze_transaction.yml | 2 +- .../algorand__asset_transfer_transaction.sql | 2 +- .../algorand__asset_transfer_transaction.yml | 2 +- models/algorand/gold/algorand__block.sql | 2 +- models/algorand/gold/algorand__block.yml | 2 +- ...algorand__key_registration_transaction.sql | 2 +- ...algorand__key_registration_transaction.yml | 2 +- .../gold/algorand__payment_transaction.sql | 2 +- .../gold/algorand__payment_transaction.yml | 2 +- .../algorand/gold/algorand__transactions.sql | 2 +- .../algorand/gold/algorand__transactions.yml | 2 +- ...algorand__application_call_transaction.sql | 10 +++---- ...algorand__application_call_transaction.yml | 2 +- ...orand__asset_configuration_transaction.sql | 10 +++---- ...orand__asset_configuration_transaction.yml | 2 +- ...ver_algorand__asset_freeze_transaction.sql | 10 +++---- ...ver_algorand__asset_freeze_transaction.yml | 2 +- ...r_algorand__asset_transfer_transaction.sql | 10 +++---- ...r_algorand__asset_transfer_transaction.yml | 2 +- .../silver/silver_algorand__block.sql | 2 +- .../silver/silver_algorand__block.yml | 2 +- .../silver/silver_algorand__inner_txids.sql | 19 +++++--------- .../silver/silver_algorand__inner_txids.yml | 11 +------- ...algorand__key_registration_transaction.sql | 10 +++---- ...algorand__key_registration_transaction.yml | 2 +- .../silver_algorand__payment_transaction.sql | 10 +++---- .../silver_algorand__payment_transaction.yml | 2 +- ...er_algorand__transaction_participation.sql | 26 ++++++++++++++----- .../silver/silver_algorand__transactions.sql | 10 +++---- .../silver/silver_algorand__transactions.yml | 2 +- models/sources.yml | 2 +- 36 files changed, 88 insertions(+), 90 deletions(-) diff --git a/models/algorand/gold/algorand__application_call_transaction.sql b/models/algorand/gold/algorand__application_call_transaction.sql index 7f12835c..aa509c56 100644 --- a/models/algorand/gold/algorand__application_call_transaction.sql +++ b/models/algorand/gold/algorand__application_call_transaction.sql @@ -14,7 +14,7 @@ SELECT app_id, tx_type, tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra FROM diff --git a/models/algorand/gold/algorand__application_call_transaction.yml b/models/algorand/gold/algorand__application_call_transaction.yml index 20320b78..6a807163 100644 --- a/models/algorand/gold/algorand__application_call_transaction.yml +++ b/models/algorand/gold/algorand__application_call_transaction.yml @@ -32,7 +32,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/gold/algorand__asset_configuration_transaction.sql b/models/algorand/gold/algorand__asset_configuration_transaction.sql index 3d0335f8..026b1b1e 100644 --- a/models/algorand/gold/algorand__asset_configuration_transaction.sql +++ b/models/algorand/gold/algorand__asset_configuration_transaction.sql @@ -16,7 +16,7 @@ SELECT asset_parameters, tx_type, tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra FROM diff --git a/models/algorand/gold/algorand__asset_configuration_transaction.yml b/models/algorand/gold/algorand__asset_configuration_transaction.yml index b23daebf..8c36ed25 100644 --- a/models/algorand/gold/algorand__asset_configuration_transaction.yml +++ b/models/algorand/gold/algorand__asset_configuration_transaction.yml @@ -36,7 +36,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/gold/algorand__asset_freeze_transaction.sql b/models/algorand/gold/algorand__asset_freeze_transaction.sql index 8ca71d62..a5ab0e45 100644 --- a/models/algorand/gold/algorand__asset_freeze_transaction.sql +++ b/models/algorand/gold/algorand__asset_freeze_transaction.sql @@ -16,7 +16,7 @@ SELECT fee, tx_type, tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra FROM diff --git a/models/algorand/gold/algorand__asset_freeze_transaction.yml b/models/algorand/gold/algorand__asset_freeze_transaction.yml index ad30a09e..b7dceee6 100644 --- a/models/algorand/gold/algorand__asset_freeze_transaction.yml +++ b/models/algorand/gold/algorand__asset_freeze_transaction.yml @@ -37,7 +37,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/gold/algorand__asset_transfer_transaction.sql b/models/algorand/gold/algorand__asset_transfer_transaction.sql index 9610c3ff..62d111d8 100644 --- a/models/algorand/gold/algorand__asset_transfer_transaction.sql +++ b/models/algorand/gold/algorand__asset_transfer_transaction.sql @@ -18,7 +18,7 @@ SELECT asset_transferred, tx_type, tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra FROM diff --git a/models/algorand/gold/algorand__asset_transfer_transaction.yml b/models/algorand/gold/algorand__asset_transfer_transaction.yml index fda3b4b9..e3225f52 100644 --- a/models/algorand/gold/algorand__asset_transfer_transaction.yml +++ b/models/algorand/gold/algorand__asset_transfer_transaction.yml @@ -27,7 +27,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/gold/algorand__block.sql b/models/algorand/gold/algorand__block.sql index 68616f22..75fc062d 100644 --- a/models/algorand/gold/algorand__block.sql +++ b/models/algorand/gold/algorand__block.sql @@ -8,7 +8,7 @@ SELECT block_timestamp, rewardslevel, network, - genisis_hash, + genesis_hash, prev_block_hash, txn_root, header diff --git a/models/algorand/gold/algorand__block.yml b/models/algorand/gold/algorand__block.yml index 07f0be9f..829916c3 100644 --- a/models/algorand/gold/algorand__block.yml +++ b/models/algorand/gold/algorand__block.yml @@ -18,7 +18,7 @@ models: - name: NETWORK tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: PREV_BLOCK_HASH diff --git a/models/algorand/gold/algorand__key_registration_transaction.sql b/models/algorand/gold/algorand__key_registration_transaction.sql index 94b46343..fbdfb79c 100644 --- a/models/algorand/gold/algorand__key_registration_transaction.sql +++ b/models/algorand/gold/algorand__key_registration_transaction.sql @@ -19,7 +19,7 @@ SELECT vote_keydilution, tx_type, tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra FROM diff --git a/models/algorand/gold/algorand__key_registration_transaction.yml b/models/algorand/gold/algorand__key_registration_transaction.yml index a0e3944b..ab2b7c0a 100644 --- a/models/algorand/gold/algorand__key_registration_transaction.yml +++ b/models/algorand/gold/algorand__key_registration_transaction.yml @@ -34,7 +34,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/gold/algorand__payment_transaction.sql b/models/algorand/gold/algorand__payment_transaction.sql index 8ae91f5a..46a99996 100644 --- a/models/algorand/gold/algorand__payment_transaction.sql +++ b/models/algorand/gold/algorand__payment_transaction.sql @@ -16,7 +16,7 @@ SELECT fee, tx_type, tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra FROM diff --git a/models/algorand/gold/algorand__payment_transaction.yml b/models/algorand/gold/algorand__payment_transaction.yml index 8ce031ec..3de70ae6 100644 --- a/models/algorand/gold/algorand__payment_transaction.yml +++ b/models/algorand/gold/algorand__payment_transaction.yml @@ -27,7 +27,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/gold/algorand__transactions.sql b/models/algorand/gold/algorand__transactions.sql index 6630b95d..ab710e89 100644 --- a/models/algorand/gold/algorand__transactions.sql +++ b/models/algorand/gold/algorand__transactions.sql @@ -14,7 +14,7 @@ SELECT fee, tx_type, tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra FROM diff --git a/models/algorand/gold/algorand__transactions.yml b/models/algorand/gold/algorand__transactions.yml index 1bb360b2..4b1a761e 100644 --- a/models/algorand/gold/algorand__transactions.yml +++ b/models/algorand/gold/algorand__transactions.yml @@ -21,7 +21,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/silver/silver_algorand__application_call_transaction.sql b/models/algorand/silver/silver_algorand__application_call_transaction.sql index 54e528d7..c6d9e03e 100644 --- a/models/algorand/silver/silver_algorand__application_call_transaction.sql +++ b/models/algorand/silver/silver_algorand__application_call_transaction.sql @@ -12,11 +12,11 @@ WITH allTXN AS ( b.round AS block_id, txn :txn :grp :: STRING AS tx_group_id, CASE - WHEN b.txid :: STRING = '' THEN ft.txn_txn_id :: text + WHEN b.txid IS NULL THEN ft.txn_txn_id :: text ELSE b.txid :: text END AS tx_id, CASE - WHEN b.txid :: STRING = '' THEN 'true' + WHEN b.txid IS NULL THEN 'true' ELSE 'false' END AS inner_tx, txn :txn :snd :: text AS sender, @@ -27,9 +27,9 @@ WITH allTXN AS ( txn :txn :apid AS app_id, txn :txn :type :: STRING AS tx_type, CASE - WHEN b.txid :: STRING = '' THEN ft.genisis_hash :: text + WHEN b.txid IS NULL THEN ft.genesis_hash :: text ELSE txn :txn :gh :: STRING - END AS genisis_hash, + END AS genesis_hash, txn AS tx_message, extra, b._FIVETRAN_SYNCED AS _FIVETRAN_SYNCED @@ -62,7 +62,7 @@ SELECT app_id, csv.type AS tx_type, csv.name AS tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra, concat_ws( diff --git a/models/algorand/silver/silver_algorand__application_call_transaction.yml b/models/algorand/silver/silver_algorand__application_call_transaction.yml index f795636a..2a7f6666 100644 --- a/models/algorand/silver/silver_algorand__application_call_transaction.yml +++ b/models/algorand/silver/silver_algorand__application_call_transaction.yml @@ -32,7 +32,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql b/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql index 05ae811e..8d14d46f 100644 --- a/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql +++ b/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql @@ -12,11 +12,11 @@ WITH allTXN AS ( b.round AS block_id, txn :txn :grp :: STRING AS tx_group_id, CASE - WHEN b.txid :: STRING = '' THEN ft.txn_txn_id :: text + WHEN b.txid IS NULL THEN ft.txn_txn_id :: text ELSE b.txid :: text END AS tx_id, CASE - WHEN b.txid :: STRING = '' THEN 'true' + WHEN b.txid IS NULL THEN 'true' ELSE 'false' END AS inner_tx, asset AS asset_id, @@ -29,9 +29,9 @@ WITH allTXN AS ( txn :txn :apar AS asset_parameters, txn :txn :type :: STRING AS tx_type, CASE - WHEN b.txid :: STRING = '' THEN ft.genisis_hash :: text + WHEN b.txid IS NULL THEN ft.genesis_hash :: text ELSE txn :txn :gh :: STRING - END AS genisis_hash, + END AS genesis_hash, txn AS tx_message, extra, b._FIVETRAN_SYNCED AS _FIVETRAN_SYNCED @@ -65,7 +65,7 @@ SELECT asset_parameters, csv.type AS tx_type, csv.name AS tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra, concat_ws( diff --git a/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml b/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml index 4c30e404..4f0d355a 100644 --- a/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml +++ b/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml @@ -36,7 +36,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/silver/silver_algorand__asset_freeze_transaction.sql b/models/algorand/silver/silver_algorand__asset_freeze_transaction.sql index b895631d..90093872 100644 --- a/models/algorand/silver/silver_algorand__asset_freeze_transaction.sql +++ b/models/algorand/silver/silver_algorand__asset_freeze_transaction.sql @@ -12,11 +12,11 @@ WITH allTXN AS ( b.round AS block_id, txn :txn :grp :: STRING AS tx_group_id, CASE - WHEN b.txid :: STRING = '' THEN ft.txn_txn_id :: text + WHEN b.txid IS NULL THEN ft.txn_txn_id :: text ELSE b.txid :: text END AS tx_id, CASE - WHEN b.txid :: STRING = '' THEN 'true' + WHEN b.txid IS NULL THEN 'true' ELSE 'false' END AS inner_tx, asset AS asset_id, @@ -29,9 +29,9 @@ WITH allTXN AS ( ) AS fee, txn :txn :type :: STRING AS tx_type, CASE - WHEN b.txid :: STRING = '' THEN ft.genisis_hash :: text + WHEN b.txid IS NULL THEN ft.genesis_hash :: text ELSE txn :txn :gh :: STRING - END AS genisis_hash, + END AS genesis_hash, txn AS tx_message, extra, b._FIVETRAN_SYNCED AS _FIVETRAN_SYNCED @@ -67,7 +67,7 @@ SELECT fee, csv.type AS tx_type, csv.name AS tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra, concat_ws( diff --git a/models/algorand/silver/silver_algorand__asset_freeze_transaction.yml b/models/algorand/silver/silver_algorand__asset_freeze_transaction.yml index aaa0cbe6..42a80948 100644 --- a/models/algorand/silver/silver_algorand__asset_freeze_transaction.yml +++ b/models/algorand/silver/silver_algorand__asset_freeze_transaction.yml @@ -37,7 +37,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/silver/silver_algorand__asset_transfer_transaction.sql b/models/algorand/silver/silver_algorand__asset_transfer_transaction.sql index c7920697..1884bacb 100644 --- a/models/algorand/silver/silver_algorand__asset_transfer_transaction.sql +++ b/models/algorand/silver/silver_algorand__asset_transfer_transaction.sql @@ -12,11 +12,11 @@ WITH allTXN AS ( b.round AS block_id, txn :txn :grp :: STRING AS tx_group_id, CASE - WHEN b.txid :: STRING = '' THEN ft.txn_txn_id :: text + WHEN b.txid IS NULL THEN ft.txn_txn_id :: text ELSE b.txid :: text END AS tx_id, CASE - WHEN b.txid :: STRING = '' THEN 'true' + WHEN b.txid IS NULL THEN 'true' ELSE 'false' END AS inner_tx, asset AS asset_id, @@ -31,9 +31,9 @@ WITH allTXN AS ( txn :txn :xaid AS asset_transferred, txn :txn :type :: STRING AS tx_type, CASE - WHEN b.txid :: STRING = '' THEN ft.genisis_hash :: text + WHEN b.txid IS NULL THEN ft.genesis_hash :: text ELSE txn :txn :gh :: STRING - END AS genisis_hash, + END AS genesis_hash, txn AS tx_message, extra, b._FIVETRAN_SYNCED AS _FIVETRAN_SYNCED @@ -73,7 +73,7 @@ SELECT asset_transferred, csv.type AS tx_type, csv.name AS tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra, concat_ws( diff --git a/models/algorand/silver/silver_algorand__asset_transfer_transaction.yml b/models/algorand/silver/silver_algorand__asset_transfer_transaction.yml index 84a54b95..4ca5b79e 100644 --- a/models/algorand/silver/silver_algorand__asset_transfer_transaction.yml +++ b/models/algorand/silver/silver_algorand__asset_transfer_transaction.yml @@ -31,7 +31,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/silver/silver_algorand__block.sql b/models/algorand/silver/silver_algorand__block.sql index 9f25d5a3..50879200 100644 --- a/models/algorand/silver/silver_algorand__block.sql +++ b/models/algorand/silver/silver_algorand__block.sql @@ -10,7 +10,7 @@ SELECT realtime :: TIMESTAMP AS block_timestamp, rewardslevel AS rewardslevel, header :gen :: STRING AS network, - header :gh :: STRING AS genisis_hash, + header :gh :: STRING AS genesis_hash, header :prev :: STRING AS prev_block_hash, header :txn :: STRING AS txn_root, header, diff --git a/models/algorand/silver/silver_algorand__block.yml b/models/algorand/silver/silver_algorand__block.yml index 08f3d8d5..ccf0bd90 100644 --- a/models/algorand/silver/silver_algorand__block.yml +++ b/models/algorand/silver/silver_algorand__block.yml @@ -18,7 +18,7 @@ models: - name: NETWORK tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: PREV_BLOCK_HASH diff --git a/models/algorand/silver/silver_algorand__inner_txids.sql b/models/algorand/silver/silver_algorand__inner_txids.sql index 8748ee06..d41594ff 100644 --- a/models/algorand/silver/silver_algorand__inner_txids.sql +++ b/models/algorand/silver/silver_algorand__inner_txids.sql @@ -11,6 +11,7 @@ WITH emptyROUNDS AS ( ROUND, intra, txn, + extra, _FIVETRAN_SYNCED FROM {{ source( @@ -18,37 +19,29 @@ WITH emptyROUNDS AS ( 'TXN' ) }} WHERE - txid :: STRING = '' + txid IS NULL ), fulljson AS ( SELECT ROUND, txid, intra, - t.value AS message, - txn, txn :txn :gh :: STRING AS gh FROM {{ source( 'algorand', 'TXN' - ) }}, - LATERAL FLATTEN( - input => txn :dt :itx - ) t + ) }} WHERE - txn :dt :itx IS NOT NULL + txid IS NOT NULL ) SELECT f.round AS txn_round, er.round AS inner_round, f.txid AS txn_txn_id, - f.txn AS txn_body, - f.message AS txn_inner_txn, - er.txn AS inner_message, er.intra AS inner_intra, f.intra AS txn_intra, - f.gh AS genisis_hash, + f.gh AS genesis_hash, concat_ws( '-', er.round :: STRING, @@ -58,7 +51,7 @@ SELECT FROM emptyROUNDS er LEFT JOIN fulljson f - ON er.txn = f.message + ON er.extra :"root-intra" :: NUMBER = f.intra AND er.round = f.round WHERE f.round IS NOT NULL diff --git a/models/algorand/silver/silver_algorand__inner_txids.yml b/models/algorand/silver/silver_algorand__inner_txids.yml index e879bed0..ccfa6393 100644 --- a/models/algorand/silver/silver_algorand__inner_txids.yml +++ b/models/algorand/silver/silver_algorand__inner_txids.yml @@ -16,21 +16,12 @@ models: - name: txn_txn_id tests: - not_null - - name: txn_body - tests: - - not_null - - name: txn_inner_txn - tests: - - not_null - - name: inner_message - tests: - - not_null - name: inner_intra tests: - not_null - name: txn_intra tests: - not_null - - name: genisis_hash + - name: genesis_hash tests: - not_null diff --git a/models/algorand/silver/silver_algorand__key_registration_transaction.sql b/models/algorand/silver/silver_algorand__key_registration_transaction.sql index 59125388..a5f6cec7 100644 --- a/models/algorand/silver/silver_algorand__key_registration_transaction.sql +++ b/models/algorand/silver/silver_algorand__key_registration_transaction.sql @@ -12,11 +12,11 @@ WITH allTXN AS ( b.round AS block_id, txn :txn :grp :: STRING AS tx_group_id, CASE - WHEN b.txid :: STRING = '' THEN ft.txn_txn_id :: text + WHEN b.txid IS NULL THEN ft.txn_txn_id :: text ELSE b.txid :: text END AS tx_id, CASE - WHEN b.txid :: STRING = '' THEN 'true' + WHEN b.txid IS NULL THEN 'true' ELSE 'false' END AS inner_tx, asset AS asset_id, @@ -32,9 +32,9 @@ WITH allTXN AS ( txn :txn :votekd AS vote_keydilution, txn :txn :type :: STRING AS tx_type, CASE - WHEN b.txid :: STRING = '' THEN ft.genisis_hash :: text + WHEN b.txid IS NULL THEN ft.genesis_hash :: text ELSE txn :txn :gh :: STRING - END AS genisis_hash, + END AS genesis_hash, txn AS tx_message, extra, b._FIVETRAN_SYNCED AS _FIVETRAN_SYNCED @@ -75,7 +75,7 @@ SELECT vote_keydilution, csv.type AS tx_type, csv.name AS tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra, concat_ws( diff --git a/models/algorand/silver/silver_algorand__key_registration_transaction.yml b/models/algorand/silver/silver_algorand__key_registration_transaction.yml index 0e913b66..365633b5 100644 --- a/models/algorand/silver/silver_algorand__key_registration_transaction.yml +++ b/models/algorand/silver/silver_algorand__key_registration_transaction.yml @@ -34,7 +34,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/silver/silver_algorand__payment_transaction.sql b/models/algorand/silver/silver_algorand__payment_transaction.sql index 64bfde36..107706cf 100644 --- a/models/algorand/silver/silver_algorand__payment_transaction.sql +++ b/models/algorand/silver/silver_algorand__payment_transaction.sql @@ -12,11 +12,11 @@ WITH allTXN AS ( b.round AS block_id, txn :txn :grp :: STRING AS tx_group_id, CASE - WHEN b.txid :: STRING = '' THEN ft.txn_txn_id :: text + WHEN b.txid IS NULL THEN ft.txn_txn_id :: text ELSE b.txid :: text END AS tx_id, CASE - WHEN b.txid :: STRING = '' THEN 'true' + WHEN b.txid IS NULL THEN 'true' ELSE 'false' END AS inner_tx, asset AS asset_id, @@ -32,9 +32,9 @@ WITH allTXN AS ( ) AS fee, txn :txn :type :: STRING AS tx_type, CASE - WHEN b.txid :: STRING = '' THEN ft.genisis_hash :: text + WHEN b.txid IS NULL THEN ft.genesis_hash :: text ELSE txn :txn :gh :: STRING - END AS genisis_hash, + END AS genesis_hash, txn AS tx_message, extra, b._FIVETRAN_SYNCED AS _FIVETRAN_SYNCED @@ -70,7 +70,7 @@ SELECT fee, csv.type AS tx_type, csv.name AS tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra, concat_ws( diff --git a/models/algorand/silver/silver_algorand__payment_transaction.yml b/models/algorand/silver/silver_algorand__payment_transaction.yml index 823b4d5a..c82b48f2 100644 --- a/models/algorand/silver/silver_algorand__payment_transaction.yml +++ b/models/algorand/silver/silver_algorand__payment_transaction.yml @@ -31,7 +31,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/algorand/silver/silver_algorand__transaction_participation.sql b/models/algorand/silver/silver_algorand__transaction_participation.sql index 95a9634f..1de59b3e 100644 --- a/models/algorand/silver/silver_algorand__transaction_participation.sql +++ b/models/algorand/silver/silver_algorand__transaction_participation.sql @@ -5,11 +5,28 @@ tags = ['snowflake', 'algorand', 'transaction_participation', 'silver_algorand'] ) }} +WITH inner_tx_individual AS( + + SELECT + ROUND AS block_id, + intra, + addr :: text AS address, + MIN(_FIVETRAN_SYNCED) AS _FIVETRAN_SYNCED + FROM + {{ source( + 'algorand', + 'TXN_PARTICIPATION' + ) }} + GROUP BY + block_id, + intra, + address +) SELECT - ROUND AS block_id, + block_id, intra, algorand_decode_hex_addr( - addr :: text + address :: text ) AS address, concat_ws( '-', @@ -19,10 +36,7 @@ SELECT ) AS _unique_key, _FIVETRAN_SYNCED FROM - {{ source( - 'algorand', - 'TXN_PARTICIPATION' - ) }} + inner_tx_individual WHERE 1 = 1 diff --git a/models/algorand/silver/silver_algorand__transactions.sql b/models/algorand/silver/silver_algorand__transactions.sql index 42e1135f..532094ee 100644 --- a/models/algorand/silver/silver_algorand__transactions.sql +++ b/models/algorand/silver/silver_algorand__transactions.sql @@ -12,11 +12,11 @@ WITH allTXN AS ( b.round AS block_id, txn :txn :grp :: STRING AS tx_group_id, CASE - WHEN b.txid :: STRING = '' THEN ft.txn_txn_id :: text + WHEN b.txid IS NULL THEN ft.txn_txn_id :: text ELSE b.txid :: text END AS tx_id, CASE - WHEN b.txid :: STRING = '' THEN 'true' + WHEN b.txid IS NULL THEN 'true' ELSE 'false' END AS inner_tx, CASE @@ -30,9 +30,9 @@ WITH allTXN AS ( ) AS fee, txn :txn :type :: STRING AS tx_type, CASE - WHEN b.txid :: STRING = '' THEN ft.genisis_hash :: text + WHEN b.txid IS NULL THEN ft.genesis_hash :: text ELSE txn :txn :gh :: STRING - END AS genisis_hash, + END AS genesis_hash, txn AS tx_message, extra, b._FIVETRAN_SYNCED AS _FIVETRAN_SYNCED @@ -62,7 +62,7 @@ SELECT fee, csv.type AS tx_type, csv.name AS tx_type_name, - genisis_hash, + genesis_hash, tx_message, extra, concat_ws( diff --git a/models/algorand/silver/silver_algorand__transactions.yml b/models/algorand/silver/silver_algorand__transactions.yml index 679eb4c3..41e8f189 100644 --- a/models/algorand/silver/silver_algorand__transactions.yml +++ b/models/algorand/silver/silver_algorand__transactions.yml @@ -25,7 +25,7 @@ models: - name: TX_TYPE_NAME tests: - not_null - - name: GENISIS_HASH + - name: GENESIS_HASH tests: - not_null - name: TX_MESSAGE diff --git a/models/sources.yml b/models/sources.yml index 9f19697f..f2b2874f 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -134,7 +134,7 @@ sources: tables: - name: daily_balances - name: algorand - schema: BRONZE_ALGORAND_2_6_4_PUBLIC + schema: BRONZE_ALGORAND_2_8_0_PUBLIC tables: - name: ACCOUNT - name: ACCOUNT_APP From 2346498e4c5fad1b7e5aba60dbc30a74a7245fa3 Mon Sep 17 00:00:00 2001 From: Ava Masucci <31529186+amasucci13@users.noreply.github.com> Date: Wed, 26 Jan 2022 12:49:53 -0800 Subject: [PATCH 38/45] An 634/anchor tables updates (#269) --- models/terra/anchor/anchor__collateral.sql | 7 ++-- models/terra/anchor/anchor__collateral.yml | 1 + models/terra/anchor/anchor__redeem.sql | 41 +++++++++++++++------- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/models/terra/anchor/anchor__collateral.sql b/models/terra/anchor/anchor__collateral.sql index 60792940..72183395 100644 --- a/models/terra/anchor/anchor__collateral.sql +++ b/models/terra/anchor/anchor__collateral.sql @@ -46,7 +46,7 @@ msgs AS ( 'withdraw' AS action, msg_value :sender :: STRING AS sender, COALESCE(msg_value :execute_msg :send :contract :: STRING, msg_value :contract :: STRING) AS contract_address, - l.address AS contract_label + l.address_name AS contract_label FROM {{ ref('silver_terra__msgs') }} m @@ -153,8 +153,9 @@ FROM ) = o.hour AND msg_value :contract :: STRING = o.currency WHERE - msg_value :execute_msg :send :msg :deposit_collateral IS NOT NULL - AND tx_status = 'SUCCEEDED' + tx_id in (select tx_id from silver_terra.msgs where msg_value:execute_msg:lock_collateral is not null) + and tx_status = 'SUCCEEDED' + and msg_value :execute_msg :send :contract::STRING IN ('terra1ptjp2vfjrwh0j0faj9r6katm640kgjxnwwq9kn', 'terra10cxuzggyvvv44magvrh3thpdnk9cmlgk93gmx2') --Anchor Custody Contracts {% if is_incremental() %} AND block_timestamp :: DATE >= ( diff --git a/models/terra/anchor/anchor__collateral.yml b/models/terra/anchor/anchor__collateral.yml index c4d7e3e1..81c04781 100644 --- a/models/terra/anchor/anchor__collateral.yml +++ b/models/terra/anchor/anchor__collateral.yml @@ -6,6 +6,7 @@ models: combination_of_columns: - TX_ID - BLOCK_ID + - EVENT_TYPE - AMOUNT - CURRENCY columns: diff --git a/models/terra/anchor/anchor__redeem.sql b/models/terra/anchor/anchor__redeem.sql index 6b8bfc12..0b1ed4ca 100644 --- a/models/terra/anchor/anchor__redeem.sql +++ b/models/terra/anchor/anchor__redeem.sql @@ -35,15 +35,33 @@ GROUP BY 1, 2, 3 -) -SELECT - m.blockchain, +), + +redeem_events AS ( + SELECT + blockchain, chain_id, block_id, block_timestamp, tx_id, + tx_status, + msg_index, + event_attributes + FROM {{ ref('silver_terra__msg_events') }} + WHERE tx_id in (select tx_id from terra.msgs WHERE msg_value:contract::string = 'terra1hzh9vpxhsk8253se0vv5jj6etdvxu3nv8z07zu') + AND event_type = 'from_contract' + AND event_attributes:redeem_amount is not null + AND tx_status = 'SUCCEEDED' +) + +SELECT + e.blockchain, + e.chain_id, + e.block_id, + e.block_timestamp, + e.tx_id, msg_value :sender :: STRING AS sender, - msg_value :execute_msg :send :amount / pow( + event_attributes:redeem_amount::FLOAT / pow( 10, 6 ) AS amount, @@ -51,23 +69,22 @@ SELECT msg_value :contract :: STRING AS currency, COALESCE(msg_value :execute_msg :send :contract :: STRING, '') AS contract_address, COALESCE(l.address_name, '') AS contract_label -FROM - {{ ref('silver_terra__msgs') }} - m +FROM redeem_events e +JOIN {{ ref('silver_terra__msgs') }} m +ON e.tx_id = m.tx_id +AND e.msg_index = m.msg_index LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS l ON msg_value :execute_msg :send :contract :: STRING = l.address AND l.blockchain = 'terra' AND l.creator = 'flipside' LEFT OUTER JOIN prices r ON DATE_TRUNC( 'hour', - block_timestamp + e.block_timestamp ) = HOUR AND msg_value :contract :: STRING = r.currency -WHERE - msg_value :execute_msg :send :msg :redeem_stable IS NOT NULL - AND tx_status = 'SUCCEEDED' +WHERE e.tx_status = 'SUCCEEDED' {% if is_incremental() %} -AND block_timestamp :: DATE >= ( +AND e.block_timestamp :: DATE >= ( SELECT MAX( block_timestamp :: DATE From aee64a2cc17bbdfd24d9fe83730648d3dbf69a5f Mon Sep 17 00:00:00 2001 From: yulike Date: Wed, 26 Jan 2022 13:21:58 -0500 Subject: [PATCH 39/45] Update anchor__liquidations.sql --- models/terra/anchor/anchor__liquidations.sql | 165 ++++++++++++++++--- 1 file changed, 141 insertions(+), 24 deletions(-) diff --git a/models/terra/anchor/anchor__liquidations.sql b/models/terra/anchor/anchor__liquidations.sql index 469a3258..6cee0c3d 100644 --- a/models/terra/anchor/anchor__liquidations.sql +++ b/models/terra/anchor/anchor__liquidations.sql @@ -254,10 +254,136 @@ AND block_timestamp :: DATE >= ( ), multiple_repay_tbl_raw AS ( SELECT - * + *, + SPLIT_PART(key, '_', 0) AS key_index FROM multiple_repay_tbl_events_raw , lateral flatten ( input => event_attributes ) - WHERE key LIKE '%_repay_amount' AND split(key, '_')[0]%2 = 1 +), +multiple_repay_borrower_tbl_raw_index AS ( + SELECT + tx_id, + SPLIT_PART(key, '_', 0) AS key_index + FROM multiple_repay_tbl_raw + WHERE key LIKE '%_borrower' +), +multiple_repay_borrower_tbl_raw_value AS ( + SELECT + a.blockchain, + a.chain_id, + a.block_id, + a.block_timestamp, + a.tx_id, + a.key_index, +// key, + SUBSTRING(key, LEN(split_part(key, '_', 1))+2, LEN(key)) AS tx_subtype, + value + FROM multiple_repay_tbl_raw a + INNER JOIN multiple_repay_borrower_tbl_raw_index b + ON a.tx_id = b.tx_id AND a.key_index = b.key_index +), +multiple_repay_borrower_tbl AS ( + SELECT + tx_id, + block_timestamp, + blockchain, + chain_id, + block_id, + key_index, + RANK() OVER (PARTITION BY tx_id ORDER BY key_index ASC) AS key_index_rank, + SUM(count_key_index-2) OVER (PARTITION BY tx_id ORDER BY key_index ASC) AS count_key_index, + borrower, + repay_amount + FROM ( + SELECT + tx_id, + block_timestamp, + blockchain, + chain_id, + block_id, + key_index, + MAX(key_index) OVER (PARTITION BY tx_id, borrower) AS max_key_index, + COUNT(DISTINCT key_index) OVER (PARTITION BY tx_id, borrower) AS count_key_index, + borrower, + repay_amount + FROM ( + SELECT + tx_id, + block_timestamp, + blockchain, + chain_id, + block_id, + key_index, + "'borrower'"::STRING AS borrower, + "'repay_amount'" AS repay_amount + FROM multiple_repay_borrower_tbl_raw_value + pivot (max(value) for tx_subtype IN ('borrower', 'repay_amount')) p + ) + ) + WHERE key_index = max_key_index + ORDER BY + tx_id, + block_timestamp, + blockchain, + chain_id, + block_id, + key_index +), +multiple_repay_collateral_tbl_raw_index AS ( + SELECT + tx_id, + SPLIT_PART(key, '_', 0) AS key_index + FROM multiple_repay_tbl_raw + WHERE key LIKE '%_collateral_amount' +), +multiple_repay_collateral_tbl_raw_value AS ( + SELECT + a.blockchain, + a.chain_id, + a.block_id, + a.block_timestamp, + a.tx_id, + a.key_index, + -- key, + SUBSTRING(key, LEN(split_part(key, '_', 1))+2, LEN(key)) AS tx_subtype, + value + FROM multiple_repay_tbl_raw a + INNER JOIN multiple_repay_collateral_tbl_raw_index b + ON a.tx_id = b.tx_id AND a.key_index = b.key_index +), +multiple_repay_collateral_tbl AS ( + SELECT + tx_id, + block_timestamp, + blockchain, + chain_id, + block_id, + key_index, + "'stable_denom'"::STRING AS stable_denom, + "'liquidator_fee'" AS liquidator_fee, + "'liquidator'"::STRING AS liquidator, + "'collateral_token'"::STRING AS collateral_token, + "'collateral_amount'" AS collateral_amount, + "'bid_fee'" AS bid_fee + FROM multiple_repay_collateral_tbl_raw_value + pivot (max(value) for tx_subtype IN ('stable_denom', 'liquidator_fee', 'liquidator', 'collateral_token', 'collateral_amount', 'bid_fee')) p +), +multiple_repay_tbl_borrower_collateral AS ( + SELECT + a.blockchain, + a.chain_id, + a.block_id, + a.block_timestamp, + a.tx_id, + b.bid_fee, + a.borrower, + b.liquidator, + b.collateral_amount, + b.collateral_token, + a.repay_amount, + b.stable_denom + FROM multiple_repay_borrower_tbl a + LEFT JOIN multiple_repay_collateral_tbl b + ON a.tx_id = b.tx_id AND (a.key_index_rank-1) = b.key_index ), multiple_repay_tbl AS ( SELECT @@ -266,50 +392,41 @@ multiple_repay_tbl AS ( block_id, block_timestamp, tx_id, - COALESCE(this :bid_fee / pow( - 10, - 6 - ), this :"0_bid_fee" / pow( - 10, - 6 - )) AS bid_fee, - - this :"0_borrower" :: STRING AS borrower, - COALESCE(this :liquidator :: STRING, this :"0_liquidator" :: STRING) AS liquidator, - COALESCE(this :collateral_amount / pow( + bid_fee / pow( 10, 6 - ), this :"0_collateral_amount" / pow( + ) AS bid_fee, + borrower AS borrower, + liquidator AS liquidator, + collateral_amount / pow( 10, 6 - )) AS liquidated_amount, + ) AS liquidated_amount, liquidated_amount * l.price AS liquidated_amount_usd, - COALESCE(this :collateral_token :: STRING, this :"0_collateral_token" :: STRING) AS liquidated_currency, - value / pow( + collateral_token AS liquidated_currency, + repay_amount / pow( 10, 6 ) AS repay_amount, - repay_amount * r.price AS repay_amount_usd, - COALESCE(this :stable_denom :: STRING, this :"0_stable_denom" :: STRING) AS repay_currency, + repay_amount / pow(10,6) * r.price AS repay_amount_usd, + stable_denom::STRING AS repay_currency, 'terra1tmnqgvg567ypvsvk6rwsga3srp7e3lg6u0elp8' AS contract_address, 'Overseer' AS contract_label - FROM multiple_repay_tbl_raw + FROM multiple_repay_tbl_borrower_collateral LEFT OUTER JOIN prices l ON DATE_TRUNC( 'hour', block_timestamp ) = l.hour - AND COALESCE(event_attributes :collateral_token :: STRING, event_attributes :"0_collateral_token" :: STRING) = l.currency + AND collateral_token = l.currency LEFT OUTER JOIN prices r ON DATE_TRUNC( 'hour', block_timestamp ) = r.hour - AND COALESCE(event_attributes :stable_denom :: STRING, event_attributes :"0_stable_denom" :: STRING) = r.currency + AND stable_denom = r.currency ) SELECT * FROM single_payment_tbl UNION ALL SELECT * FROM multiple_repay_tbl - - From e36850029122dd83525d5fd1c879ba0db87783f5 Mon Sep 17 00:00:00 2001 From: yulike Date: Wed, 26 Jan 2022 19:19:07 -0500 Subject: [PATCH 40/45] updated the liquidations and repay tables --- models/terra/anchor/anchor__liquidations.sql | 25 ++- models/terra/anchor/anchor__repay.sql | 216 +++++++++++++++---- 2 files changed, 193 insertions(+), 48 deletions(-) diff --git a/models/terra/anchor/anchor__liquidations.sql b/models/terra/anchor/anchor__liquidations.sql index 6cee0c3d..2856aae2 100644 --- a/models/terra/anchor/anchor__liquidations.sql +++ b/models/terra/anchor/anchor__liquidations.sql @@ -221,9 +221,9 @@ single_payment_tbl AS ( liquidated_amount, liquidated_amount_usd, liquidated_currency, - repay_amount, - repay_amount_usd, - repay_currency, + -- repay_amount, + -- repay_amount_usd, + -- repay_currency, contract_address, contract_label FROM @@ -274,7 +274,7 @@ multiple_repay_borrower_tbl_raw_value AS ( a.block_timestamp, a.tx_id, a.key_index, -// key, + -- key, SUBSTRING(key, LEN(split_part(key, '_', 1))+2, LEN(key)) AS tx_subtype, value FROM multiple_repay_tbl_raw a @@ -302,6 +302,7 @@ multiple_repay_borrower_tbl AS ( block_id, key_index, MAX(key_index) OVER (PARTITION BY tx_id, borrower) AS max_key_index, + MIN(key_index) OVER (PARTITION BY tx_id, borrower) AS min_key_index, COUNT(DISTINCT key_index) OVER (PARTITION BY tx_id, borrower) AS count_key_index, borrower, repay_amount @@ -319,7 +320,7 @@ multiple_repay_borrower_tbl AS ( pivot (max(value) for tx_subtype IN ('borrower', 'repay_amount')) p ) ) - WHERE key_index = max_key_index + WHERE key_index <> min_key_index ORDER BY tx_id, block_timestamp, @@ -367,6 +368,7 @@ multiple_repay_collateral_tbl AS ( FROM multiple_repay_collateral_tbl_raw_value pivot (max(value) for tx_subtype IN ('stable_denom', 'liquidator_fee', 'liquidator', 'collateral_token', 'collateral_amount', 'bid_fee')) p ), + multiple_repay_tbl_borrower_collateral AS ( SELECT a.blockchain, @@ -404,12 +406,12 @@ multiple_repay_tbl AS ( ) AS liquidated_amount, liquidated_amount * l.price AS liquidated_amount_usd, collateral_token AS liquidated_currency, - repay_amount / pow( - 10, - 6 - ) AS repay_amount, - repay_amount / pow(10,6) * r.price AS repay_amount_usd, - stable_denom::STRING AS repay_currency, + -- repay_amount / pow( + -- 10, + -- 6 + -- ) AS repay_amount, + -- repay_amount / pow(10,6) * r.price AS repay_amount_usd, + -- stable_denom::STRING AS repay_currency, 'terra1tmnqgvg567ypvsvk6rwsga3srp7e3lg6u0elp8' AS contract_address, 'Overseer' AS contract_label FROM multiple_repay_tbl_borrower_collateral @@ -430,3 +432,4 @@ multiple_repay_tbl AS ( SELECT * FROM single_payment_tbl UNION ALL SELECT * FROM multiple_repay_tbl + diff --git a/models/terra/anchor/anchor__repay.sql b/models/terra/anchor/anchor__repay.sql index e1cc22ff..c457d4ee 100644 --- a/models/terra/anchor/anchor__repay.sql +++ b/models/terra/anchor/anchor__repay.sql @@ -35,44 +35,186 @@ GROUP BY 1, 2, 3 -) -SELECT - m.blockchain, - chain_id, - block_id, - block_timestamp, - tx_id, - msg_value :sender :: STRING AS sender, - msg_value :coins [0] :amount / pow( - 10, - 6 - ) AS amount, - amount * price AS amount_usd, - msg_value :coins [0] :denom :: STRING AS currency, - COALESCE(msg_value :contract :: STRING, '') AS contract_address, - COALESCE(l.address_name, '') AS contract_label -FROM - {{ ref('silver_terra__msgs') }} - m - LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS l - ON msg_value :contract :: STRING = l.address AND l.blockchain = 'terra' AND l.creator = 'flipside' - LEFT OUTER JOIN prices r - ON DATE_TRUNC( - 'hour', - block_timestamp - ) = HOUR - AND msg_value :coins [0] :denom :: STRING = r.currency -WHERE - msg_value :execute_msg :repay_stable IS NOT NULL - AND tx_status = 'SUCCEEDED' - -{% if is_incremental() %} -AND block_timestamp :: DATE >= ( +), +single_payment_tbl AS ( SELECT - MAX( - block_timestamp :: DATE - ) + m.blockchain, + chain_id, + block_id, + block_timestamp, + tx_id, + msg_value :sender :: STRING AS sender, + msg_value :coins [0] :amount / pow( + 10, + 6 + ) AS amount, + amount * price AS amount_usd, + msg_value :coins [0] :denom :: STRING AS currency, + COALESCE(msg_value :contract :: STRING, '') AS contract_address, + COALESCE(l.address_name, '') AS contract_label FROM {{ ref('silver_terra__msgs') }} + m + LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS l + ON msg_value :contract :: STRING = l.address AND l.blockchain = 'terra' AND l.creator = 'flipside' + LEFT OUTER JOIN prices r + ON DATE_TRUNC( + 'hour', + block_timestamp + ) = HOUR + AND msg_value :coins [0] :denom :: STRING = r.currency + WHERE + msg_value :execute_msg :repay_stable IS NOT NULL + AND tx_status = 'SUCCEEDED' + + {% if is_incremental() %} + AND block_timestamp :: DATE >= ( + SELECT + MAX( + block_timestamp :: DATE + ) + FROM + {{ ref('silver_terra__msgs') }} + ) + {% endif %} +), +multiple_repay_tbl_events_raw AS ( + SELECT + * + FROM {{ ref('silver_terra__msg_events') }} + WHERE + event_type = 'from_contract' + AND tx_status = 'SUCCEEDED' + AND event_attributes :"0_action" :: STRING = 'liquidate_collateral' + AND event_attributes :"2_repay_amount" IS NOT NULL + {% if is_incremental() %} + AND block_timestamp :: DATE >= ( + SELECT + MAX( + block_timestamp :: DATE + ) + FROM + {{ ref('silver_terra__msgs') }} + ) + {% endif %} +), +multiple_repay_tbl_raw AS ( + SELECT + *, + SPLIT_PART(key, '_', 0) AS key_index + FROM multiple_repay_tbl_events_raw + , lateral flatten ( input => event_attributes ) +), +multiple_repay_borrower_tbl_raw_index AS ( + SELECT + tx_id, + SPLIT_PART(key, '_', 0) AS key_index + FROM multiple_repay_tbl_raw + WHERE key LIKE '%_borrower' +), +multiple_repay_borrower_tbl_raw_value AS ( + SELECT + a.blockchain, + a.chain_id, + a.block_id, + a.block_timestamp, + a.tx_id, + a.key_index, + -- key, + SUBSTRING(key, LEN(split_part(key, '_', 1))+2, LEN(key)) AS tx_subtype, + value + FROM multiple_repay_tbl_raw a + INNER JOIN multiple_repay_borrower_tbl_raw_index b + ON a.tx_id = b.tx_id AND a.key_index = b.key_index +), +multiple_repay_borrower_tbl AS ( + SELECT + tx_id, + block_timestamp, + blockchain, + chain_id, + block_id, + key_index, + RANK() OVER (PARTITION BY tx_id ORDER BY key_index ASC) AS key_index_rank, + SUM(count_key_index-2) OVER (PARTITION BY tx_id ORDER BY key_index ASC) AS count_key_index, + sender, + borrower, + amount, + currency, + contract_address + FROM ( + SELECT + tx_id, + block_timestamp, + blockchain, + chain_id, + block_id, + key_index, + MAX(key_index) OVER (PARTITION BY tx_id, borrower) AS max_key_index, + COUNT(DISTINCT key_index) OVER (PARTITION BY tx_id, borrower) AS count_key_index, + sender, + borrower, + repay_amount AS amount, + stable_denom AS currency, + contract_address + FROM ( + SELECT + tx_id, + block_timestamp, + blockchain, + chain_id, + block_id, + key_index, + "'liquidator'"::STRING AS sender, + "'borrower'"::STRING AS borrower, + "'repay_amount'" AS repay_amount, + "'stable_denom'"::STRING AS stable_denom, + "'contract_address'"::STRING AS contract_address + FROM multiple_repay_borrower_tbl_raw_value + pivot (max(value) for tx_subtype IN ('liquidator', 'borrower', 'repay_amount', 'stable_denom', 'contract_address')) p + ) + ) + WHERE key_index = max_key_index + ORDER BY + tx_id, + block_timestamp, + blockchain, + chain_id, + block_id, + key_index +), +multiple_payment_tbl AS ( + SELECT + m.blockchain, + chain_id, + block_id, + block_timestamp, + tx_id, + last_value(sender ignore nulls) over (partition by tx_id order by sender) as sender, + amount / pow( + 10, + 6 + ) AS amount, + amount / pow( + 10, + 6 + ) * price AS amount_usd, + COALESCE(m.currency, 'uusd') AS currency, + contract_address, + COALESCE(l.address_name, '') AS contract_label + FROM + multiple_repay_borrower_tbl + m + LEFT OUTER JOIN {{ ref('silver_crosschain__address_labels') }} AS l + ON m.contract_address = l.address AND l.blockchain = 'terra' AND l.creator = 'flipside' + LEFT OUTER JOIN prices r + ON DATE_TRUNC( + 'hour', + block_timestamp + ) = HOUR + AND COALESCE(m.currency, 'uusd') = r.currency ) -{% endif %} + +SELECT * FROM multiple_payment_tbl +UNION ALL +SELECT * FROM single_payment_tbl \ No newline at end of file From e331b831223c45b9451bd24b3c3a71a68b2ba113 Mon Sep 17 00:00:00 2001 From: yulike Date: Wed, 26 Jan 2022 20:59:25 -0500 Subject: [PATCH 41/45] update the repay table --- models/terra/anchor/anchor__repay.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/terra/anchor/anchor__repay.sql b/models/terra/anchor/anchor__repay.sql index c457d4ee..e27ec56e 100644 --- a/models/terra/anchor/anchor__repay.sql +++ b/models/terra/anchor/anchor__repay.sql @@ -152,7 +152,7 @@ multiple_repay_borrower_tbl AS ( key_index, MAX(key_index) OVER (PARTITION BY tx_id, borrower) AS max_key_index, COUNT(DISTINCT key_index) OVER (PARTITION BY tx_id, borrower) AS count_key_index, - sender, + last_value(sender ignore nulls) over (partition by tx_id order by sender) as sender, borrower, repay_amount AS amount, stable_denom AS currency, @@ -190,7 +190,7 @@ multiple_payment_tbl AS ( block_id, block_timestamp, tx_id, - last_value(sender ignore nulls) over (partition by tx_id order by sender) as sender, + sender, amount / pow( 10, 6 From eb4b9e08ff49334d82d4f578d3684c1ba963ece7 Mon Sep 17 00:00:00 2001 From: omkar Date: Mon, 24 Jan 2022 15:51:32 -0500 Subject: [PATCH 42/45] Terraswap pool reserves models --- .../terra_dbt__terraswap_pool_reserves.sql | 42 ++++++++++++++ .../silver_terra__terraswap_pool_reserves.sql | 27 +++++++++ .../silver_terra__terraswap_pool_reserves.yml | 46 +++++++++++++++ .../terra/swap/terraswap__pool_reserves.sql | 21 +++++++ .../terra/swap/terraswap__pool_reserves.yml | 56 +++++++++++++++++++ 5 files changed, 192 insertions(+) create mode 100644 models/terra/dbt/terra_dbt__terraswap_pool_reserves.sql create mode 100644 models/terra/silver/silver_terra__terraswap_pool_reserves.sql create mode 100644 models/terra/silver/silver_terra__terraswap_pool_reserves.yml create mode 100644 models/terra/swap/terraswap__pool_reserves.sql create mode 100644 models/terra/swap/terraswap__pool_reserves.yml diff --git a/models/terra/dbt/terra_dbt__terraswap_pool_reserves.sql b/models/terra/dbt/terra_dbt__terraswap_pool_reserves.sql new file mode 100644 index 00000000..e5e2b6dd --- /dev/null +++ b/models/terra/dbt/terra_dbt__terraswap_pool_reserves.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', chain_id, block_id, contract_address)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], + tags = ['snowflake', 'terra', 'terraswap', 'pool_reserves'] +) }} + +select + ( + record_metadata :CreateTime :: INT / 1000 + ) :: TIMESTAMP AS system_created_at, + r.value:chain_id :: STRING as chain_id, + r.value:block_id :: INT as block_id, + r.value:blockchain :: STRING as blockchain, + r.value:block_timestamp :: TIMESTAMP as block_timestamp, + r.value:contract_address :: STRING as contract_address, + r.value:value_obj:total_share :: INTEGER as total_share, + r.value:value_obj:asset_1 :: STRING as token_0_currency, + r.value:value_obj:asset_1_value :: INTEGER as token_0_amount, + r.value:value_obj:asset_2 :: STRING as token_1_currency, + r.value:value_obj:asset_2_value :: INTEGER as token_1_amount +from + {{ source( + 'bronze', + 'prod_terra_sink_645110886' + ) }}, + lateral flatten(input => record_content:results) as r +where + record_content:model.name = 'terra-5_terraswap_pool_reserves' + and record_content:model.run_id = 'v2022.01.25.0' +{% if is_incremental() %} +AND ( + record_metadata :CreateTime :: INT / 1000 +) :: TIMESTAMP :: DATE >= ( + SELECT + DATEADD('day', -1, MAX(system_created_at :: DATE)) + FROM + {{ this }} +) +{% endif %} +order by block_id asc \ No newline at end of file diff --git a/models/terra/silver/silver_terra__terraswap_pool_reserves.sql b/models/terra/silver/silver_terra__terraswap_pool_reserves.sql new file mode 100644 index 00000000..7cb50e90 --- /dev/null +++ b/models/terra/silver/silver_terra__terraswap_pool_reserves.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', chain_id, block_id, contract_address)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], + tags = ['snowflake', 'terra_silver', 'terraswap', 'pool_reserves'] +) }} + +SELECT + * +FROM + {{ ref('terra_dbt__terraswap_pool_reserves') }} +WHERE + 1 = 1 + +{% if is_incremental() %} +AND system_created_at :: DATE >= ( + SELECT + DATEADD('day', -1, MAX(system_created_at :: DATE)) + FROM + {{ this }} AS msgs +) +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY block_id, contract_address +ORDER BY + system_created_at DESC)) = 1 diff --git a/models/terra/silver/silver_terra__terraswap_pool_reserves.yml b/models/terra/silver/silver_terra__terraswap_pool_reserves.yml new file mode 100644 index 00000000..6fbc8a21 --- /dev/null +++ b/models/terra/silver/silver_terra__terraswap_pool_reserves.yml @@ -0,0 +1,46 @@ +version: 2 +models: + - name: silver_terra__terraswap_pool_reserves + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - CHAIN_ID + - CONTRACT_ADDRESS + columns: + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCK_ID + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: CHAIN_ID + tests: + - not_null + - name: CONTRACT_ADDRESS + tests: + - not_null + - name: TOTAL_SHARE + tests: + - not_null + - name: TOKEN_0_CURRENCY + tests: + - not_null + - name: TOKEN_0_AMOUNT + tests: + - not_null + - name: SYSTEM_CREATED_AT + tests: + - not_null + - name: TOKEN_1_CURRENCY + tests: + - not_null + - name: TOKEN_1_AMOUNT + tests: + - not_null diff --git a/models/terra/swap/terraswap__pool_reserves.sql b/models/terra/swap/terraswap__pool_reserves.sql new file mode 100644 index 00000000..1e32aefe --- /dev/null +++ b/models/terra/swap/terraswap__pool_reserves.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', chain_id, block_id, contract_address)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], + tags = ['snowflake', 'terraswap', 'terra', 'pool_reserves'] +) }} + +SELECT + 'terra' as blockchain, + chain_id, + block_id, + block_timestamp, + contract_address, + total_share / pow(10,6) AS total_share, + token_0_currency, + token_0_amount / pow(10,6) AS token_0_amount, + token_1_currency, + token_1_amount / pow(10,6) AS token_1_amount +FROM + {{ ref('silver_terra__terraswap_pool_reserves') }} diff --git a/models/terra/swap/terraswap__pool_reserves.yml b/models/terra/swap/terraswap__pool_reserves.yml new file mode 100644 index 00000000..27f1f652 --- /dev/null +++ b/models/terra/swap/terraswap__pool_reserves.yml @@ -0,0 +1,56 @@ +version: 2 +models: + - name: terraswap__pool_reserves + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - CHAIN_ID + - CONTRACT_ADDRESS + columns: + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCK_ID + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: CHAIN_ID + tests: + - not_null + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: terra[0-9a-z]{39,39} + - name: TOTAL_SHARE + tests: + - not_null + - name: TOKEN_0_CURRENCY + tests: + - not_null + - name: TOKEN_0_AMOUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - decimal + - float + - name: SYSTEM_CREATED_AT + tests: + - not_null + - name: TOKEN_1_CURRENCY + tests: + - not_null + - name: TOKEN_1_AMOUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - decimal + - float From 43313ed62042bb9fa9744bdf2d27bc62a0961e4d Mon Sep 17 00:00:00 2001 From: Jessica Huhnke <91915469+jhuhnke@users.noreply.github.com> Date: Thu, 27 Jan 2022 14:59:41 -0600 Subject: [PATCH 43/45] An 573 solana events (#270) * Added tables that could not be reviewed back into branch * Reduced recency tests from 3 days to 1 day * events + votes + update to transfers * Addressed comments in PR * Removed gold tests, added incrementals for instructions/inner instructions in silver_events * Added pre and post token balances to transfers - needed when token transfered is not solana * Changed ORDER BY statement in the silver events table --- .../solana/bronze/bronze_solana__blocks.sql | 4 +- .../bronze/bronze_solana__transactions.sql | 4 +- models/solana/gold/solana__blocks.sql | 4 +- models/solana/gold/solana__blocks.yml | 44 ---- models/solana/gold/solana__events.sql | 21 ++ models/solana/gold/solana__transactions.sql | 2 +- models/solana/gold/solana__transactions.yml | 50 ---- models/solana/gold/solana__transfers.sql | 31 ++- models/solana/gold/solana__transfers.yml | 46 ---- models/solana/gold/solana__votes.sql | 23 ++ .../solana/silver/silver_solana__blocks.sql | 12 +- .../solana/silver/silver_solana__blocks.yml | 9 +- .../solana/silver/silver_solana__events.sql | 97 ++++---- .../solana/silver/silver_solana__events.yml | 17 +- models/solana/silver/silver_solana__swaps.sql | 231 ------------------ models/solana/silver/silver_solana__swaps.yml | 48 ---- .../silver/silver_solana__transactions.sql | 6 +- .../silver/silver_solana__transactions.yml | 8 +- models/solana/silver/silver_solana__votes.sql | 10 +- models/solana/silver/silver_solana__votes.yml | 4 +- .../solana/solana_dbt__inner_instructions.sql | 30 +++ models/solana/solana_dbt__instructions.sql | 30 +++ 22 files changed, 205 insertions(+), 526 deletions(-) delete mode 100644 models/solana/gold/solana__blocks.yml create mode 100644 models/solana/gold/solana__events.sql delete mode 100644 models/solana/gold/solana__transactions.yml delete mode 100644 models/solana/gold/solana__transfers.yml create mode 100644 models/solana/gold/solana__votes.sql delete mode 100644 models/solana/silver/silver_solana__swaps.sql delete mode 100644 models/solana/silver/silver_solana__swaps.yml create mode 100644 models/solana/solana_dbt__inner_instructions.sql create mode 100644 models/solana/solana_dbt__instructions.sql diff --git a/models/solana/bronze/bronze_solana__blocks.sql b/models/solana/bronze/bronze_solana__blocks.sql index b5b3f8a3..a9c7c6b0 100644 --- a/models/solana/bronze/bronze_solana__blocks.sql +++ b/models/solana/bronze/bronze_solana__blocks.sql @@ -5,8 +5,8 @@ SELECT record_id, - offset_id, - block_id, + offset_id AS block_id, + block_id AS offset_id, block_timestamp, network, chain_id, diff --git a/models/solana/bronze/bronze_solana__transactions.sql b/models/solana/bronze/bronze_solana__transactions.sql index a0eda6f6..49f56a60 100644 --- a/models/solana/bronze/bronze_solana__transactions.sql +++ b/models/solana/bronze/bronze_solana__transactions.sql @@ -7,8 +7,8 @@ SELECT record_id, tx_id, tx_block_index, - offset_id, - block_id, + offset_id AS block_id, + block_id AS offset_id, block_timestamp, network, chain_id, diff --git a/models/solana/gold/solana__blocks.sql b/models/solana/gold/solana__blocks.sql index 164ccf68..e6176693 100644 --- a/models/solana/gold/solana__blocks.sql +++ b/models/solana/gold/solana__blocks.sql @@ -11,9 +11,9 @@ SELECT tx_count, block_height, block_time, - blockhash, + block_hash, previous_block_id, - previous_blockhash, + previous_block_hash, ingested_at FROM {{ ref('silver_solana__blocks') }} \ No newline at end of file diff --git a/models/solana/gold/solana__blocks.yml b/models/solana/gold/solana__blocks.yml deleted file mode 100644 index 2e019105..00000000 --- a/models/solana/gold/solana__blocks.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: 2 -models: - - name: solana__blocks - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - BLOCK_ID - columns: - - name: BLOCK_ID - tests: - - not_null - - name: BLOCK_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 3 - - name: NETWORK - tests: - - not_null - - name: BLOCKCHAIN - tests: - - not_null - - name: TX_COUNT - tests: - - not_null - - name: BLOCK_HEIGHT - tests: - - not_null - - name: BLOCK_TIME - tests: - - not_null - - name: BLOCKHASH - tests: - - not_null - - name: PREVIOUS_BLOCK_ID - tests: - - not_null - - name: PREVIOUS_BLOCKHASH - tests: - - not_null - - name: INGESTED_AT - tests: - - not_null \ No newline at end of file diff --git a/models/solana/gold/solana__events.sql b/models/solana/gold/solana__events.sql new file mode 100644 index 00000000..ca127363 --- /dev/null +++ b/models/solana/gold/solana__events.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'view', + tags = ['snowflake', 'solana', 'gold_solana', 'solana_events'] +) }} + +SELECT + block_timestamp, + block_id, + blockchain, + recent_block_hash, + tx_id, + succeeded, + preTokenBalances, + postTokenBalances, + index, + event_type, + instruction, + inner_instruction, + ingested_at + +FROM {{ ref('silver_solana__events') }} diff --git a/models/solana/gold/solana__transactions.sql b/models/solana/gold/solana__transactions.sql index 3aa153b3..1490e8fa 100644 --- a/models/solana/gold/solana__transactions.sql +++ b/models/solana/gold/solana__transactions.sql @@ -6,7 +6,7 @@ SELECT block_timestamp, block_id, - recent_blockhash, + recent_block_hash, tx_id, pre_mint, post_mint, diff --git a/models/solana/gold/solana__transactions.yml b/models/solana/gold/solana__transactions.yml deleted file mode 100644 index cb68905c..00000000 --- a/models/solana/gold/solana__transactions.yml +++ /dev/null @@ -1,50 +0,0 @@ -version: 2 -models: - - name: solana__transactions - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - BLOCK_ID - - TX_ID - columns: - - name: BLOCK_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 3 - - name: BLOCK_ID - tests: - - not_null - - name: RECENT_BLOCKHASH - tests: - - not_null - - name: TX_ID - tests: - - not_null - - name: PRE_MINT - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: POST_MINT - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: TX_FROM_ADDRESS - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: TX_TO_ADDRESS - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: FEE - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: PROGRAM_ID - tests: - - not_null - - name: INGESTED_AT - tests: - - not_null \ No newline at end of file diff --git a/models/solana/gold/solana__transfers.sql b/models/solana/gold/solana__transfers.sql index c18a3410..d9b83726 100644 --- a/models/solana/gold/solana__transfers.sql +++ b/models/solana/gold/solana__transfers.sql @@ -5,21 +5,30 @@ SELECT block_timestamp, - block_id, + block_id, blockchain, - recent_blockhash, + recent_block_hash, tx_id, - ei.value:parsed:info:authority :: STRING AS tx_from_address, - ei.value:parsed:info:destination :: STRING AS tx_to_address, - ei.value:parsed:info:amount/POW(10,9) :: INTEGER AS solana_amount, + event_type, + preTokenBalances, + postTokenBalances, + instruction:parsed:info:destination :: STRING AS destination, + instruction:parsed:info:source :: STRING AS source, + instruction:parsed:info:authority :: STRING AS authority, + CASE + WHEN event_type = 'transferChecked' THEN instruction:parsed:info:tokenAmount:amount/POW(10,6) + WHEN event_type = 'transfer' AND instruction:programId :: STRING = '11111111111111111111111111111111' THEN instruction:parsed:info:lamports/POW(10,9) + ELSE instruction:parsed:info:amount/POW(10,6) END + AS amount, + instruction:programId :: STRING AS program_id, succeeded, - ei.value:programId :: STRING AS program_id, ingested_at, CASE WHEN program_id <> 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' THEN TRUE ELSE FALSE END AS transfer_tx_flag -FROM {{ ref('silver_solana__events') }}, +FROM {{ ref('silver_solana__events') }} -LATERAL FLATTEN ( - input => event_info ) ei +WHERE event_type = 'transfer' +OR event_type = 'transferChecked' -WHERE ei.value:parsed:type :: STRING = 'transfer' -AND ei.value:program :: STRING = 'spl-token' +qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +ORDER BY + ingested_at DESC)) = 1 diff --git a/models/solana/gold/solana__transfers.yml b/models/solana/gold/solana__transfers.yml deleted file mode 100644 index e5bb3d57..00000000 --- a/models/solana/gold/solana__transfers.yml +++ /dev/null @@ -1,46 +0,0 @@ -version: 2 -models: - - name: solana__transfers - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - BLOCK_ID - - TX_ID - columns: - - name: BLOCK_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 3 - - name: BLOCK_ID - tests: - - not_null - - name: BLOCKCHAIN - tests: - - not_null - - name: RECENT_BLOCKHASH - tests: - - not_null - - name: TX_ID - tests: - - not_null - - name: TX_FROM_ADDRESS - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: TX_TO_ADDRESS - tests: - - not_null - - name: SOLANA_AMOUNT - tests: - - not_null - - name: SUCCEEDED - tests: - - not_null - - name: PROGRAM_ID - tests: - - not_null - - name: INGESTED_AT - tests: - - not_null \ No newline at end of file diff --git a/models/solana/gold/solana__votes.sql b/models/solana/gold/solana__votes.sql new file mode 100644 index 00000000..f3d22f5d --- /dev/null +++ b/models/solana/gold/solana__votes.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view', + tags = ['snowflake', 'solana', 'gold_solana', 'solana_events', 'solana_votes'] +) }} + +SELECT + block_timestamp, + block_id, + blockchain, + recent_block_hash, + tx_id, + succeeded, + program_type, + program_id, + vote_account, + vote_authority, + ingested_at, + transfer_tx_flag +FROM {{ ref('silver_solana__votes') }} + +qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +ORDER BY + ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/silver/silver_solana__blocks.sql b/models/solana/silver/silver_solana__blocks.sql index c59f7fab..08dec13c 100644 --- a/models/solana/silver/silver_solana__blocks.sql +++ b/models/solana/silver/silver_solana__blocks.sql @@ -12,10 +12,8 @@ WITH base_tables AS ( FROM {{ ref('bronze_solana__blocks') }} - WHERE - 1 = 1 {% if is_incremental() %} - AND ingested_at >= ( + WHERE ingested_at >= ( SELECT MAX( ingested_at @@ -27,21 +25,19 @@ WITH base_tables AS ( ) SELECT - offset_id :: INTEGER AS block_id, + block_id :: INTEGER AS block_id, block_timestamp :: TIMESTAMP AS block_timestamp, network :: STRING AS network, chain_id :: STRING AS blockchain, tx_count :: INTEGER AS tx_count, header :blockHeight :: INTEGER AS block_height, header :blockTime :: INTEGER AS block_time, - header :blockhash :: VARCHAR AS blockhash, + header :blockhash :: VARCHAR AS block_hash, header :parentSlot :: INTEGER AS previous_block_id, - header :previousBlockhash :: VARCHAR AS previous_blockhash, + header :previousBlockhash :: VARCHAR AS previous_block_hash, ingested_at :: TIMESTAMP AS ingested_at FROM base_tables -WHERE - 1 = 1 qualify(ROW_NUMBER() over(PARTITION BY block_id ORDER BY diff --git a/models/solana/silver/silver_solana__blocks.yml b/models/solana/silver/silver_solana__blocks.yml index 679ea33a..2fe6fdc3 100644 --- a/models/solana/silver/silver_solana__blocks.yml +++ b/models/solana/silver/silver_solana__blocks.yml @@ -8,13 +8,14 @@ models: columns: - name: BLOCK_ID tests: - - not_null + - not_null: + where: "block_timestamp > current_date - 2" - name: BLOCK_TIMESTAMP tests: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 3 + interval: 1 - name: NETWORK tests: - not_null @@ -30,13 +31,13 @@ models: - name: BLOCK_TIME tests: - not_null - - name: BLOCKHASH + - name: BLOCK_HASH tests: - not_null - name: PREVIOUS_BLOCK_ID tests: - not_null - - name: PREVIOUS_BLOCKHASH + - name: PREVIOUS_BLOCK_HASH tests: - not_null - name: INGESTED_AT diff --git a/models/solana/silver/silver_solana__events.sql b/models/solana/silver/silver_solana__events.sql index 5c04ca08..4bb9c926 100644 --- a/models/solana/silver/silver_solana__events.sql +++ b/models/solana/silver/silver_solana__events.sql @@ -1,62 +1,59 @@ {{ config( materialized = 'incremental', - unique_key = "CONCAT_WS('-', block_id, tx_id)", + unique_key = "CONCAT_WS('-', block_id, tx_id, index)", incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE'], tags = ['snowflake', 'solana', 'silver_solana', 'solana_events'] ) }} -WITH base_table as ( - SELECT - block_timestamp :: TIMESTAMP AS block_timestamp, - offset_id :: INTEGER AS block_id, - chain_id :: STRING AS blockchain, - tx :transaction:message:recentBlockhash :: STRING AS recent_blockhash, - tx_id :: STRING AS tx_id, - tx :meta:preTokenBalances[0]:owner :: STRING AS tx_from_address, - tx :meta:postTokenBalances[0]:owner :: STRING AS tx_to_address, - CASE WHEN tx :meta:status:Err IS NULL THEN TRUE ELSE FALSE END AS succeeded, - tx :meta:innerInstructions[0]:instructions :: ARRAY AS event_meta, - tx :meta:postTokenBalances :: ARRAY AS postTokenBalances, - tx :transaction:message:instructions :: ARRAY AS event_info, - ingested_at :: TIMESTAMP AS ingested_at, - CASE WHEN len(tx :meta:postTokenBalances[0]) > 0 AND len(tx :meta:preTokenBalances[0]) > 0 AND SUCCEEDED = TRUE THEN TRUE ELSE FALSE END AS transfer_tx_flag - FROM {{ ref('bronze_solana__transactions') }} +SELECT + t.block_timestamp :: TIMESTAMP AS block_timestamp, + t.block_id :: INTEGER AS block_id, + t.chain_id :: STRING AS blockchain, + t.tx :transaction:message:recentBlockhash :: STRING AS recent_block_hash, + t.tx_id :: STRING AS tx_id, + CASE WHEN t.tx :meta:status:Err IS NULL THEN TRUE ELSE FALSE END AS succeeded, + t.tx :meta:preTokenBalances :: ARRAY AS preTokenBalances, + t.tx :meta:postTokenBalances :: ARRAY AS postTokenBalances, + i.index :: INTEGER AS index, + i.event_type :: STRING AS event_type, + i.value AS instruction, + ii.value as inner_instruction, + t.ingested_at :: TIMESTAMP AS ingested_at +FROM {{ ref('solana_dbt__instructions') }} i - WHERE - 1 = 1 - AND tx :transaction:message:instructions[0]:parsed:type :: STRING IS NOT NULL - AND tx :transaction:message:instructions[0]:programId :: STRING <> 'Vote111111111111111111111111111111111111111' - - {% if is_incremental() %} - AND ingested_at >= ( - SELECT - MAX( - ingested_at - ) - FROM - {{ this }} - ) - {% endif %} -) +LEFT OUTER JOIN {{ ref('solana_dbt__inner_instructions') }} ii +ON ii.block_id = i.block_id +AND ii.tx_id = i.tx_id +AND ii.mapped_event_index = i.index -SELECT - block_timestamp, - block_id, - blockchain, - recent_blockhash, - tx_id, - tx_from_address, - tx_to_address, - succeeded, - event_meta, - postTokenBalances, - event_info, - ingested_at, - transfer_tx_flag -FROM base_table +{% if is_incremental() %} + AND ii.ingested_at >= ( + SELECT + MAX( + ingested_at + ) + FROM + {{ this }} + ) + {% endif %} -qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id +LEFT OUTER JOIN {{ ref('bronze_solana__transactions') }} t +ON t.block_id = i.block_id +AND t.tx_id = i.tx_id + + {% if is_incremental() %} + AND t.ingested_at >= ( + SELECT + MAX( + ingested_at + ) + FROM + {{ this }} + ) + {% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY t.block_id, t.tx_id, i.index ORDER BY - ingested_at DESC)) = 1 \ No newline at end of file + t.ingested_at DESC)) = 1 \ No newline at end of file diff --git a/models/solana/silver/silver_solana__events.yml b/models/solana/silver/silver_solana__events.yml index 97aca407..6d86047f 100644 --- a/models/solana/silver/silver_solana__events.yml +++ b/models/solana/silver/silver_solana__events.yml @@ -6,40 +6,29 @@ models: combination_of_columns: - BLOCK_ID - TX_ID + - INDEX columns: - name: BLOCK_TIMESTAMP tests: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 3 + interval: 1 - name: BLOCK_ID tests: - not_null - name: BLOCKCHAIN tests: - not_null - - name: RECENT_BLOCKHASH + - name: RECENT_BLOCK_HASH tests: - not_null - name: TX_ID tests: - not_null - - name: TX_FROM_ADDRESS - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - - name: TX_TO_ADDRESS - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - name: SUCCEEDED tests: - not_null - - name: EVENT_INFO - tests: - - not_null: - where: TRANSFER_TX_FLAG = TRUE - name: INGESTED_AT tests: - not_null \ No newline at end of file diff --git a/models/solana/silver/silver_solana__swaps.sql b/models/solana/silver/silver_solana__swaps.sql deleted file mode 100644 index d4ba567f..00000000 --- a/models/solana/silver/silver_solana__swaps.sql +++ /dev/null @@ -1,231 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "CONCAT_WS('-', block_id, tx_id)", - incremental_strategy = 'delete+insert', - cluster_by = ['block_timestamp::DATE'], - tags = ['snowflake', 'solana', 'silver_solana', 'solana_events', 'solana_swaps'] -) }} - -WITH base_table AS ( - SELECT - block_timestamp, - block_id, - blockchain, - recent_blockhash, - tx_id, - event_info[0]:parsed:info:source :: STRING AS sender, - - CASE WHEN event_info[2]:programId :: STRING IN ('9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin', '675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8') -- SERUM DEX || Raydium Liquidity Pool V4 - THEN event_info[1]:parsed:info:mint :: STRING - WHEN event_info[3]:programId :: STRING = 'SSwpkEEcbUqx4vtoEByFjSkhKdCT862DNVb52nZg1UZ' -- SABER STABLE SWAP - THEN event_meta[1]:parsed:info:destination :: STRING - ELSE COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - ) END AS token_sent_account, - - CASE WHEN event_info[3]:programId :: STRING IN ('SSwpkEEcbUqx4vtoEByFjSkhKdCT862DNVb52nZg1UZ', 'SSwpMgqNDsyV7mAgN9ady4bDVu5ySjmmXejXvy2vLt1') -- SABER STABLE SWAP || STEP FINANCE - THEN postTokenBalances[1]:mint :: STRING - WHEN event_info[4]:programId :: STRING = '82yxjeMsvaURa4MbZZ7WZZHfobirZYkH1zF8fmeGtyaQ' -- ORCA AQUAFARM - THEN postTokenBalances[3]:mint :: STRING - WHEN event_info[2]:programId :: STRING IN ('9W959DqEETiGZocYWCQPaJ6sBmUzgfxXfqGeTEdp3aQP', '9W959DqEETiGZocYWCQPaJ6sBmUzgfxXfqGeTEdp3aQP')-- ORCA TOKEN SWAP - THEN postTokenBalances[0]:mint :: STRING - WHEN event_info[2]:programId :: STRING IN ('2nAAsYdXF3eTQzaeUQS3fr4o782dDg8L28mX39Wr5j8N', '31ARfyxZg6fr1J9hVs1NqBWkdeYeCKipuY71bMovNpy9') -- UNKNOWN PROGRAM - THEN postTokenBalances[1]:mint :: STRING - ELSE postTokenBalances[0]:mint :: STRING END AS t0_sent, - - CASE WHEN event_info[2]:programId :: STRING = 'LendZqTs7gn5CTSJU1jWKhKuVpjJGom45nnwPb2AMTi' - THEN event_meta[1]:parsed:info:amount :: INTEGER - ELSE event_meta[0]:parsed:info:amount :: INTEGER END AS amount_sent, - - CASE WHEN event_info[2]:programId :: STRING IN ('9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin', '675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8') -- SERUM DEX || Raydium Liquidity Pool V4 - THEN COALESCE( - event_meta[0]:parsed:info:destination :: STRING, - event_meta[1]:parsed:info:destination :: STRING - ) - WHEN event_info[3]:programId :: STRING IN ('9W959DqEETiGZocYWCQPaJ6sBmUzgfxXfqGeTEdp3aQP' , 'SWiMDJYFUGj6cPrQ6QYYYWZtvXQdRChSVAygDZDsCHC')-- ETH WORMHOLE - THEN postTokenBalances[1]:mint :: STRING - WHEN event_info[4]:programId :: STRING = 'MERLuDFBMmsHnsBPZw2sDQZHvXFMwp8EdjudcU2HKky' -- Mercurial Stable Swap - THEN postTokenBalances[3]:mint :: STRING - WHEN event_info[3]:programId :: STRING = 'MERLuDFBMmsHnsBPZw2sDQZHvXFMwp8EdjudcU2HKky' -- Mercurial Stable Swap - THEN postTokenBalances[2]:mint :: STRING - WHEN event_info[4]:programId :: STRING = '82yxjeMsvaURa4MbZZ7WZZHfobirZYkH1zF8fmeGtyaQ' -- ORCA AQUAFARM - THEN postTokenBalances[0]:mint :: STRING - WHEN event_info[2]:programId :: STRING IN ('9W959DqEETiGZocYWCQPaJ6sBmUzgfxXfqGeTEdp3aQP','SSwpMgqNDsyV7mAgN9ady4bDVu5ySjmmXejXvy2vLt1', 'DmzAmomATKpNp2rCBfYLS7CSwQqeQTsgRYJA1oSSAJaP', '31ARfyxZg6fr1J9hVs1NqBWkdeYeCKipuY71bMovNpy9', 'Bt2WPMmbwHPk36i4CRucNDyLcmoGdC7xEdrVuxgJaNE6', 'PSwapMdSai8tjrEXcxFeQth87xC4rRsa4VA5mhGhXkP', 'SWiMDJYFUGj6cPrQ6QYYYWZtvXQdRChSVAygDZDsCHC') -- ORCA TOKEN SWAP V2 || STEP FINANCE || UNKNOWN PROGRAM - THEN postTokenBalances[1]:mint :: STRING - WHEN event_info[3]:programId :: STRING IN ('DjVE6JNiYqPL2QXyCUUh8rNjHrbz9hXHNYt99MQ59qw1', '2rcQtPTFZd3UWzEWwa86TtYn3ePi4U2EDaZxf3BKSvfd') -- ORCA TOKEN SWAP V1 || UNKNOWN - THEN postTokenBalances[1]:mint :: STRING - WHEN event_info[2]:programId :: STRING = '2nAAsYdXF3eTQzaeUQS3fr4o782dDg8L28mX39Wr5j8N' -- UNKNOWN PROGRAM - THEN postTokenBalances[0]:mint :: STRING - WHEN event_info[2]:programId :: STRING = 'SSwpkEEcbUqx4vtoEByFjSkhKdCT862DNVb52nZg1UZ' - THEN COALESCE ( - event_info[1]:parsed:info:mint :: STRING, - postTokenBalances[3]:mint :: STRING - ) - WHEN event_info[2]:programId :: STRING = 'LendZqTs7gn5CTSJU1jWKhKuVpjJGom45nnwPb2AMTi' - THEN postTokenBalances[4]:mint :: STRING - ELSE event_info[1]:parsed:info:mint :: STRING END AS token_received, - - CASE WHEN event_info[2]:programId :: STRING IN ('SSwpMgqNDsyV7mAgN9ady4bDVu5ySjmmXejXvy2vLt1', 'PSwapMdSai8tjrEXcxFeQth87xC4rRsa4VA5mhGhXkP') - THEN event_meta[1]:parsed:info:amount :: INTEGER - WHEN event_info[4]:programId :: STRING = 'MERLuDFBMmsHnsBPZw2sDQZHvXFMwp8EdjudcU2HKky' -- Mercurial Stable Swap - THEN event_info[3]:parsed:info:amount :: INTEGER - WHEN event_info[3]:programId :: STRING = 'SWiMDJYFUGj6cPrQ6QYYYWZtvXQdRChSVAygDZDsCHC' - THEN event_meta[1]:parsed:info:amount :: INTEGER - WHEN event_meta[2]:parsed:type :: STRING <> 'burn' - THEN COALESCE( - event_meta[2]:parsed:info:amount :: INTEGER, - event_meta[1]:parsed:info:amount :: INTEGER - ) - ELSE event_meta[1]:parsed:info:amount :: INTEGER - END AS amount_received, - - succeeded, - ingested_at - -FROM "FLIPSIDE_DEV_DB"."SILVER_SOLANA"."EVENTS" - -WHERE event_info[1]:program = 'spl-token' -AND event_info[1]:programId = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' -AND event_meta[0]:parsed:info:amount :: INTEGER IS NOT NULL -- IF THESE ARE NULL, THEN NFT MINT -AND event_meta[1]:parsed:info:amount :: INTEGER IS NOT NULL -AND token_received IS NOT NULL - -{% if is_incremental() %} - AND ingested_at >= ( - SELECT - MAX( - ingested_at - ) - FROM - {{ this }} - ) -{% endif %} -), - -token_account AS ( - SELECT - tx:transaction:message:instructions[1]:program :: STRING as program, - tx:transaction:message:instructions[1]:parsed:info:wallet :: STRING as owner_address, - tx:transaction:message:instructions[1]:parsed:info:account :: STRING as associated_token_address, - tx:transaction:message:instructions[1]:parsed:info:mint :: STRING as token_address, - ingested_at - FROM {{ ref('bronze_solana__transactions') }} - WHERE program = 'spl-associated-token-account' - AND - 1 = 1 - {% if is_incremental() %} - AND ingested_at >= ( - SELECT - MAX( - ingested_at - ) - FROM - {{ this }} - ) -{% endif %} -), - -token_address_table AS ( - SELECT - block_timestamp, - block_id, - blockchain, - recent_blockhash, - tx_id, - sender, - token_sent_account, - t0_sent, - t.token_address AS token_sent2, - amount_sent, - token_received, - amount_received, - succeeded, - b.ingested_at - - FROM base_table b - - LEFT OUTER JOIN token_account t - ON token_sent_account = t.associated_token_address -) - -SELECT - block_timestamp, - block_id, - blockchain, - recent_blockhash, - tx_id, - sender, - - COALESCE ( - token_sent2, - t0_sent, - token_sent_account) AS token_sent, - - CASE WHEN token_sent_account IN ('48F1neXh5bGgKr8G6CM6tFZkaC51UgtVb5pqGLC27Doi', - '2Zkvs84qBpMvWFSh9MFek2ZPAqAEjfufrs7Sbu42PbzBRTSv5Qx1bTWTGsbaSsuAPe4yx6ku3KmwVaGRbdANytF6', - 'DD6oh3HRCvMzqHkGeUW3za4pLgWNPJdV6aNYW3gVjXXi', - 'DQyrAcCrDXQ7NeoqGgDCZwBvWDcYmFCjSb9JtteuvPpz', - '9wArqjNDHm43RJGSBSXuHygqJzEaW1xYA3XmmpU671kC', - '5sHXydo4z6p2x7Jqfx8T4FnPRM9S2y8wpgNG1pAZf9Cg', - 'ArwS8uBqFTKNgLAy7r3poSCujCazc4GiqBZguUzQPrKj', - '4aiKnDHFmNnLsopVsDyRBh8sbVohZYgdGzh3P9orpqNB', - '2YuVnXkSmW714ziXkfPfmvAnniBc1tHiAB7gYgRpx43a', - 'mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So', - 'StepAscQoEioFxxWGnh2sLBDFp9d8rvKz2Yp39iDpyT', - 'So11111111111111111111111111111111111111112', - 'DubwWZNWiNGMMeeQHPnMATNj77YZPZSAz2WVR5WjLJqz', - 'H7Qc9APCWWGDVxGD5fJHmLTmdEgT9GFatAKFNg6sHh8A', - '7dHbWXmci3dT8UFYWYZweBLXgycu7Y3iL6trKn1Y7ARj') - THEN amount_sent/POW(10,9) :: INTEGER - WHEN token_sent_account = '7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs' - THEN amount_sent/POW(10,8) :: INTEGER - WHEN token_sent IN ('48F1neXh5bGgKr8G6CM6tFZkaC51UgtVb5pqGLC27Doi', - '2Zkvs84qBpMvWFSh9MFek2ZPAqAEjfufrs7Sbu42PbzBRTSv5Qx1bTWTGsbaSsuAPe4yx6ku3KmwVaGRbdANytF6', - 'DD6oh3HRCvMzqHkGeUW3za4pLgWNPJdV6aNYW3gVjXXi', - 'DQyrAcCrDXQ7NeoqGgDCZwBvWDcYmFCjSb9JtteuvPpz', - '9wArqjNDHm43RJGSBSXuHygqJzEaW1xYA3XmmpU671kC', - '5sHXydo4z6p2x7Jqfx8T4FnPRM9S2y8wpgNG1pAZf9Cg', - 'ArwS8uBqFTKNgLAy7r3poSCujCazc4GiqBZguUzQPrKj', - '4aiKnDHFmNnLsopVsDyRBh8sbVohZYgdGzh3P9orpqNB', - '2YuVnXkSmW714ziXkfPfmvAnniBc1tHiAB7gYgRpx43a', - 'mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So', - 'StepAscQoEioFxxWGnh2sLBDFp9d8rvKz2Yp39iDpyT', - 'So11111111111111111111111111111111111111112', - 'DubwWZNWiNGMMeeQHPnMATNj77YZPZSAz2WVR5WjLJqz', - 'H7Qc9APCWWGDVxGD5fJHmLTmdEgT9GFatAKFNg6sHh8A', - '7dHbWXmci3dT8UFYWYZweBLXgycu7Y3iL6trKn1Y7ARj') - THEN amount_sent/POW(10,9) :: INTEGER - WHEN token_sent IN ('7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs', '9TE7ebz1dsFo1uQ2T4oYAKSm39Y6fWuHrd6Uk6XaiD16') - THEN amount_sent/POW(10,8) :: INTEGER - ELSE amount_sent/POW(10,6) :: INTEGER END AS amount_sent, - - token_received, - - CASE WHEN token_received IN ('48F1neXh5bGgKr8G6CM6tFZkaC51UgtVb5pqGLC27Doi', - '2Zkvs84qBpMvWFSh9MFek2ZPAqAEjfufrs7Sbu42PbzBRTSv5Qx1bTWTGsbaSsuAPe4yx6ku3KmwVaGRbdANytF6', - 'DD6oh3HRCvMzqHkGeUW3za4pLgWNPJdV6aNYW3gVjXXi', - 'DQyrAcCrDXQ7NeoqGgDCZwBvWDcYmFCjSb9JtteuvPpz', - '9wArqjNDHm43RJGSBSXuHygqJzEaW1xYA3XmmpU671kC', - '5sHXydo4z6p2x7Jqfx8T4FnPRM9S2y8wpgNG1pAZf9Cg', - 'ArwS8uBqFTKNgLAy7r3poSCujCazc4GiqBZguUzQPrKj', - '4aiKnDHFmNnLsopVsDyRBh8sbVohZYgdGzh3P9orpqNB', - '2YuVnXkSmW714ziXkfPfmvAnniBc1tHiAB7gYgRpx43a', - 'mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So', - 'StepAscQoEioFxxWGnh2sLBDFp9d8rvKz2Yp39iDpyT', - 'So11111111111111111111111111111111111111112', - 'AGFEad2et2ZJif9jaGpdMixQqvW5i81aBdvKe7PHNfz3', - 'DubwWZNWiNGMMeeQHPnMATNj77YZPZSAz2WVR5WjLJqz') - THEN amount_received/POW(10,9) :: INTEGER - WHEN token_received IN ('7vfCXTUXx5WJV5JADk17DUJ4ksgau7utNKj4b963voxs', '9TE7ebz1dsFo1uQ2T4oYAKSm39Y6fWuHrd6Uk6XaiD16') - THEN amount_received/POW(10,8) :: INTEGER - ELSE amount_received/POW(10,6) :: INTEGER END AS amount_received, - - succeeded, - ingested_at - -FROM token_address_table - -qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id -ORDER BY - ingested_at DESC)) = 1 diff --git a/models/solana/silver/silver_solana__swaps.yml b/models/solana/silver/silver_solana__swaps.yml deleted file mode 100644 index 1b57d2df..00000000 --- a/models/solana/silver/silver_solana__swaps.yml +++ /dev/null @@ -1,48 +0,0 @@ -version: 2 -models: - - name: silver_solana__swaps - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - BLOCK_ID - - TX_ID - columns: - - name: BLOCK_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 3 - - name: BLOCK_ID - tests: - - not_null - - name: BLOCKCHAIN - tests: - - not_null - - name: RECENT_BLOCKHASH - tests: - - not_null - - name: TX_ID - tests: - - not_null - - name: SENDER - tests: - - not_null - - name: TOKEN_SENT - tests: - - not_null - - name: AMOUNT_SENT - tests: - - not_null - - name: TOKEN_RECEIVED - tests: - - not_null - - name: AMOUNT_RECEIVED - tests: - - not_null - - name: SUCCEEDED - tests: - - not_null - - name: INGESTED_AT - tests: - - not_null \ No newline at end of file diff --git a/models/solana/silver/silver_solana__transactions.sql b/models/solana/silver/silver_solana__transactions.sql index ab29b84b..fe5b2fe7 100644 --- a/models/solana/silver/silver_solana__transactions.sql +++ b/models/solana/silver/silver_solana__transactions.sql @@ -9,8 +9,8 @@ WITH base_table as ( SELECT block_timestamp :: TIMESTAMP AS block_timestamp, - offset_id :: INTEGER AS block_id, - tx :transaction:message:recentBlockhash :: STRING AS recent_blockhash, + block_id :: INTEGER AS block_id, + tx :transaction:message:recentBlockhash :: STRING AS recent_block_hash, tx_id :: STRING AS tx_id, tx :meta:preTokenBalances[0]:mint :: STRING as pre_mint, tx :meta:postTokenBalances[0]:mint :: STRING as post_mint, @@ -54,7 +54,7 @@ AND ingested_at >= ( SELECT block_timestamp, block_id, - recent_blockhash, + recent_block_hash, tx_id, pre_mint, post_mint, diff --git a/models/solana/silver/silver_solana__transactions.yml b/models/solana/silver/silver_solana__transactions.yml index b2c0e7df..1fc8e6c6 100644 --- a/models/solana/silver/silver_solana__transactions.yml +++ b/models/solana/silver/silver_solana__transactions.yml @@ -12,11 +12,11 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 3 + interval: 1 - name: BLOCK_ID tests: - not_null - - name: RECENT_BLOCKHASH + - name: RECENT_BLOCK_HASH tests: - not_null - name: TX_ID @@ -26,18 +26,22 @@ models: tests: - not_null: where: TRANSFER_TX_FLAG = TRUE + enabled: False # null values currently expected for some txs - name: POST_MINT tests: - not_null: where: TRANSFER_TX_FLAG = TRUE + enabled: False # null values currently expected for some txs - name: TX_FROM_ADDRESS tests: - not_null: where: TRANSFER_TX_FLAG = TRUE + enabled: False # null values currently expected for some txs - name: TX_TO_ADDRESS tests: - not_null: where: TRANSFER_TX_FLAG = TRUE + enabled: False # null values currently expected for some txs - name: FEE tests: - not_null: diff --git a/models/solana/silver/silver_solana__votes.sql b/models/solana/silver/silver_solana__votes.sql index fc4ed059..413b5de2 100644 --- a/models/solana/silver/silver_solana__votes.sql +++ b/models/solana/silver/silver_solana__votes.sql @@ -9,9 +9,9 @@ WITH base_table as ( SELECT block_timestamp :: TIMESTAMP AS block_timestamp, - offset_id :: INTEGER AS block_id, + block_id :: INTEGER AS block_id, chain_id :: STRING AS blockchain, - tx :transaction:message:recentBlockhash :: STRING AS recent_blockhash, + tx :transaction:message:recentBlockhash :: STRING AS recent_block_hash, tx_id :: STRING AS tx_id, CASE WHEN tx :meta:status:Err IS NULL THEN TRUE ELSE FALSE END AS succeeded, tx :transaction:message:instructions[0]:program :: STRING AS program_type, @@ -23,9 +23,7 @@ WITH base_table as ( FROM {{ ref('bronze_solana__transactions') }} - WHERE - 1 = 1 - AND tx :transaction:message:instructions[0]:parsed:type :: STRING IS NOT NULL + WHERE tx :transaction:message:instructions[0]:parsed:type :: STRING IS NOT NULL AND tx :transaction:message:instructions[0]:programId :: STRING = 'Vote111111111111111111111111111111111111111' {% if is_incremental() %} @@ -44,7 +42,7 @@ SELECT block_timestamp, block_id, blockchain, - recent_blockhash, + recent_block_hash, tx_id, succeeded, program_type, diff --git a/models/solana/silver/silver_solana__votes.yml b/models/solana/silver/silver_solana__votes.yml index 4551ae59..f43a8696 100644 --- a/models/solana/silver/silver_solana__votes.yml +++ b/models/solana/silver/silver_solana__votes.yml @@ -12,14 +12,14 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 3 + interval: 1 - name: BLOCK_ID tests: - not_null - name: BLOCKCHAIN tests: - not_null - - name: RECENT_BLOCKHASH + - name: RECENT_BLOCK_HASH tests: - not_null - name: TX_ID diff --git a/models/solana/solana_dbt__inner_instructions.sql b/models/solana/solana_dbt__inner_instructions.sql new file mode 100644 index 00000000..b0dbc4fd --- /dev/null +++ b/models/solana/solana_dbt__inner_instructions.sql @@ -0,0 +1,30 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', block_id, tx_id, mapped_event_index)", + incremental_strategy = 'delete+insert', + cluster_by = ['ingested_at::DATE'], + tags = ['snowflake', 'solana', 'silver_solana', 'solana_events'] +) }} + +SELECT + block_timestamp, + block_id, + tx_id, + e.index, + e.value:index::number AS mapped_event_index, + e.value, + ingested_at +FROM {{ ref('bronze_solana__transactions') }} t, +table(flatten(tx:meta:innerInstructions)) as e +WHERE e.value:parsed:type:: STRING <> 'vote' + +{% if is_incremental() %} + AND ingested_at >= ( + SELECT + MAX( + ingested_at + ) + FROM + {{ this }} + ) + {% endif %} \ No newline at end of file diff --git a/models/solana/solana_dbt__instructions.sql b/models/solana/solana_dbt__instructions.sql new file mode 100644 index 00000000..e4760bbb --- /dev/null +++ b/models/solana/solana_dbt__instructions.sql @@ -0,0 +1,30 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', block_id, tx_id, event_type)", + incremental_strategy = 'delete+insert', + cluster_by = ['ingested_at::DATE'], + tags = ['snowflake', 'solana', 'silver_solana', 'solana_events'] +) }} + +SELECT + block_timestamp, + block_id, + tx_id, + e.index, + e.value:parsed:type:: STRING AS event_type, + e.value, + ingested_at +FROM {{ ref('bronze_solana__transactions') }} t, +table(flatten(tx:transaction:message:instructions)) AS e +WHERE e.value:parsed:type:: STRING <> 'vote' + + {% if is_incremental() %} + AND ingested_at >= ( + SELECT + MAX( + ingested_at + ) + FROM + {{ this }} + ) + {% endif %} \ No newline at end of file From 5d7e7a918d4225bf441cc9a40f680fb9a1daa269 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Thu, 27 Jan 2022 13:24:26 -0800 Subject: [PATCH 44/45] adding typo --- .../gold/crosschain__address_labels.sql | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/models/crosschain/gold/crosschain__address_labels.sql b/models/crosschain/gold/crosschain__address_labels.sql index 689d168c..85cf3b9c 100644 --- a/models/crosschain/gold/crosschain__address_labels.sql +++ b/models/crosschain/gold/crosschain__address_labels.sql @@ -1,17 +1,17 @@ {{ config( - materialized = 'view', - tags = ['snowflake', 'crosschain', 'address_labels', 'gold_address_labels'], + materialized = 'view', + tags = ['snowflake', 'crosschain', 'address_labels', 'gold_address_labels'], ) }} -SELECT - system_created_at, - insert_date, - blockchain, - address, - creator, - l1_label AS label_type, - l2_label AS label_subtype, - address_name, +SELECT + system_created_at, + insert_date, + blockchain, + address, + creator, + l1_label AS label_typexxx, + l2_label AS label_subtype, + address_name, project_name -FROM - {{ ref('silver_crosschain__address_labels') }} \ No newline at end of file +FROM + {{ ref('silver_crosschain__address_labels') }} From 287114a92c8afe32bac4b2bec530db844064dc0c Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Thu, 27 Jan 2022 14:24:45 -0800 Subject: [PATCH 45/45] - undo typo --- .../gold/crosschain__address_labels.sql | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/models/crosschain/gold/crosschain__address_labels.sql b/models/crosschain/gold/crosschain__address_labels.sql index 85cf3b9c..689d168c 100644 --- a/models/crosschain/gold/crosschain__address_labels.sql +++ b/models/crosschain/gold/crosschain__address_labels.sql @@ -1,17 +1,17 @@ {{ config( - materialized = 'view', - tags = ['snowflake', 'crosschain', 'address_labels', 'gold_address_labels'], + materialized = 'view', + tags = ['snowflake', 'crosschain', 'address_labels', 'gold_address_labels'], ) }} -SELECT - system_created_at, - insert_date, - blockchain, - address, - creator, - l1_label AS label_typexxx, - l2_label AS label_subtype, - address_name, +SELECT + system_created_at, + insert_date, + blockchain, + address, + creator, + l1_label AS label_type, + l2_label AS label_subtype, + address_name, project_name -FROM - {{ ref('silver_crosschain__address_labels') }} +FROM + {{ ref('silver_crosschain__address_labels') }} \ No newline at end of file