From 184faec3e59b169fd1a73053abb4d32cd2f8a892 Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 21 Jan 2022 16:40:25 -0500 Subject: [PATCH 01/35] update the balance to repoint to the bronze model --- models/terra/dbt/terra_dbt__balances.sql | 46 +++++++++++++++++++ .../silver/silver_terra__daily_balances.sql | 5 +- 2 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 models/terra/dbt/terra_dbt__balances.sql diff --git a/models/terra/dbt/terra_dbt__balances.sql b/models/terra/dbt/terra_dbt__balances.sql new file mode 100644 index 00000000..8698a324 --- /dev/null +++ b/models/terra/dbt/terra_dbt__balances.sql @@ -0,0 +1,46 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', address, block_number)", + incremental_strategy = 'delete+insert', + tags = ['snowflake', 'terra_silver', 'terra_balances'] +) }} + +WITH base_tables AS ( + + SELECT + * + FROM + {{ source( + 'bronze', + 'prod_terra_sink_645110886' + ) }} + WHERE + record_content :model :name :: STRING IN ( + 'terra_balances', + 'terra-5_balances' + ) + +{% if is_incremental() %} +AND ( + record_metadata :CreateTime :: INT / 1000 +) :: TIMESTAMP :: DATE >= ( + SELECT + DATEADD('day', -1, MAX(system_created_at :: DATE)) + FROM + {{ this }} +) +{% endif %} +) +SELECT + t.value :address :: STRING AS address, + t.value :balance :: INT AS balance, + t.value :balance_type :: STRING AS balance_type, + t.value :block_id :: INT AS block_number, + t.value :block_timestamp :: TIMESTAMP AS block_timestamp, + t.value :blockchain :: STRING AS blockchain, + t.value :currency :: STRING AS currency +FROM + base_tables, + LATERAL FLATTEN( + input => record_content :results + ) t diff --git a/models/terra/silver/silver_terra__daily_balances.sql b/models/terra/silver/silver_terra__daily_balances.sql index 4a740938..f6691ddf 100644 --- a/models/terra/silver/silver_terra__daily_balances.sql +++ b/models/terra/silver/silver_terra__daily_balances.sql @@ -65,10 +65,7 @@ eth_balances AS ( 'terra' AS blockchain, balance FROM - {{ source( - 'shared', - 'terra_balances' - ) }} + {{ ref( "terra_dbt__balances" ) }} qualify(ROW_NUMBER() over(PARTITION BY address, currency, block_timestamp :: DATE, balance_type ORDER BY balance DESC)) = 1 From 86466f47cf87a04511d38f55153e168a8d151489 Mon Sep 17 00:00:00 2001 From: yulike Date: Wed, 26 Jan 2022 19:50:51 -0500 Subject: [PATCH 02/35] Update terra_dbt__balances.sql --- models/terra/dbt/terra_dbt__balances.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/terra/dbt/terra_dbt__balances.sql b/models/terra/dbt/terra_dbt__balances.sql index 8698a324..1bee1ba8 100644 --- a/models/terra/dbt/terra_dbt__balances.sql +++ b/models/terra/dbt/terra_dbt__balances.sql @@ -17,7 +17,9 @@ WITH base_tables AS ( WHERE record_content :model :name :: STRING IN ( 'terra_balances', - 'terra-5_balances' + 'terra-5_balances', + 'terra-5_synthetic_balances_model', + 'terra_synthetic_balances_model' ) {% if is_incremental() %} From c27049b232d2971b3aadfac8ea6726676ba559a8 Mon Sep 17 00:00:00 2001 From: yulike Date: Wed, 23 Feb 2022 10:28:00 -0500 Subject: [PATCH 03/35] update the terra balance source table --- models/ethereum/ethereum__erc20_balances.yml | 8 +- .../silver/silver_terra__daily_balances.sql | 203 ++++++++++-------- 2 files changed, 123 insertions(+), 88 deletions(-) diff --git a/models/ethereum/ethereum__erc20_balances.yml b/models/ethereum/ethereum__erc20_balances.yml index 06dc37cb..33d9735c 100644 --- a/models/ethereum/ethereum__erc20_balances.yml +++ b/models/ethereum/ethereum__erc20_balances.yml @@ -43,8 +43,8 @@ models: - not_null - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ - config: - enabled: false + # config: + # enabled: false - name: CONTRACT_LABEL description: Name of the contract address if we have one. tests: @@ -87,5 +87,5 @@ models: - not_null - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ - config: - enabled: false + # config: + # enabled: false diff --git a/models/terra/silver/silver_terra__daily_balances.sql b/models/terra/silver/silver_terra__daily_balances.sql index 5b27403d..c715d7ea 100644 --- a/models/terra/silver/silver_terra__daily_balances.sql +++ b/models/terra/silver/silver_terra__daily_balances.sql @@ -6,88 +6,110 @@ tags = ['snowflake', 'silver_terra', 'silver_terra__daily_balances'] ) }} -with latest as ( -select DATE, - address, - currency, - balance_type, - blockchain, - CASE - WHEN currency IN ('LUNA', 'UST', 'KRT', 'MNT', 'SDT', 'krw', 'mnt', 'sdr') THEN balance - ELSE balance * POW(10,6) - END AS balance -from {{ this }} -where date = (select dateadd('day',-1,max(date)) from {{ this }}) -), -new as ( -select block_timestamp::date as date, -address, -currency, -balance, -blockchain, -balance_type, -1 as rank -from {{ source( - 'shared', - 'terra_balances' - ) }} -where block_timestamp::date >= (select dateadd('day',-1,max(date)) from {{ this }}) -qualify(row_number() over(partition by address, currency,blockchain, balance_type,block_timestamp::date order by block_timestamp desc)) = 1 -), -incremental as ( -select DATE, - address, - currency, - balance_type, - blockchain, - balance -from ( -select DATE, - address, - currency, - balance_type, - blockchain, - balance, -2 as rank -from latest +WITH latest AS ( -union + SELECT + DATE, + address, + currency, + balance_type, + blockchain, + CASE + WHEN currency IN ( + 'LUNA', + 'UST', + 'KRT', + 'MNT', + 'SDT', + 'krw', + 'mnt', + 'sdr' + ) THEN balance + ELSE balance * pow( + 10, + 6 + ) + END AS balance + FROM + {{ this }} + WHERE + DATE = ( + SELECT + DATEADD('day', -1, MAX(DATE)) + FROM + {{ this }}) + ), + NEW AS ( + SELECT + block_timestamp :: DATE AS DATE, + address, + currency, + balance, + blockchain, + balance_type, + 1 AS RANK + FROM + {{ ref( + "terra_dbt__balances" + ) }} + WHERE + block_timestamp :: DATE >= ( + SELECT + DATEADD('day', -1, MAX(DATE)) + FROM + {{ this }}) qualify(ROW_NUMBER() over(PARTITION BY address, currency, blockchain, balance_type, block_timestamp :: DATE + ORDER BY + block_timestamp DESC)) = 1 + ), + incremental AS ( + SELECT + DATE, + address, + currency, + balance_type, + blockchain, + balance + FROM + ( + SELECT + DATE, + address, + currency, + balance_type, + blockchain, + balance, + 2 AS RANK + FROM + latest + UNION + SELECT + DATE, + address, + currency, + balance_type, + blockchain, + balance, + 1 AS RANK + FROM + NEW + ) qualify(ROW_NUMBER() over(PARTITION BY address, currency, blockchain, balance_type, DATE + ORDER BY + RANK ASC)) = 1 + ), + base_balances AS ( -select DATE, - address, - currency, - balance_type, - blockchain, - balance, -1 as rank -from new -) -qualify(row_number() over(partition by address, currency,blockchain, balance_type, date order by rank asc)) = 1 -), -base_balances as ( {% if is_incremental() %} -select date as block_timestamp, - address, - currency, - balance_type, - blockchain, - balance -from incremental +SELECT + DATE AS block_timestamp, address, currency, balance_type, blockchain, balance +FROM + incremental {% else %} -select block_timestamp, - address, - currency, - balance_type, - blockchain, - balance -from {{source( - 'shared', - 'terra_balances' - ) }} -{% endif %} -), - address_ranges AS ( - +SELECT + block_timestamp, address, currency, balance_type, blockchain, balance +FROM + {{ ref("terra_dbt__balances") }} +{% endif %}), +address_ranges AS ( SELECT address, currency, @@ -142,9 +164,10 @@ terra_balances AS ( 'terra' AS blockchain, balance FROM - {{ ref( "terra_dbt__balances" ) }} - base_balances - qualify(ROW_NUMBER() over(PARTITION BY address, currency, block_timestamp :: DATE, balance_type + {{ ref( + "terra_dbt__balances" + ) }} + base_balances qualify(ROW_NUMBER() over(PARTITION BY address, currency, block_timestamp :: DATE, balance_type ORDER BY balance DESC)) = 1 ), @@ -154,8 +177,20 @@ balance_tmp AS ( d.address, d.currency, CASE - WHEN d.currency IN ('LUNA', 'UST', 'KRT', 'MNT', 'SDT', 'krw', 'mnt', 'sdr') THEN b.balance - ELSE b.balance / POW(10,6) + WHEN d.currency IN ( + 'LUNA', + 'UST', + 'KRT', + 'MNT', + 'SDT', + 'krw', + 'mnt', + 'sdr' + ) THEN b.balance + ELSE b.balance / pow( + 10, + 6 + ) END AS balance, d.balance_type, d.blockchain @@ -174,7 +209,7 @@ SELECT currency, balance_type, blockchain, - TRUE as is_native, + TRUE AS is_native, LAST_VALUE( balance ignore nulls ) over( From d0661ec827ffaf0b351068e5c627a77f7bc88591 Mon Sep 17 00:00:00 2001 From: austinFlipside Date: Wed, 23 Feb 2022 19:43:31 -0500 Subject: [PATCH 04/35] test and model updates --- .../dex/ethereum__dex_liquidity_pools.sql | 4 +- .../dex/ethereum__dex_liquidity_pools.yml | 1 - models/ethereum/ethereum__contracts.yml | 8 + models/ethereum/ethereum__reads.yml | 17 ++ .../ethereum__token_prices_hourly.sql | 245 ++++++++++++------ models/ethereum/nft/ethereum__nft_events.yml | 15 ++ 6 files changed, 212 insertions(+), 78 deletions(-) create mode 100644 models/ethereum/ethereum__contracts.yml create mode 100644 models/ethereum/ethereum__reads.yml create mode 100644 models/ethereum/nft/ethereum__nft_events.yml diff --git a/models/ethereum/dex/ethereum__dex_liquidity_pools.sql b/models/ethereum/dex/ethereum__dex_liquidity_pools.sql index 2bc03126..ff149840 100644 --- a/models/ethereum/dex/ethereum__dex_liquidity_pools.sql +++ b/models/ethereum/dex/ethereum__dex_liquidity_pools.sql @@ -261,4 +261,6 @@ SELECT FROM curve WHERE - pool_address IS NOT NULL + pool_address IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY pool_address +ORDER BY + tokens DESC)) = 1 diff --git a/models/ethereum/dex/ethereum__dex_liquidity_pools.yml b/models/ethereum/dex/ethereum__dex_liquidity_pools.yml index 86bd0cfb..a0dac1fa 100644 --- a/models/ethereum/dex/ethereum__dex_liquidity_pools.yml +++ b/models/ethereum/dex/ethereum__dex_liquidity_pools.yml @@ -5,7 +5,6 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - POOL_ADDRESS - enabled: False columns: - name: CREATION_TIME tests: diff --git a/models/ethereum/ethereum__contracts.yml b/models/ethereum/ethereum__contracts.yml new file mode 100644 index 00000000..274d45de --- /dev/null +++ b/models/ethereum/ethereum__contracts.yml @@ -0,0 +1,8 @@ +version: 2 +models: + - name: ethereum__contracts + description: Metadata for Ethereum Contracts + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ADDRESS diff --git a/models/ethereum/ethereum__reads.yml b/models/ethereum/ethereum__reads.yml new file mode 100644 index 00000000..abaa07f7 --- /dev/null +++ b/models/ethereum/ethereum__reads.yml @@ -0,0 +1,17 @@ +version: 2 +models: + - name: ethereum__reads + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - CONTRACT_ADDRESS + - FUNCTION_NAME + - INPUTS + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 \ No newline at end of file diff --git a/models/ethereum/ethereum__token_prices_hourly.sql b/models/ethereum/ethereum__token_prices_hourly.sql index 76ae8707..b6cde47a 100644 --- a/models/ethereum/ethereum__token_prices_hourly.sql +++ b/models/ethereum/ethereum__token_prices_hourly.sql @@ -1,81 +1,174 @@ -{{ - config( - materialized='incremental', - sort='hour', - unique_key='hour', - incremental_strategy='delete+insert', - tags=['snowflake', 'ethereum', 'events'] - ) -}} +{{ config( + materialized = 'incremental', + sort = 'hour', + unique_key = 'hour', + incremental_strategy = 'delete+insert', + tags = ['snowflake', 'ethereum', 'events'] +) }} -with full_decimals as ( - select - lower(address) as contract_address, - meta :decimals :: int as decimals - from +WITH full_decimals AS ( + + SELECT + LOWER(address) AS contract_address, + meta :decimals :: INT AS decimals + FROM {{ ref('silver_ethereum__contracts') }} - where - meta :decimals not like '%00%' qualify(row_number() over(partition by contract_address order by decimals desc) = 1) --need the %00% filter to exclude messy data + WHERE + meta :decimals NOT LIKE '%00%' qualify(ROW_NUMBER() over(PARTITION BY contract_address + ORDER BY + decimals DESC) = 1) --need the %00% filter to exclude messy data ), -hourly_prices as ( - select - p.symbol, - date_trunc('hour', recorded_at) as hour, - lower(a.token_address) as token_address, - d.decimals, - avg(price) as price - from {{ source('shared', 'prices_v2') }} p - left outer join {{ source('shared', 'market_asset_metadata') }} a on p.asset_id = a.asset_id - left outer join full_decimals d on d.contract_address = lower(a.token_address) - where (a.platform_id = '1027' or a.asset_id = '1027' or a.platform_id = 'ethereum') - {% if is_incremental() %} - and recorded_at >= current_date - 45 - {% else %} - and recorded_at >= '2020-05-05' -- first date with valid prices data - {% endif %} - group by 1,2,3,4 -) -, token_addresses as ( - select distinct lower(a.token_address) as token_address - from {{ source('shared', 'prices_v2') }} p - left outer join {{ source('shared', 'market_asset_metadata') }} a on p.asset_id = a.asset_id - where (a.platform_id = '1027' or a.asset_id = '1027' or a.platform_id = 'ethereum') - and (token_address is not null or p.symbol = 'ETH') -) -, hour_token_addresses_pair as ( - select * - from silver.hours - cross join token_addresses - {% if is_incremental() %} - where hour between current_date - 45 and date_trunc('hour',sysdate()) - {% else %} - where hour between '2020-05-05' and date_trunc('hour',sysdate())-- first date with valid prices data - {% endif %} -) -, imputed as ( - select +hourly_prices AS ( + SELECT + p.symbol, + DATE_TRUNC( + 'hour', + recorded_at + ) AS HOUR, + LOWER( + A.token_address + ) AS token_address, + d.decimals, + AVG(price) AS price + FROM + {{ source( + 'shared', + 'prices_v2' + ) }} + p + LEFT OUTER JOIN {{ source( + 'shared', + 'market_asset_metadata' + ) }} A + ON p.asset_id = A.asset_id + LEFT OUTER JOIN full_decimals d + ON d.contract_address = LOWER( + A.token_address + ) + WHERE + ( + A.platform_id = '1027' + OR A.asset_id = '1027' + OR A.platform_id = 'ethereum' + ) + +{% if is_incremental() %} +AND recorded_at >= CURRENT_DATE - 45 +{% else %} + AND recorded_at >= '2020-05-05' -- first date with valid prices data +{% endif %} +GROUP BY + 1, + 2, + 3, + 4 +), +token_addresses AS ( + SELECT + DISTINCT LOWER( + A.token_address + ) AS token_address + FROM + {{ source( + 'shared', + 'prices_v2' + ) }} + p + LEFT OUTER JOIN {{ source( + 'shared', + 'market_asset_metadata' + ) }} A + ON p.asset_id = A.asset_id + WHERE + ( + A.platform_id = '1027' + OR A.asset_id = '1027' + OR A.platform_id = 'ethereum' + ) + AND ( + token_address IS NOT NULL + OR p.symbol = 'ETH' + ) +), +hour_token_addresses_pair AS ( + SELECT + * + FROM + silver.hours + CROSS JOIN token_addresses + +{% if is_incremental() %} +WHERE + HOUR BETWEEN CURRENT_DATE - 45 + AND DATE_TRUNC('hour', SYSDATE()) +{% else %} +WHERE + HOUR BETWEEN '2020-05-05' + AND DATE_TRUNC('hour', SYSDATE() +) -- first date with valid prices data +{% endif %}), +imputed AS ( + SELECT + h.hour, + h.token_address, + p.symbol, + p.decimals, + p.price AS avg_price, + LAG( + p.symbol + ) ignore nulls over ( + PARTITION BY h.token_address + ORDER BY + h.hour + ) AS lag_symbol, + LAG( + p.decimals + ) ignore nulls over ( + PARTITION BY h.token_address + ORDER BY + h.hour + ) AS lag_decimals, + LAG( + p.price + ) ignore nulls over ( + PARTITION BY h.token_address + ORDER BY h.hour - , h.token_address - , p.symbol - , p.decimals - , p.price as avg_price - , lag(p.symbol) ignore nulls over (partition by h.token_address order by h.hour) as lag_symbol - , lag(p.decimals) ignore nulls over (partition by h.token_address order by h.hour) as lag_decimals - , lag(p.price) ignore nulls over (partition by h.token_address order by h.hour) as imputed_price - from hour_token_addresses_pair h - left outer join hourly_prices p on p.hour = h.hour - and (p.token_address = h.token_address or (h.token_address is null and p.symbol = 'ETH')) + ) AS imputed_price + FROM + hour_token_addresses_pair h + LEFT OUTER JOIN hourly_prices p + ON p.hour = h.hour + AND ( + p.token_address = h.token_address + OR ( + h.token_address IS NULL + AND p.symbol = 'ETH' + ) + ) ) -select - p.hour as hour - , p.token_address - , case when symbol is not null then symbol - else lag_symbol end as symbol - , case when decimals is not null then decimals - else lag_decimals end as decimals - , case when avg_price is not null then avg_price - else imputed_price end as price - , case when avg_price is null then TRUE - else FALSE end as is_imputed -from imputed p -where price is not null +SELECT + p.hour AS HOUR, + p.token_address, + CASE + WHEN symbol IS NOT NULL THEN symbol + ELSE lag_symbol + END AS symbol, + CASE + WHEN decimals IS NOT NULL THEN decimals + ELSE lag_decimals + END AS decimals, + CASE + WHEN avg_price IS NOT NULL THEN avg_price + ELSE imputed_price + END AS price, + CASE + WHEN avg_price IS NULL THEN TRUE + ELSE FALSE + END AS is_imputed +FROM + imputed p +WHERE + price IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY HOUR, token_address, symbol +ORDER BY + price DESC)) = 1 diff --git a/models/ethereum/nft/ethereum__nft_events.yml b/models/ethereum/nft/ethereum__nft_events.yml new file mode 100644 index 00000000..d9c425ba --- /dev/null +++ b/models/ethereum/nft/ethereum__nft_events.yml @@ -0,0 +1,15 @@ +# version: 2 +# models: +# - name: ethereum__nft_events +# tests: +# - dbt_utils.unique_combination_of_columns: +# combination_of_columns: +# - TX_ID +# - TOKEN_ID +# columns: +# - name: BLOCK_TIMESTAMP +# tests: +# - not_null +# - dbt_expectations.expect_row_values_to_have_recent_data: +# datepart: day +# interval: 3 \ No newline at end of file From 46fdd2827ae931a9f297af8252456285c6d1a900 Mon Sep 17 00:00:00 2001 From: yulike Date: Wed, 23 Feb 2022 22:02:38 -0500 Subject: [PATCH 05/35] update the table --- models/terra/dbt/terra_dbt__balances.sql | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/models/terra/dbt/terra_dbt__balances.sql b/models/terra/dbt/terra_dbt__balances.sql index 1bee1ba8..cf3aacdb 100644 --- a/models/terra/dbt/terra_dbt__balances.sql +++ b/models/terra/dbt/terra_dbt__balances.sql @@ -16,10 +16,12 @@ WITH base_tables AS ( ) }} WHERE record_content :model :name :: STRING IN ( - 'terra_balances', + 'terra_balances', 'terra-5_balances', - 'terra-5_synthetic_balances_model', - 'terra_synthetic_balances_model' + 'terra_delegations', + 'terra-5_delegations', + 'terra-5_delegations_bison', + 'terra-5_balances_bison' ) {% if is_incremental() %} From 52fec9420a33e75701df7116854515e5f4650ebe Mon Sep 17 00:00:00 2001 From: austinFlipside Date: Thu, 24 Feb 2022 10:53:34 -0500 Subject: [PATCH 06/35] more tests! --- .../silver/silver_ethereum__balances.yml | 16 ++++++++++++++++ .../silver/silver_ethereum__events.yml | 18 ++++++++++++++++++ .../silver/silver_ethereum__prices.yml | 16 ++++++++++++++++ .../ethereum/silver/silver_ethereum__reads.yml | 16 ++++------------ 4 files changed, 54 insertions(+), 12 deletions(-) create mode 100644 models/ethereum/silver/silver_ethereum__balances.yml create mode 100644 models/ethereum/silver/silver_ethereum__events.yml create mode 100644 models/ethereum/silver/silver_ethereum__prices.yml diff --git a/models/ethereum/silver/silver_ethereum__balances.yml b/models/ethereum/silver/silver_ethereum__balances.yml new file mode 100644 index 00000000..b8efac67 --- /dev/null +++ b/models/ethereum/silver/silver_ethereum__balances.yml @@ -0,0 +1,16 @@ +version: 2 +models: + - name: silver_ethereum__balances + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ADDRESS + - BLOCK_ID + - CONTRACT_ADDRESS + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 \ No newline at end of file diff --git a/models/ethereum/silver/silver_ethereum__events.yml b/models/ethereum/silver/silver_ethereum__events.yml new file mode 100644 index 00000000..f5a94ae9 --- /dev/null +++ b/models/ethereum/silver/silver_ethereum__events.yml @@ -0,0 +1,18 @@ +version: 2 +models: + - name: silver_ethereum__events + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - COALESCE(LOG_INDEX, -1) + - TX_HASH + - FROM_UK + - TO_UK + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 diff --git a/models/ethereum/silver/silver_ethereum__prices.yml b/models/ethereum/silver/silver_ethereum__prices.yml new file mode 100644 index 00000000..87170527 --- /dev/null +++ b/models/ethereum/silver/silver_ethereum__prices.yml @@ -0,0 +1,16 @@ +version: 2 +models: + - name: silver_ethereum__prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TOKEN_ADDRESS + - SYMBOL + - HOUR + columns: + - name: HOUR + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 \ No newline at end of file diff --git a/models/ethereum/silver/silver_ethereum__reads.yml b/models/ethereum/silver/silver_ethereum__reads.yml index 3682101b..d4e329e2 100644 --- a/models/ethereum/silver/silver_ethereum__reads.yml +++ b/models/ethereum/silver/silver_ethereum__reads.yml @@ -12,21 +12,13 @@ models: - name: BLOCK_TIMESTAMP tests: - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 - name: BLOCK_ID tests: - not_null - name: CONTRACT_ADDRESS tests: - not_null - - name: CONTRACT_NAME - tests: - - not_null - - name: FUNCTION_NAME - tests: - - not_null - - name: PROJECT_ID - tests: - - not_null - - name: PROJECT_NAME - tests: - - not_null + From a639d491ed94f11d052dab3bf2683570fa66547b Mon Sep 17 00:00:00 2001 From: austinFlipside Date: Thu, 24 Feb 2022 14:13:53 -0500 Subject: [PATCH 07/35] improve incremental --- .../ethereum__token_prices_hourly.sql | 62 +++++++++++-------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/models/ethereum/ethereum__token_prices_hourly.sql b/models/ethereum/ethereum__token_prices_hourly.sql index b6cde47a..c46f2e70 100644 --- a/models/ethereum/ethereum__token_prices_hourly.sql +++ b/models/ethereum/ethereum__token_prices_hourly.sql @@ -53,7 +53,7 @@ hourly_prices AS ( ) {% if is_incremental() %} -AND recorded_at >= CURRENT_DATE - 45 +AND recorded_at >= CURRENT_DATE - 3 {% else %} AND recorded_at >= '2020-05-05' -- first date with valid prices data {% endif %} @@ -99,13 +99,12 @@ hour_token_addresses_pair AS ( {% if is_incremental() %} WHERE - HOUR BETWEEN CURRENT_DATE - 45 + HOUR BETWEEN CURRENT_DATE - 3 AND DATE_TRUNC('hour', SYSDATE()) {% else %} WHERE HOUR BETWEEN '2020-05-05' - AND DATE_TRUNC('hour', SYSDATE() -) -- first date with valid prices data + AND DATE_TRUNC('hour', SYSDATE()) -- first date with valid prices data {% endif %}), imputed AS ( SELECT @@ -146,29 +145,40 @@ imputed AS ( AND p.symbol = 'ETH' ) ) +), +FINAL AS ( + SELECT + p.hour AS HOUR, + p.token_address, + CASE + WHEN symbol IS NOT NULL THEN symbol + ELSE lag_symbol + END AS symbol, + CASE + WHEN decimals IS NOT NULL THEN decimals + ELSE lag_decimals + END AS decimals, + CASE + WHEN avg_price IS NOT NULL THEN avg_price + ELSE imputed_price + END AS price, + CASE + WHEN avg_price IS NULL THEN TRUE + ELSE FALSE + END AS is_imputed + FROM + imputed p + WHERE + price IS NOT NULL ) SELECT - p.hour AS HOUR, - p.token_address, - CASE - WHEN symbol IS NOT NULL THEN symbol - ELSE lag_symbol - END AS symbol, - CASE - WHEN decimals IS NOT NULL THEN decimals - ELSE lag_decimals - END AS decimals, - CASE - WHEN avg_price IS NOT NULL THEN avg_price - ELSE imputed_price - END AS price, - CASE - WHEN avg_price IS NULL THEN TRUE - ELSE FALSE - END AS is_imputed + HOUR, + token_address, + symbol, + decimals, + price, + is_imputed FROM - imputed p -WHERE - price IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY HOUR, token_address, symbol + FINAL qualify(ROW_NUMBER() over(PARTITION BY HOUR, token_address, symbol ORDER BY - price DESC)) = 1 + decimals DESC) = 1) From b5371caaa3f2937ef34c52852160ece8c785c22d Mon Sep 17 00:00:00 2001 From: yulike Date: Thu, 24 Feb 2022 22:31:04 -0500 Subject: [PATCH 08/35] update the daily balance table to use the bronze only --- models/terra/dbt/terra_dbt__balances.sql | 62 ++++++++-- .../silver/silver_terra__daily_balances.sql | 110 ++++++++++-------- 2 files changed, 109 insertions(+), 63 deletions(-) diff --git a/models/terra/dbt/terra_dbt__balances.sql b/models/terra/dbt/terra_dbt__balances.sql index cf3aacdb..1e49dd0a 100644 --- a/models/terra/dbt/terra_dbt__balances.sql +++ b/models/terra/dbt/terra_dbt__balances.sql @@ -34,17 +34,55 @@ AND ( {{ this }} ) {% endif %} +), + +terra_balance_dbt_table AS ( + SELECT + t.value :address :: STRING AS address, + t.value :balance :: INT AS balance, + t.value :balance_type :: STRING AS balance_type, + t.value :block_id :: INT AS block_number, + t.value :block_timestamp :: TIMESTAMP AS block_timestamp, + t.value :blockchain :: STRING AS blockchain, + t.value :currency :: STRING AS currency + FROM + base_tables, + LATERAL FLATTEN( + input => record_content :results + ) t +), + +terra_balance_columbus_3_table AS ( + SELECT + address, + balance, + balance_type, + block_number, + block_timestamp, + 'columbus-3' AS blockchain, + currency + FROM {{ source('shared', 'terra_balances') }} + WHERE date(block_timestamp) < '2020-10-04' AND block_number <= 3820000 ) + +SELECT + address, + balance, + balance_type, + block_number, + block_timestamp, + blockchain, + currency +FROM terra_balance_dbt_table + +UNION ALL + SELECT - t.value :address :: STRING AS address, - t.value :balance :: INT AS balance, - t.value :balance_type :: STRING AS balance_type, - t.value :block_id :: INT AS block_number, - t.value :block_timestamp :: TIMESTAMP AS block_timestamp, - t.value :blockchain :: STRING AS blockchain, - t.value :currency :: STRING AS currency -FROM - base_tables, - LATERAL FLATTEN( - input => record_content :results - ) t + address, + balance, + balance_type, + block_number, + block_timestamp, + blockchain, + currency +FROM terra_balance_columbus_3_table \ No newline at end of file diff --git a/models/terra/silver/silver_terra__daily_balances.sql b/models/terra/silver/silver_terra__daily_balances.sql index c715d7ea..2beca873 100644 --- a/models/terra/silver/silver_terra__daily_balances.sql +++ b/models/terra/silver/silver_terra__daily_balances.sql @@ -39,65 +39,67 @@ WITH latest AS ( FROM {{ this }}) ), - NEW AS ( + +NEW AS ( + SELECT + block_timestamp :: DATE AS DATE, + address, + currency, + balance, + blockchain, + balance_type, + 1 AS RANK + FROM + {{ ref( + "terra_dbt__balances" + ) }} + WHERE + block_timestamp :: DATE >= ( + SELECT + DATEADD('day', -1, MAX(DATE)) + FROM + {{ this }}) qualify(ROW_NUMBER() over(PARTITION BY address, currency, blockchain, balance_type, block_timestamp :: DATE + ORDER BY + block_timestamp DESC)) = 1 + ), + +incremental AS ( + SELECT + DATE, + address, + currency, + balance_type, + blockchain, + balance + FROM + ( SELECT - block_timestamp :: DATE AS DATE, + DATE, address, currency, - balance, + balance_type, blockchain, + balance, + 2 AS RANK + FROM + latest + UNION + SELECT + DATE, + address, + currency, balance_type, + blockchain, + balance, 1 AS RANK FROM - {{ ref( - "terra_dbt__balances" - ) }} - WHERE - block_timestamp :: DATE >= ( - SELECT - DATEADD('day', -1, MAX(DATE)) - FROM - {{ this }}) qualify(ROW_NUMBER() over(PARTITION BY address, currency, blockchain, balance_type, block_timestamp :: DATE - ORDER BY - block_timestamp DESC)) = 1 - ), - incremental AS ( - SELECT - DATE, - address, - currency, - balance_type, - blockchain, - balance - FROM - ( - SELECT - DATE, - address, - currency, - balance_type, - blockchain, - balance, - 2 AS RANK - FROM - latest - UNION - SELECT - DATE, - address, - currency, - balance_type, - blockchain, - balance, - 1 AS RANK - FROM - NEW - ) qualify(ROW_NUMBER() over(PARTITION BY address, currency, blockchain, balance_type, DATE - ORDER BY - RANK ASC)) = 1 - ), - base_balances AS ( + NEW + ) qualify(ROW_NUMBER() over(PARTITION BY address, currency, blockchain, balance_type, DATE + ORDER BY + RANK ASC)) = 1 +), +ase_balances AS ( {% if is_incremental() %} SELECT DATE AS block_timestamp, address, currency, balance_type, blockchain, balance @@ -109,6 +111,7 @@ SELECT FROM {{ ref("terra_dbt__balances") }} {% endif %}), + address_ranges AS ( SELECT address, @@ -129,6 +132,7 @@ address_ranges AS ( 3, 4 ), + cte_my_date AS ( SELECT HOUR :: DATE AS DATE @@ -140,6 +144,7 @@ cte_my_date AS ( GROUP BY 1 ), + all_dates AS ( SELECT C.date, @@ -155,6 +160,7 @@ all_dates AS ( WHERE A.address IS NOT NULL ), + terra_balances AS ( SELECT address, @@ -171,6 +177,7 @@ terra_balances AS ( ORDER BY balance DESC)) = 1 ), + balance_tmp AS ( SELECT d.date, @@ -203,6 +210,7 @@ balance_tmp AS ( AND d.balance_type = b.balance_type AND d.blockchain = b.blockchain ) + SELECT DATE, address, From c12a9152b81d27907be334e18ecc52edc91d7efe Mon Sep 17 00:00:00 2001 From: xiuy001 <97179309+xiuy001@users.noreply.github.com> Date: Thu, 24 Feb 2022 22:35:02 -0500 Subject: [PATCH 09/35] Update ethereum__erc20_balances.yml --- models/ethereum/ethereum__erc20_balances.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/ethereum/ethereum__erc20_balances.yml b/models/ethereum/ethereum__erc20_balances.yml index 33d9735c..06dc37cb 100644 --- a/models/ethereum/ethereum__erc20_balances.yml +++ b/models/ethereum/ethereum__erc20_balances.yml @@ -43,8 +43,8 @@ models: - not_null - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ - # config: - # enabled: false + config: + enabled: false - name: CONTRACT_LABEL description: Name of the contract address if we have one. tests: @@ -87,5 +87,5 @@ models: - not_null - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ - # config: - # enabled: false + config: + enabled: false From d8e45403f234657f45abcbc26440581531133d3e Mon Sep 17 00:00:00 2001 From: austinFlipside Date: Fri, 25 Feb 2022 11:03:00 -0500 Subject: [PATCH 10/35] swaps updates --- models/ethereum/dex/ethereum__dex_swaps.sql | 633 ++++++++++++-------- models/ethereum/dex/ethereum__dex_swaps.yml | 6 +- 2 files changed, 394 insertions(+), 245 deletions(-) diff --git a/models/ethereum/dex/ethereum__dex_swaps.sql b/models/ethereum/dex/ethereum__dex_swaps.sql index 3b36180c..289bf75d 100644 --- a/models/ethereum/dex/ethereum__dex_swaps.sql +++ b/models/ethereum/dex/ethereum__dex_swaps.sql @@ -1,300 +1,445 @@ -{{ - config( - materialized='incremental', - sort='block_timestamp', - unique_key= 'tx_id', - incremental_strategy='delete+insert', - tags=['snowflake', 'ethereum', 'dex','dex_swaps', 'address_labels'] - ) -}} - - - - +{{ config( + materialized = 'incremental', + sort = 'block_timestamp', + unique_key = 'tx_id', + incremental_strategy = 'delete+insert', + tags = ['snowflake', 'ethereum', 'dex','dex_swaps', 'address_labels'] +) }} -- this will depend on having the liquidity pool table set -- makes a table of swaps in a long format, i.e. swap_date, pool, token_address, swapped amount -- this is currently built for uniswap/sushiswap style swaps but that format would also work for pools that allow 2+ tokens (i.e. Curve's 3Pool, Balancer, etc.) -WITH decimals_raw as ( - - SELECT address AS token_address, - meta:decimals AS decimals, - 2 as weight - FROM {{ref('silver_ethereum__contracts')}} - WHERE meta:decimals IS NOT NULL +WITH decimals_raw AS ( + SELECT + address AS token_address, + meta :decimals AS decimals, + 2 AS weight + FROM + {{ ref('silver_ethereum__contracts') }} + WHERE + meta :decimals IS NOT NULL UNION - - SELECT DISTINCT token_address, - decimals, - 1 AS weight - FROM {{ref('ethereum__token_prices_hourly')}} - WHERE - {% if is_incremental() %} - hour >= getdate() - interval '31 days' AND - {% else %} - -- hour >= getdate() - interval '12 months' - {% endif %} - decimals IS NOT NULL - -), decimals AS ( - SELECT token_address,decimals,weight - FROM decimals_raw - QUALIFY (row_number() OVER (partition by token_address order by weight desc)) = 1 + SELECT + DISTINCT token_address, + decimals, + 1 AS weight + FROM + {{ ref('ethereum__token_prices_hourly') }} + WHERE + +{% if is_incremental() %} +HOUR >= getdate() - INTERVAL '31 days' +AND +{% else %} + -- hour >= getdate() - interval '12 months' +{% endif %} + +decimals IS NOT NULL +), +decimals AS ( + SELECT + token_address, + decimals, + weight + FROM + decimals_raw qualify (ROW_NUMBER() over (PARTITION BY token_address + ORDER BY + weight DESC)) = 1 ), -- deduped prices prices AS( - SELECT - token_address, - hour, - AVG(price) AS price, - MAX(decimals) AS decimals - FROM - {{ref('ethereum__token_prices_hourly')}} - WHERE 1=1 - GROUP BY 1,2 + SELECT + token_address, + HOUR, + AVG(price) AS price, + MAX(decimals) AS decimals + FROM + {{ ref('ethereum__token_prices_hourly') }} + WHERE + 1 = 1 + GROUP BY + 1, + 2 ), -- daily avg price used when hourly price is missing (it happens a lot) prices_daily_backup AS( - SELECT - token_address, - date_trunc('day',hour) AS block_date, - AVG(price) AS price, - MAX(decimals) AS decimals - FROM - prices - WHERE 1=1 - GROUP BY 1,2 + SELECT + token_address, + DATE_TRUNC( + 'day', + HOUR + ) AS block_date, + AVG(price) AS price, + MAX(decimals) AS decimals + FROM + prices + WHERE + 1 = 1 + GROUP BY + 1, + 2 ), -- - usd_swaps AS ( - SELECT DISTINCT - block_timestamp, +usd_swaps AS ( + SELECT + DISTINCT block_timestamp, p.pool_address, p.pool_name, p.token0 AS token_address, - tx_id, - event_inputs:amount0In / POWER(10, COALESCE(d0.decimals,backup0.decimals)) AS amount_in, - event_inputs:amount0Out / POWER(10, COALESCE(d0.decimals,backup0.decimals)) AS amount_out, - REGEXP_REPLACE(event_inputs:sender,'\"','') AS from_address, - REGEXP_REPLACE(event_inputs:to,'\"','') AS to_address, - CASE WHEN event_inputs:amount0In > 0 THEN event_inputs:amount0In * COALESCE(price0.price,backup0.price) / POWER(10, COALESCE(d0.decimals,backup0.decimals)) - ELSE event_inputs:amount0Out * COALESCE(price0.price,backup0.price) / POWER(10, COALESCE(d0.decimals,backup0.decimals)) END - AS amount_usd, - CASE WHEN p.factory_address = '0xc0aee478e3658e2610c5f7a4a2e1777ce9e4f2ac' THEN 'sushiswap' ELSE 'uniswap-v2' END AS platform, + tx_id, + event_inputs :amount0In / power(10, COALESCE(d0.decimals, backup0.decimals)) AS amount_in, + event_inputs :amount0Out / power(10, COALESCE(d0.decimals, backup0.decimals)) AS amount_out, + REGEXP_REPLACE( + event_inputs :sender, + '\"', + '' + ) AS from_address, + REGEXP_REPLACE( + event_inputs :to, + '\"', + '' + ) AS to_address, + CASE + WHEN event_inputs :amount0In > 0 THEN event_inputs :amount0In * COALESCE( + price0.price, + backup0.price + ) / power(10, COALESCE(d0.decimals, backup0.decimals)) + ELSE event_inputs :amount0Out * COALESCE( + price0.price, + backup0.price + ) / power(10, COALESCE(d0.decimals, backup0.decimals)) + END AS amount_usd, + CASE + WHEN p.factory_address = '0xc0aee478e3658e2610c5f7a4a2e1777ce9e4f2ac' THEN 'sushiswap' + ELSE 'uniswap-v2' + END AS platform, event_index, - CASE WHEN event_inputs:amount0In > 0 THEN 'IN' - ELSE 'OUT' END AS direction - FROM {{ref('ethereum__events_emitted')}} s0 - - LEFT JOIN {{ref('ethereum__dex_liquidity_pools')}} p + CASE + WHEN event_inputs :amount0In > 0 THEN 'IN' + ELSE 'OUT' + END AS direction + FROM + {{ ref('ethereum__events_emitted') }} + s0 + LEFT JOIN {{ ref('ethereum__dex_liquidity_pools') }} + p ON s0.contract_address = p.pool_address - - LEFT JOIN prices price0 - ON p.token0 = price0.token_address AND DATE_TRUNC('hour',s0.block_timestamp) = price0.hour - - LEFT JOIN prices_daily_backup backup0 - ON p.token0 = backup0.token_address AND DATE_TRUNC('day',s0.block_timestamp) = backup0.block_date - - LEFT JOIN decimals d0 + LEFT JOIN prices price0 + ON p.token0 = price0.token_address + AND DATE_TRUNC( + 'hour', + s0.block_timestamp + ) = price0.hour + LEFT JOIN prices_daily_backup backup0 + ON p.token0 = backup0.token_address + AND DATE_TRUNC( + 'day', + s0.block_timestamp + ) = backup0.block_date + LEFT JOIN decimals d0 ON p.token0 = d0.token_address - - WHERE event_name = 'Swap' AND platform <> 'uniswap-v3' - - {% if is_incremental() %} - AND block_timestamp >= getdate() - interval '7 days' - {% else %} - AND block_timestamp >= getdate() - interval '9 months' - {% endif %} - - UNION - - SELECT DISTINCT - block_timestamp, - p.pool_address, - p.pool_name, - p.token1 AS token_address, - tx_id, - event_inputs:amount1In / POWER(10, COALESCE(d1.decimals,backup1.decimals)) AS amount_in, - event_inputs:amount1Out / POWER(10, COALESCE(d1.decimals,backup1.decimals)) AS amount_out, - REGEXP_REPLACE(event_inputs:sender,'\"','') AS from_address, - REGEXP_REPLACE(event_inputs:to,'\"','') AS to_address, - CASE WHEN event_inputs:amount1In > 0 THEN event_inputs:amount1In * COALESCE(price1.price,backup1.price) / POWER(10, COALESCE(d1.decimals,backup1.decimals)) - ELSE event_inputs:amount1Out * COALESCE(price1.price,backup1.price) / POWER(10, COALESCE(d1.decimals,backup1.decimals)) END - AS amount_usd, - CASE WHEN p.factory_address = '0xc0aee478e3658e2610c5f7a4a2e1777ce9e4f2ac' THEN 'sushiswap' ELSE 'uniswap-v2' END AS platform, - event_index, - CASE WHEN event_inputs:amount1In > 0 THEN 'IN' - ELSE 'OUT' END AS direction - - FROM {{ref('ethereum__events_emitted')}} s0 - - LEFT JOIN {{ref('ethereum__dex_liquidity_pools')}} p - ON s0.contract_address = p.pool_address - + WHERE + event_name = 'Swap' + AND platform <> 'uniswap-v3' + +{% if is_incremental() %} +AND block_timestamp >= getdate() - INTERVAL '7 days' +{% else %} + AND block_timestamp >= getdate() - INTERVAL '9 months' +{% endif %} +UNION +SELECT + DISTINCT block_timestamp, + p.pool_address, + p.pool_name, + p.token1 AS token_address, + tx_id, + event_inputs :amount1In / power(10, COALESCE(d1.decimals, backup1.decimals)) AS amount_in, + event_inputs :amount1Out / power(10, COALESCE(d1.decimals, backup1.decimals)) AS amount_out, + REGEXP_REPLACE( + event_inputs :sender, + '\"', + '' + ) AS from_address, + REGEXP_REPLACE( + event_inputs :to, + '\"', + '' + ) AS to_address, + CASE + WHEN event_inputs :amount1In > 0 THEN event_inputs :amount1In * COALESCE( + price1.price, + backup1.price + ) / power(10, COALESCE(d1.decimals, backup1.decimals)) + ELSE event_inputs :amount1Out * COALESCE( + price1.price, + backup1.price + ) / power(10, COALESCE(d1.decimals, backup1.decimals)) + END AS amount_usd, + CASE + WHEN p.factory_address = '0xc0aee478e3658e2610c5f7a4a2e1777ce9e4f2ac' THEN 'sushiswap' + ELSE 'uniswap-v2' + END AS platform, + event_index, + CASE + WHEN event_inputs :amount1In > 0 THEN 'IN' + ELSE 'OUT' + END AS direction +FROM + {{ ref('ethereum__events_emitted') }} + s0 + LEFT JOIN {{ ref('ethereum__dex_liquidity_pools') }} + p + ON s0.contract_address = p.pool_address LEFT JOIN prices price1 - ON p.token1 = price1.token_address AND DATE_TRUNC('hour',s0.block_timestamp) = price1.hour - + ON p.token1 = price1.token_address + AND DATE_TRUNC( + 'hour', + s0.block_timestamp + ) = price1.hour LEFT JOIN prices_daily_backup backup1 - ON p.token1 = backup1.token_address AND DATE_TRUNC('day',s0.block_timestamp) = backup1.block_date - + ON p.token1 = backup1.token_address + AND DATE_TRUNC( + 'day', + s0.block_timestamp + ) = backup1.block_date LEFT JOIN decimals d1 - ON p.token1 = d1.token_address - - - WHERE - event_name = 'Swap' AND platform <> 'uniswap-v3' - {% if is_incremental() %} - AND block_timestamp >= getdate() - interval '7 days' - {% else %} - AND block_timestamp >= getdate() - interval '9 months' - {% endif %} - -), v3_swaps AS ( - - SELECT DISTINCT - s.block_timestamp,s.pool_address, -- token 0 v3 swaps - dl.pool_name, -- get from lp table + ON p.token1 = d1.token_address +WHERE + event_name = 'Swap' + AND platform <> 'uniswap-v3' + +{% if is_incremental() %} +AND block_timestamp >= getdate() - INTERVAL '7 days' +{% else %} + AND block_timestamp >= getdate() - INTERVAL '9 months' +{% endif %} +), +v3_swaps AS ( + SELECT + DISTINCT s.block_timestamp, + s.pool_address, + -- token 0 v3 swaps + dl.pool_name, + -- get from lp table dl.token0 AS token_address, s.tx_id, - CASE WHEN amount0_adjusted > 0 THEN amount0_adjusted ELSE NULL END AS amount_in, - CASE WHEN amount0_adjusted <= 0 THEN -1*amount0_adjusted ELSE NULL END AS amount_out, + CASE + WHEN amount0_adjusted > 0 THEN amount0_adjusted + ELSE NULL + END AS amount_in, + CASE + WHEN amount0_adjusted <= 0 THEN -1 * amount0_adjusted + ELSE NULL + END AS amount_out, sender AS from_address, recipient AS to_address, - COALESCE(amount_in,amount_out)*p.price AS amount_usd, + COALESCE( + amount_in, + amount_out + ) * p.price AS amount_usd, 'uniswap-v3' AS platform, ind.event_index, - CASE WHEN amount0_adjusted > 0 THEN 'IN' - ELSE 'OUT' END AS direction - FROM - -- {{source('uniswapv3_eth','uniswapv3_swaps')}} - {{ref('uniswapv3__swaps')}} s - LEFT JOIN {{ref('ethereum__dex_liquidity_pools')}} dl + CASE + WHEN amount0_adjusted > 0 THEN 'IN' + ELSE 'OUT' + END AS direction + FROM + -- {{source('uniswapv3_eth','uniswapv3_swaps')}} + {{ ref('uniswapv3__swaps') }} + s + LEFT JOIN {{ ref('ethereum__dex_liquidity_pools') }} + dl ON s.pool_address = dl.pool_address - LEFT JOIN prices p - ON dl.token0 = p.token_address AND p.hour = date_trunc('hour',block_timestamp) - LEFT JOIN {{ref('ethereum__events_emitted')}} ind - ON s.pool_address = ind.contract_address AND s.tx_id = ind.tx_id - WHERE (amount1_adjusted > 0 OR amount0_adjusted > 0) AND platform = 'uniswap-v3' - + LEFT JOIN prices p + ON dl.token0 = p.token_address + AND p.hour = DATE_TRUNC( + 'hour', + block_timestamp + ) + LEFT JOIN {{ ref('ethereum__events_emitted') }} + ind + ON s.pool_address = ind.contract_address + AND s.tx_id = ind.tx_id + WHERE + ( + amount1_adjusted > 0 + OR amount0_adjusted > 0 + ) + AND platform = 'uniswap-v3' UNION - - SELECT DISTINCT - s.block_timestamp,s.pool_address, -- token1 v3 swaps - dl.pool_name, -- get from lp table + SELECT + DISTINCT s.block_timestamp, + s.pool_address, + -- token1 v3 swaps + dl.pool_name, + -- get from lp table dl.token1 AS token_address, s.tx_id, - CASE WHEN amount1_adjusted > 0 THEN amount1_adjusted ELSE NULL END AS amount_in, - CASE WHEN amount1_adjusted <= 0 THEN -1*amount1_adjusted ELSE NULL END AS amount_out, + CASE + WHEN amount1_adjusted > 0 THEN amount1_adjusted + ELSE NULL + END AS amount_in, + CASE + WHEN amount1_adjusted <= 0 THEN -1 * amount1_adjusted + ELSE NULL + END AS amount_out, sender AS from_address, recipient AS to_address, - COALESCE(amount_in,amount_out)*p.price AS amount_usd , + COALESCE( + amount_in, + amount_out + ) * p.price AS amount_usd, 'uniswap-v3' AS platform, ind.event_index, - CASE WHEN amount1_adjusted > 0 THEN 'IN' - ELSE 'OUT' END AS direction - FROM - -- {{source('uniswapv3_eth','uniswapv3_swaps')}} s - {{ref('uniswapv3__swaps')}} s - LEFT JOIN {{ref('ethereum__dex_liquidity_pools')}} dl + CASE + WHEN amount1_adjusted > 0 THEN 'IN' + ELSE 'OUT' + END AS direction + FROM + -- {{source('uniswapv3_eth','uniswapv3_swaps')}} s + {{ ref('uniswapv3__swaps') }} + s + LEFT JOIN {{ ref('ethereum__dex_liquidity_pools') }} + dl ON s.pool_address = dl.pool_address - LEFT JOIN prices p - ON dl.token1 = p.token_address AND p.hour = date_trunc('hour',block_timestamp) - LEFT JOIN {{ref('ethereum__events_emitted')}} ind - ON s.pool_address = ind.contract_address AND s.tx_id = ind.tx_id - WHERE (amount1_adjusted > 0 OR amount0_adjusted > 0) AND platform = 'uniswap-v3' -), swaps AS ( - SELECT - block_timestamp,pool_address, + LEFT JOIN prices p + ON dl.token1 = p.token_address + AND p.hour = DATE_TRUNC( + 'hour', + block_timestamp + ) + LEFT JOIN {{ ref('ethereum__events_emitted') }} + ind + ON s.pool_address = ind.contract_address + AND s.tx_id = ind.tx_id + WHERE + ( + amount1_adjusted > 0 + OR amount0_adjusted > 0 + ) + AND platform = 'uniswap-v3' +), +swaps AS ( + SELECT + block_timestamp, + pool_address, pool_name, token_address, tx_id, - amount_in,amount_out, + amount_in, + amount_out, from_address, to_address, amount_usd, platform, event_index, direction - FROM usd_swaps - WHERE pool_address NOT IN ('0xdc6a5faf34affccc6a00d580ecb3308fc1848f22') -- stop-gap for big price swings, the actual solution adds an enormous amount of runtime - - + FROM + usd_swaps + WHERE + pool_address NOT IN ('0xdc6a5faf34affccc6a00d580ecb3308fc1848f22') -- stop-gap for big price swings, the actual solution adds an enormous amount of runtime UNION - - SELECT * - FROM v3_swaps - + SELECT + * + FROM + v3_swaps UNION - - -- Curve (in) + -- Curve (in) SELECT - c.block_timestamp, - c.pool_address, - c.pool_name, - c.token_in AS token_address, - c.tx_id, - c.amount_in AS amount_in, + C.block_timestamp, + C.pool_address, + C.pool_name, + C.token_in AS token_address, + C.tx_id, + C.amount_in AS amount_in, 0 AS amount_out, - c.swapper AS from_address, - c.swapper AS to_address, - c.amount_in_usd AS amount_usd, + C.swapper AS from_address, + C.swapper AS to_address, + C.amount_in_usd AS amount_usd, 'curve' AS platform, - c.event_index, + C.event_index, 'IN' AS direction - FROM {{ref('ethereum_dbt__curve_swaps')}} c - WHERE - {% if is_incremental() %} - block_timestamp >= getdate() - interval '7 days' - {% else %} - block_timestamp >= getdate() - interval '12 months' - {% endif %} - - UNION - + FROM + {{ ref('ethereum_dbt__curve_swaps') }} C + WHERE + +{% if is_incremental() %} +block_timestamp >= getdate() - INTERVAL '7 days' +{% else %} + block_timestamp >= getdate() - INTERVAL '12 months' +{% endif %} +UNION -- Curve (out) +SELECT + C.block_timestamp, + C.pool_address, + C.pool_name, + C.token_out AS token_address, + C.tx_id, + 0 AS amount_in, + C.amount_out AS amount_out, + C.swapper AS from_address, + C.swapper AS to_address, + C.amount_out_usd AS amount_usd, + 'curve' AS platform, + C.event_index, + 'OUT' AS direction +FROM + {{ ref('ethereum_dbt__curve_swaps') }} C +WHERE + +{% if is_incremental() %} +block_timestamp >= getdate() - INTERVAL '7 days' +{% else %} + block_timestamp >= getdate() - INTERVAL '12 months' +{% endif %} +), +FINAL AS ( SELECT - c.block_timestamp, - c.pool_address, - c.pool_name, - c.token_out AS token_address, - c.tx_id, - 0 AS amount_in, - c.amount_out AS amount_out, - c.swapper AS from_address, - c.swapper AS to_address, - c.amount_out_usd AS amount_usd, - 'curve' AS platform, - c.event_index, - 'OUT' AS direction - FROM {{ref('ethereum_dbt__curve_swaps')}} c - WHERE - {% if is_incremental() %} - block_timestamp >= getdate() - interval '7 days' - {% else %} - block_timestamp >= getdate() - interval '12 months' - {% endif %} + s.block_timestamp, + s.pool_address, + s.pool_name, + s.token_address, + s.tx_id, + s.amount_in, + s.amount_out, + s.from_address, + s.to_address, + s.amount_usd, + s.platform, + s.event_index, + s.direction, + CASE + WHEN s.from_address <> s.to_address THEN l.address + ELSE NULL + END AS router + FROM + swaps s + LEFT JOIN {{ ref('silver_crosschain__address_labels') }} + l + ON s.from_address = l.address + AND l.blockchain = 'ethereum' + AND l.creator = 'flipside' + WHERE + s.pool_name IS NOT NULL ) - - - SELECT - s.block_timestamp, - s.pool_address, - s.pool_name, - s.token_address, - s.tx_id, - s.amount_in, - s.amount_out, - s.from_address, - s.to_address, - s.amount_usd, - s.platform, - s.event_index, - s.direction, - CASE WHEN s.from_address <> s.to_address THEN l.address ELSE NULL END AS router -FROM -swaps s -LEFT JOIN -{{ref('silver_crosschain__address_labels')}} l -ON s.from_address = l.address AND l.blockchain = 'ethereum' AND l.creator = 'flipside' -WHERE s.pool_name IS NOT NULL + block_timestamp, + pool_address, + pool_name, + token_address, + tx_id, + amount_in, + amount_out, + from_address, + to_address, + amount_usd, + platform, + event_index, + direction, + router +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY tx_id, event_index, direction, token_address, amount_in, amount_out +ORDER BY + block_timestamp DESC, router DESC nulls last)) = 1 diff --git a/models/ethereum/dex/ethereum__dex_swaps.yml b/models/ethereum/dex/ethereum__dex_swaps.yml index 7fbc9cf1..80ad5cd0 100644 --- a/models/ethereum/dex/ethereum__dex_swaps.yml +++ b/models/ethereum/dex/ethereum__dex_swaps.yml @@ -7,6 +7,9 @@ models: - DIRECTION - EVENT_INDEX - TX_ID + - TOKEN_ADDRESS + - AMOUNT_IN + - AMOUNT_OUT columns: - name: AMOUNT_IN # TODO: proportional null test tests: @@ -39,7 +42,8 @@ models: value_set: ["IN", "OUT"] - name: EVENT_INDEX tests: - - not_null + - not_null: + where: PLATFORM <> 'uniswap-v3' - name: FROM_ADDRESS tests: - not_null From f9da63da1700f72328958807c81fc7a3a8a7b215 Mon Sep 17 00:00:00 2001 From: austinFlipside Date: Fri, 25 Feb 2022 11:55:50 -0500 Subject: [PATCH 11/35] more recent --- models/ethereum/ethereum__reads.yml | 2 +- models/ethereum/silver/silver_ethereum__balances.yml | 2 +- models/ethereum/silver/silver_ethereum__events.yml | 2 +- models/ethereum/silver/silver_ethereum__prices.yml | 2 +- models/ethereum/silver/silver_ethereum__reads.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/ethereum/ethereum__reads.yml b/models/ethereum/ethereum__reads.yml index abaa07f7..955b0842 100644 --- a/models/ethereum/ethereum__reads.yml +++ b/models/ethereum/ethereum__reads.yml @@ -14,4 +14,4 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 3 \ No newline at end of file + interval: 1 \ No newline at end of file diff --git a/models/ethereum/silver/silver_ethereum__balances.yml b/models/ethereum/silver/silver_ethereum__balances.yml index b8efac67..b8d58034 100644 --- a/models/ethereum/silver/silver_ethereum__balances.yml +++ b/models/ethereum/silver/silver_ethereum__balances.yml @@ -13,4 +13,4 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 3 \ No newline at end of file + interval: 1 \ No newline at end of file diff --git a/models/ethereum/silver/silver_ethereum__events.yml b/models/ethereum/silver/silver_ethereum__events.yml index f5a94ae9..857d7814 100644 --- a/models/ethereum/silver/silver_ethereum__events.yml +++ b/models/ethereum/silver/silver_ethereum__events.yml @@ -15,4 +15,4 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 3 + interval: 1 diff --git a/models/ethereum/silver/silver_ethereum__prices.yml b/models/ethereum/silver/silver_ethereum__prices.yml index 87170527..49df3317 100644 --- a/models/ethereum/silver/silver_ethereum__prices.yml +++ b/models/ethereum/silver/silver_ethereum__prices.yml @@ -13,4 +13,4 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 3 \ No newline at end of file + interval: 1 \ No newline at end of file diff --git a/models/ethereum/silver/silver_ethereum__reads.yml b/models/ethereum/silver/silver_ethereum__reads.yml index d4e329e2..0cdade41 100644 --- a/models/ethereum/silver/silver_ethereum__reads.yml +++ b/models/ethereum/silver/silver_ethereum__reads.yml @@ -14,7 +14,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 3 + interval: 1 - name: BLOCK_ID tests: - not_null From ffeb59e1cd12f3dbe44d3e4576c058a57d799cd5 Mon Sep 17 00:00:00 2001 From: austinFlipside Date: Fri, 25 Feb 2022 12:21:31 -0500 Subject: [PATCH 12/35] more aave tests --- models/ethereum/aave/silver__aave_liquidity_mining.yml | 2 +- models/ethereum/aave/silver__aave_oracle_prices.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/models/ethereum/aave/silver__aave_liquidity_mining.yml b/models/ethereum/aave/silver__aave_liquidity_mining.yml index 058e01d9..ce50be01 100644 --- a/models/ethereum/aave/silver__aave_liquidity_mining.yml +++ b/models/ethereum/aave/silver__aave_liquidity_mining.yml @@ -12,7 +12,7 @@ models: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 3 + interval: 1 - name: TOKEN_ADDRESS tests: - not_null diff --git a/models/ethereum/aave/silver__aave_oracle_prices.yml b/models/ethereum/aave/silver__aave_oracle_prices.yml index 50731cf6..fdf8d720 100644 --- a/models/ethereum/aave/silver__aave_oracle_prices.yml +++ b/models/ethereum/aave/silver__aave_oracle_prices.yml @@ -10,6 +10,9 @@ models: - name: HOUR tests: - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 - name: DECIMALS tests: - not_null From 74fe60bd7cc8a88d778c7d7ed4876326af2a00b7 Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 25 Feb 2022 13:35:13 -0500 Subject: [PATCH 13/35] Update silver_terra__daily_balances.sql --- models/terra/silver/silver_terra__daily_balances.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/terra/silver/silver_terra__daily_balances.sql b/models/terra/silver/silver_terra__daily_balances.sql index 2beca873..0b2275b5 100644 --- a/models/terra/silver/silver_terra__daily_balances.sql +++ b/models/terra/silver/silver_terra__daily_balances.sql @@ -99,7 +99,7 @@ incremental AS ( RANK ASC)) = 1 ), -ase_balances AS ( +base_balances AS ( {% if is_incremental() %} SELECT DATE AS block_timestamp, address, currency, balance_type, blockchain, balance From c5cfce7ab1a11cbb1017755a0a08fa9ce36d0191 Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 25 Feb 2022 13:35:26 -0500 Subject: [PATCH 14/35] Update terra_dbt__balances.sql --- models/terra/dbt/terra_dbt__balances.sql | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/models/terra/dbt/terra_dbt__balances.sql b/models/terra/dbt/terra_dbt__balances.sql index 1e49dd0a..bc39c2a8 100644 --- a/models/terra/dbt/terra_dbt__balances.sql +++ b/models/terra/dbt/terra_dbt__balances.sql @@ -8,20 +8,24 @@ WITH base_tables AS ( SELECT - * + record_metadata, + record_content, + _inserted_timestamp FROM {{ source( 'bronze', 'prod_terra_sink_645110886' ) }} WHERE - record_content :model :name :: STRING IN ( - 'terra_balances', - 'terra-5_balances', - 'terra_delegations', - 'terra-5_delegations', - 'terra-5_delegations_bison', - 'terra-5_balances_bison' + record_content :model :class :: STRING IN ( + 'terra.balances.models.TerraDelegationsModel', + 'terra.balances.models.TerraBalancesModel', + 'terra.balances.models.Terra5BalancesModel', + 'terra.balances.terra_synthetic_balances_model.Terra5SyntheticBalancesModel', + 'terra.balances.models.Terra5DelegationsModel', + 'terra.balances.terra_synthetic_balances_model.TerraSyntheticBalancesModel', + 'terra.balances.models.TerraBalancesModel', + 'terra.balances.models.TerraDelegationsModel' ) {% if is_incremental() %} From 53b6af003bf5d1e137d4559203671d64edef3bae Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 25 Feb 2022 13:38:03 -0500 Subject: [PATCH 15/35] updated the incremental logic --- models/terra/dbt/terra_dbt__balances.sql | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/models/terra/dbt/terra_dbt__balances.sql b/models/terra/dbt/terra_dbt__balances.sql index bc39c2a8..28a5dd4f 100644 --- a/models/terra/dbt/terra_dbt__balances.sql +++ b/models/terra/dbt/terra_dbt__balances.sql @@ -29,14 +29,12 @@ WITH base_tables AS ( ) {% if is_incremental() %} -AND ( - record_metadata :CreateTime :: INT / 1000 -) :: TIMESTAMP :: DATE >= ( - SELECT - DATEADD('day', -1, MAX(system_created_at :: DATE)) - FROM - {{ this }} -) + AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) {% endif %} ), From 804ff59528d1b4ce824cd1399ed859d4bb4adcbe Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 25 Feb 2022 16:36:18 -0500 Subject: [PATCH 16/35] Update terra_dbt__balances.sql --- models/terra/dbt/terra_dbt__balances.sql | 40 ++++++++++++++---------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/models/terra/dbt/terra_dbt__balances.sql b/models/terra/dbt/terra_dbt__balances.sql index 28a5dd4f..a5de83a7 100644 --- a/models/terra/dbt/terra_dbt__balances.sql +++ b/models/terra/dbt/terra_dbt__balances.sql @@ -33,7 +33,10 @@ WITH base_tables AS ( SELECT MAX(_inserted_timestamp) FROM - {{ this }} + {{ source( + 'bronze', + 'prod_terra_sink_645110886' + ) }} ) {% endif %} ), @@ -52,21 +55,10 @@ terra_balance_dbt_table AS ( LATERAL FLATTEN( input => record_content :results ) t -), - -terra_balance_columbus_3_table AS ( - SELECT - address, - balance, - balance_type, - block_number, - block_timestamp, - 'columbus-3' AS blockchain, - currency - FROM {{ source('shared', 'terra_balances') }} - WHERE date(block_timestamp) < '2020-10-04' AND block_number <= 3820000 ) +{% if is_incremental() %} + SELECT address, balance, @@ -77,7 +69,7 @@ SELECT currency FROM terra_balance_dbt_table -UNION ALL +{% else %} SELECT address, @@ -87,4 +79,20 @@ SELECT block_timestamp, blockchain, currency -FROM terra_balance_columbus_3_table \ No newline at end of file +FROM terra_balance_dbt_table + +UNION ALL + +SELECT + address, + balance, + balance_type, + block_number, + block_timestamp, + 'columbus-3' AS blockchain, + currency +FROM {{ source('shared', 'terra_balances') }} +WHERE date(block_timestamp) < '2020-10-04' AND block_number <= 3820000 + +{% endif %} + From d3ee16f66a2d467a943428f5a450fd99625a23e9 Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 25 Feb 2022 17:56:18 -0500 Subject: [PATCH 17/35] updated unique key --- models/terra/gold/terra__daily_balances.sql | 2 +- models/terra/silver/silver_terra__daily_balances.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/terra/gold/terra__daily_balances.sql b/models/terra/gold/terra__daily_balances.sql index 53346160..03c0ac03 100644 --- a/models/terra/gold/terra__daily_balances.sql +++ b/models/terra/gold/terra__daily_balances.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = 'date', + unique_key = "CONCAT_WS('-', date, address, currency, balance_type)", incremental_strategy = 'delete+insert', cluster_by = ['date'], tags = ['snowflake', 'terra', 'balances', 'terra_daily_balances', 'address_labels'] diff --git a/models/terra/silver/silver_terra__daily_balances.sql b/models/terra/silver/silver_terra__daily_balances.sql index 0b2275b5..1f01455f 100644 --- a/models/terra/silver/silver_terra__daily_balances.sql +++ b/models/terra/silver/silver_terra__daily_balances.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = 'date', + unique_key = "CONCAT_WS('-', date, address, currency, balance_type, blockchain)", incremental_strategy = 'delete+insert', cluster_by = ['date'], tags = ['snowflake', 'silver_terra', 'silver_terra__daily_balances'] From 692a659cd5d50ccbd209aaf6826db25bcd280e4e Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 25 Feb 2022 18:02:11 -0500 Subject: [PATCH 18/35] updated unique key --- models/terra/dbt/terra_dbt__balances.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/terra/dbt/terra_dbt__balances.sql b/models/terra/dbt/terra_dbt__balances.sql index a5de83a7..49614f98 100644 --- a/models/terra/dbt/terra_dbt__balances.sql +++ b/models/terra/dbt/terra_dbt__balances.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = "CONCAT_WS('-', address, block_number)", + unique_key = "CONCAT_WS('-', address, block_number, balance_type, currency)", incremental_strategy = 'delete+insert', tags = ['snowflake', 'terra_silver', 'terra_balances'] ) }} From 058fa21c9b3f665d12619e95f6a968d890235cf9 Mon Sep 17 00:00:00 2001 From: austinFlipside Date: Mon, 28 Feb 2022 08:41:33 -0500 Subject: [PATCH 19/35] curve change --- models/ethereum/dex/ethereum_dbt__curve_swaps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/models/ethereum/dex/ethereum_dbt__curve_swaps.yml b/models/ethereum/dex/ethereum_dbt__curve_swaps.yml index 934bc494..6cfb8259 100644 --- a/models/ethereum/dex/ethereum_dbt__curve_swaps.yml +++ b/models/ethereum/dex/ethereum_dbt__curve_swaps.yml @@ -6,6 +6,7 @@ models: combination_of_columns: - TX_ID - event_index + - POOL_NAME columns: - name: AMOUNT_IN tests: From 598898c07d00aa86dbb0edea2affedbc86bc8e7a Mon Sep 17 00:00:00 2001 From: Ava Masucci <31529186+amasucci13@users.noreply.github.com> Date: Mon, 28 Feb 2022 11:38:51 -0800 Subject: [PATCH 20/35] AN-763/oracle_prices_unique_key (#361) * Incremental prices change * changed unique key --- models/terra/gold/terra__oracle_prices.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/terra/gold/terra__oracle_prices.sql b/models/terra/gold/terra__oracle_prices.sql index df49f3eb..72f139f7 100644 --- a/models/terra/gold/terra__oracle_prices.sql +++ b/models/terra/gold/terra__oracle_prices.sql @@ -1,8 +1,8 @@ {{ config( materialized = 'incremental', - sort = 'block_timestamp', - unique_key = "CONCAT_WS('-', block_timestamp)", + unique_key = "CONCAT_WS('-', block_timestamp, currency, source)", incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], tags = ['snowflake', 'terra', 'oracle', 'terra_oracle', 'address_labels'] ) }} From bf7c3ba1d12e4f34a6bdb639b8609b40ef607e6d Mon Sep 17 00:00:00 2001 From: yulike Date: Mon, 28 Feb 2022 17:17:54 -0500 Subject: [PATCH 21/35] quick fix the issue --- models/terra/dbt/terra_dbt__balances.sql | 98 --------- .../silver/silver_terra__daily_balances.sql | 208 +++++++----------- 2 files changed, 82 insertions(+), 224 deletions(-) delete mode 100644 models/terra/dbt/terra_dbt__balances.sql diff --git a/models/terra/dbt/terra_dbt__balances.sql b/models/terra/dbt/terra_dbt__balances.sql deleted file mode 100644 index 49614f98..00000000 --- a/models/terra/dbt/terra_dbt__balances.sql +++ /dev/null @@ -1,98 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "CONCAT_WS('-', address, block_number, balance_type, currency)", - incremental_strategy = 'delete+insert', - tags = ['snowflake', 'terra_silver', 'terra_balances'] -) }} - -WITH base_tables AS ( - - SELECT - record_metadata, - record_content, - _inserted_timestamp - FROM - {{ source( - 'bronze', - 'prod_terra_sink_645110886' - ) }} - WHERE - record_content :model :class :: STRING IN ( - 'terra.balances.models.TerraDelegationsModel', - 'terra.balances.models.TerraBalancesModel', - 'terra.balances.models.Terra5BalancesModel', - 'terra.balances.terra_synthetic_balances_model.Terra5SyntheticBalancesModel', - 'terra.balances.models.Terra5DelegationsModel', - 'terra.balances.terra_synthetic_balances_model.TerraSyntheticBalancesModel', - 'terra.balances.models.TerraBalancesModel', - 'terra.balances.models.TerraDelegationsModel' - ) - -{% if is_incremental() %} - AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ source( - 'bronze', - 'prod_terra_sink_645110886' - ) }} - ) -{% endif %} -), - -terra_balance_dbt_table AS ( - SELECT - t.value :address :: STRING AS address, - t.value :balance :: INT AS balance, - t.value :balance_type :: STRING AS balance_type, - t.value :block_id :: INT AS block_number, - t.value :block_timestamp :: TIMESTAMP AS block_timestamp, - t.value :blockchain :: STRING AS blockchain, - t.value :currency :: STRING AS currency - FROM - base_tables, - LATERAL FLATTEN( - input => record_content :results - ) t -) - -{% if is_incremental() %} - -SELECT - address, - balance, - balance_type, - block_number, - block_timestamp, - blockchain, - currency -FROM terra_balance_dbt_table - -{% else %} - -SELECT - address, - balance, - balance_type, - block_number, - block_timestamp, - blockchain, - currency -FROM terra_balance_dbt_table - -UNION ALL - -SELECT - address, - balance, - balance_type, - block_number, - block_timestamp, - 'columbus-3' AS blockchain, - currency -FROM {{ source('shared', 'terra_balances') }} -WHERE date(block_timestamp) < '2020-10-04' AND block_number <= 3820000 - -{% endif %} - diff --git a/models/terra/silver/silver_terra__daily_balances.sql b/models/terra/silver/silver_terra__daily_balances.sql index 1f01455f..99e84da9 100644 --- a/models/terra/silver/silver_terra__daily_balances.sql +++ b/models/terra/silver/silver_terra__daily_balances.sql @@ -1,118 +1,93 @@ {{ config( materialized = 'incremental', - unique_key = "CONCAT_WS('-', date, address, currency, balance_type, blockchain)", + unique_key = 'date', incremental_strategy = 'delete+insert', cluster_by = ['date'], tags = ['snowflake', 'silver_terra', 'silver_terra__daily_balances'] ) }} -WITH latest AS ( - - SELECT - DATE, - address, - currency, - balance_type, - blockchain, - CASE - WHEN currency IN ( - 'LUNA', - 'UST', - 'KRT', - 'MNT', - 'SDT', - 'krw', - 'mnt', - 'sdr' - ) THEN balance - ELSE balance * pow( - 10, - 6 - ) +with latest as ( +select DATE, + address, + currency, + balance_type, + blockchain, + CASE + WHEN currency IN ('LUNA', 'UST', 'KRT', 'MNT', 'SDT', 'krw', 'mnt', 'sdr') THEN balance + ELSE balance * POW(10,6) END AS balance - FROM - {{ this }} - WHERE - DATE = ( - SELECT - DATEADD('day', -1, MAX(DATE)) - FROM - {{ this }}) - ), - -NEW AS ( - SELECT - block_timestamp :: DATE AS DATE, - address, - currency, - balance, - blockchain, - balance_type, - 1 AS RANK - FROM - {{ ref( - "terra_dbt__balances" +from {{ this }} +where date = (select dateadd('day',-1,max(date)) from {{ this }}) +), +new as ( +select block_timestamp::date as date, +address, +currency, +balance, +blockchain, +balance_type, +1 as rank +from {{ source( + 'shared', + 'terra_balances' ) }} - WHERE - block_timestamp :: DATE >= ( - SELECT - DATEADD('day', -1, MAX(DATE)) - FROM - {{ this }}) qualify(ROW_NUMBER() over(PARTITION BY address, currency, blockchain, balance_type, block_timestamp :: DATE - ORDER BY - block_timestamp DESC)) = 1 - ), +where block_timestamp::date >= (select dateadd('day',-1,max(date)) from {{ this }}) +qualify(row_number() over(partition by address, currency,blockchain, balance_type,block_timestamp::date order by block_timestamp desc)) = 1 +), +incremental as ( +select DATE, + address, + currency, + balance_type, + blockchain, + balance +from ( +select DATE, + address, + currency, + balance_type, + blockchain, + balance, +2 as rank +from latest -incremental AS ( - SELECT - DATE, - address, - currency, - balance_type, - blockchain, - balance - FROM - ( - SELECT - DATE, - address, - currency, - balance_type, - blockchain, - balance, - 2 AS RANK - FROM - latest - UNION - SELECT - DATE, - address, - currency, - balance_type, - blockchain, - balance, - 1 AS RANK - FROM - NEW - ) qualify(ROW_NUMBER() over(PARTITION BY address, currency, blockchain, balance_type, DATE - ORDER BY - RANK ASC)) = 1 -), +union -base_balances AS ( +select DATE, + address, + currency, + balance_type, + blockchain, + balance, +1 as rank +from new +) +qualify(row_number() over(partition by address, currency,blockchain, balance_type, date order by rank asc)) = 1 +), +base_balances as ( {% if is_incremental() %} -SELECT - DATE AS block_timestamp, address, currency, balance_type, blockchain, balance -FROM - incremental +select date as block_timestamp, + address, + currency, + balance_type, + blockchain, + balance +from incremental {% else %} -SELECT - block_timestamp, address, currency, balance_type, blockchain, balance -FROM - {{ ref("terra_dbt__balances") }} -{% endif %}), +select block_timestamp, + address, + currency, + balance_type, + blockchain, + balance +from {{source( + 'shared', + 'terra_balances' + ) }} +{% endif %} +), + address_ranges AS ( -address_ranges AS ( SELECT address, currency, @@ -132,7 +107,6 @@ address_ranges AS ( 3, 4 ), - cte_my_date AS ( SELECT HOUR :: DATE AS DATE @@ -144,7 +118,6 @@ cte_my_date AS ( GROUP BY 1 ), - all_dates AS ( SELECT C.date, @@ -160,7 +133,6 @@ all_dates AS ( WHERE A.address IS NOT NULL ), - terra_balances AS ( SELECT address, @@ -170,34 +142,19 @@ terra_balances AS ( 'terra' AS blockchain, balance FROM - {{ ref( - "terra_dbt__balances" - ) }} - base_balances qualify(ROW_NUMBER() over(PARTITION BY address, currency, block_timestamp :: DATE, balance_type + base_balances + qualify(ROW_NUMBER() over(PARTITION BY address, currency, block_timestamp :: DATE, balance_type ORDER BY balance DESC)) = 1 ), - balance_tmp AS ( SELECT d.date, d.address, d.currency, CASE - WHEN d.currency IN ( - 'LUNA', - 'UST', - 'KRT', - 'MNT', - 'SDT', - 'krw', - 'mnt', - 'sdr' - ) THEN b.balance - ELSE b.balance / pow( - 10, - 6 - ) + WHEN d.currency IN ('LUNA', 'UST', 'KRT', 'MNT', 'SDT', 'krw', 'mnt', 'sdr') THEN b.balance + ELSE b.balance / POW(10,6) END AS balance, d.balance_type, d.blockchain @@ -210,14 +167,13 @@ balance_tmp AS ( AND d.balance_type = b.balance_type AND d.blockchain = b.blockchain ) - SELECT DATE, address, currency, balance_type, blockchain, - TRUE AS is_native, + TRUE as is_native, LAST_VALUE( balance ignore nulls ) over( @@ -229,4 +185,4 @@ SELECT DATE ASC rows unbounded preceding ) AS balance FROM - balance_tmp + balance_tmp \ No newline at end of file From d598d17b0305a6298350492d75d0632bd8e44dfd Mon Sep 17 00:00:00 2001 From: desmond-hui <97470747+desmond-hui@users.noreply.github.com> Date: Wed, 2 Mar 2022 07:54:09 -0800 Subject: [PATCH 22/35] rewrite jupyter swaps (#366) * wip * updates to .yml files Co-authored-by: jhuhnke --- .../silver_solana__swaps_jupiter_dex.sql | 382 +++++++++++------- .../silver_solana__swaps_jupiter_dex.yml | 17 +- 2 files changed, 244 insertions(+), 155 deletions(-) diff --git a/models/solana/silver/silver_solana__swaps_jupiter_dex.sql b/models/solana/silver/silver_solana__swaps_jupiter_dex.sql index 3cff0624..79f5360d 100644 --- a/models/solana/silver/silver_solana__swaps_jupiter_dex.sql +++ b/models/solana/silver/silver_solana__swaps_jupiter_dex.sql @@ -25,8 +25,40 @@ WITH jupiter_dex_txs AS ( i.value :programId :: STRING = 'JUP2jxvXaqu7NQY1GmNF4m1vodw12LVXYxbFL2uJvfo' -- jupiter aggregator v2 {% if is_incremental() %} -AND i.ingested_at >= CURRENT_DATE - 2 -AND t.ingested_at >= CURRENT_DATE - 2 +AND i.ingested_at :: DATE >= CURRENT_DATE - 2 +AND t.block_timestamp :: DATE >= CURRENT_DATE - 2 +{% endif %} +), +signers AS ( + SELECT + t.tx_id, + A.value :pubkey :: STRING AS acct, + A.index + FROM + jupiter_dex_txs t, + TABLE(FLATTEN(t.account_keys)) A + WHERE + A.value :signer = TRUE qualify(ROW_NUMBER() over (PARTITION BY t.tx_id + ORDER BY + A.index DESC)) = 1 +), +post_balances_acct_map AS ( + SELECT + t.tx_id, + t.account_keys [b.account_index] :pubkey :: STRING AS middle_acct, + b.owner, + b.mint, + b.decimal, + b.amount + FROM + {{ ref('solana_dbt__post_token_balances') }} + b + INNER JOIN jupiter_dex_txs t + ON t.tx_id = b.tx_id + +{% if is_incremental() %} +WHERE + b.ingested_at :: DATE >= CURRENT_DATE - 2 {% endif %} ), destinations AS ( @@ -38,6 +70,8 @@ destinations AS ( i.index, ii.index AS inner_index, ii.value :parsed :info :destination :: STRING AS destination, + ii.value :parsed :info :authority :: STRING AS authority, + ii.value :parsed :info :source :: STRING AS source, ii.value :parsed :info :amount AS amount, ROW_NUMBER() over ( PARTITION BY i.tx_id @@ -57,180 +91,232 @@ destinations AS ( ii.value :programId :: STRING, '' ) <> '11111111111111111111111111111111' + AND i.instruction :programId :: STRING = 'JUP2jxvXaqu7NQY1GmNF4m1vodw12LVXYxbFL2uJvfo' {% if is_incremental() %} -AND i.ingested_at >= CURRENT_DATE - 2 +AND i.block_timestamp :: DATE >= CURRENT_DATE - 2 {% endif %} ), -post_balances_acct_map AS ( +destination_acct_map AS ( SELECT - t.tx_id, - t.account_keys [b.account_index] :pubkey :: STRING AS middle_acct, - b.owner, - b.mint, - b.decimal, - b.amount + tx_id, + authority, + source FROM - {{ ref('solana_dbt__post_token_balances') }} - b - INNER JOIN jupiter_dex_txs t - ON t.tx_id = b.tx_id - -{% if is_incremental() %} -WHERE - b.ingested_at >= CURRENT_DATE - 2 -{% endif %} + destinations + GROUP BY + 1, + 2, + 3 ), -pre_balances_acct_map AS ( +swaps_tmp_1 AS ( SELECT - t.tx_id, - t.account_keys [b.account_index] :pubkey :: STRING AS middle_acct, - b.owner, - b.mint, - b.decimal, - b.amount + s.acct AS swapper, + COALESCE( + p1.owner, + d2.authority + ) AS destination_owner, + COALESCE( + p1.mint, + p2.mint + ) AS mint, + COALESCE( + p1.decimal, + p2.decimal + ) AS DECIMAL, + d.* FROM - {{ ref('solana_dbt__pre_token_balances') }} - b - INNER JOIN jupiter_dex_txs t - ON t.tx_id = b.tx_id - -{% if is_incremental() %} -WHERE - b.ingested_at >= CURRENT_DATE - 2 -{% endif %} + destinations d + LEFT OUTER JOIN signers s + ON s.acct = d.authority + AND s.tx_id = d.tx_id + LEFT OUTER JOIN post_balances_acct_map p1 + ON p1.middle_acct = d.destination + AND p1.tx_id = d.tx_id + LEFT OUTER JOIN post_balances_acct_map p2 + ON p2.middle_acct = d.source + AND p2.tx_id = d.tx_id + LEFT OUTER JOIN destination_acct_map d2 + ON d2.source = d.destination + AND d2.tx_id = d.tx_id ), -signers AS ( +swapper_min_rn AS ( SELECT - t.tx_id, - A.value :pubkey :: STRING AS acct, - A.index + DISTINCT s.tx_id, + MIN(rn) over ( + PARTITION BY s.tx_id + ) AS min_swapper_rn FROM - jupiter_dex_txs t, - TABLE(FLATTEN(t.account_keys)) A + swaps_tmp_1 s WHERE - A.value :signer = TRUE qualify(ROW_NUMBER() over (PARTITION BY t.tx_id - ORDER BY - A.index DESC)) = 1 + swapper IS NOT NULL ), -signers_info AS ( +swaps_tmp AS ( SELECT - t.tx_id, - t.fee, - s.acct, - b.sol_pre_balances [s.index] AS signer_sol_pre_balance, - b.sol_post_balances [s.index] AS signer_sol_post_balance + s.block_id, + s.block_timestamp, + s.tx_id, + s.succeeded, + s.swapper, + s.destination_owner, + s.mint, + s.decimal, + s.index, + s.inner_index, + s.destination, + s.authority, + s.source, + s.amount, + ROW_NUMBER() over ( + PARTITION BY s.tx_id + ORDER BY + s.index, + s.inner_index + ) AS rn FROM - {{ ref('solana_dbt__sol_balances') }} - b - INNER JOIN jupiter_dex_txs t - ON t.tx_id = b.tx_id - LEFT OUTER JOIN signers s - ON s.tx_id = b.tx_id -- WHERE - -- s.acct NOT IN ( - -- 'GS4FJiLur4dUCjMNGsxoyEyjtxxAkFWKfHBbWLa9uNKg', - -- '5z5h8D5FWUiCJsLjvYL8sWLc9xtP9iTrkEBmckf9AbZY', - -- '4Jfinpcv8KKAB9sTavsxQhxmsUAu7DktNi58VnCz414g' - -- ) -- this is some odd co-signer acct. Is never the actual initiator of the swap - -{% if is_incremental() %} -AND b.ingested_at >= CURRENT_DATE - 2 -{% endif %} + swaps_tmp_1 s + INNER JOIN swapper_min_rn m + ON s.tx_id = m.tx_id + WHERE + s.rn >= m.min_swapper_rn ), -swap_actions AS ( +mint_acct_map AS ( SELECT - d.*, - m.owner, - m.mint, - m.decimal, - MAX(rn) over ( - PARTITION BY d.tx_id - ) AS max_rn + tx_id, + source, + mint, + DECIMAL FROM - destinations d - LEFT OUTER JOIN post_balances_acct_map m - ON d.tx_id = m.tx_id - AND d.destination = m.middle_acct - WHERE + swaps_tmp + GROUP BY + 1, + 2, + 3, + 4 +), +swap_actions AS ( + SELECT + s1.block_id, + s1.block_timestamp, + s1.tx_id, + s1.succeeded, + s1.swapper, + s1.destination_owner, + s1.destination, + s1.source, COALESCE( - d.amount :: STRING, - '-1' - ) <> '0' + s1.mint, + s2.mint + ) AS mint, + COALESCE( + s1.decimal, + s2.decimal + ) AS DECIMAL, + s1.amount :: bigint AS amount, + s1.rn + FROM + swaps_tmp s1 + LEFT OUTER JOIN mint_acct_map s2 + ON s1.destination = s2.source + AND s1.tx_id = s2.tx_id ), -swaps_tmp AS ( +swap_actions_with_refund AS ( + SELECT + s1.*, + s3.mint AS originating_mint, + CASE + WHEN s2.amount < s1.amount THEN s2.amount + ELSE NULL + END AS refund, + MAX(refund) over ( + PARTITION BY s1.tx_id + ) AS max_refund, + s1.amount - COALESCE( + refund, + 0 + ) AS final_amt + FROM + swap_actions s1 + LEFT OUTER JOIN swap_actions s2 + ON s1.tx_id = s2.tx_id + AND s1.swapper = s2.destination_owner + AND s1.mint = s2.mint + AND s1.rn = 1 + LEFT OUTER JOIN swap_actions s3 + ON s1.tx_id = s3.tx_id + AND s3.rn = 1 +), +swap_actions_final AS ( SELECT * FROM - swap_actions + swap_actions_with_refund WHERE - ( - rn = 1 - OR rn = max_rn - ) + swapper IS NOT NULL + OR mint <> originating_mint + OR ( + originating_mint = mint + AND max_refund IS NULL + ) -- need to do this for situations where it appears the user swaps back to the same mint... +), +agg_tmp AS ( + SELECT + block_id, + block_timestamp, + tx_id, + succeeded, + swapper, + mint, + DECIMAL, + SUM(final_amt) AS amt, + MIN(rn) AS rn + FROM + swap_actions_final + GROUP BY + 1, + 2, + 3, + 4, + 5, + 6, + 7 +), +agg AS ( + SELECT + *, + MAX(rn) over ( + PARTITION BY tx_id + ) AS max_rn + FROM + agg_tmp + WHERE + amt <> 0 ) SELECT - s1.tx_id, - s1.block_id, - s1.block_timestamp, - s1.succeeded, - COALESCE( - s2.owner, - s.acct - ) AS swapper, + a1.block_id, + a1.block_timestamp, + a1.tx_id, + a1.succeeded, + a1.swapper, + a1.mint AS from_mint, CASE - WHEN s1.succeeded = FALSE THEN 0 - ELSE GREATEST(COALESCE(s1.amount * pow(10,- s1.decimal), 0), COALESCE((pre_from.amount - post_from.amount) * pow(10,- s1.decimal), 0), COALESCE(s.signer_sol_pre_balance - s.signer_sol_post_balance + s.fee, 0) * pow(10, -9)) - END AS swap_from_amount, - COALESCE( - s1.mint, - 'So11111111111111111111111111111111111111112' - ) AS swap_from_mint, + WHEN a1.succeeded THEN a1.amt * pow( + 10,- a1.decimal + ) + ELSE 0 + END AS from_amt, + a2.mint AS to_mint, CASE - WHEN s1.succeeded = FALSE THEN 0 - ELSE GREATEST(COALESCE(s2.amount * pow(10,- s2.decimal), 0), COALESCE((post_to.amount - pre_to.amount) * pow(10,- s2.decimal), 0), COALESCE(s.signer_sol_post_balance - s.signer_sol_pre_balance + s.fee, 0) * pow(10, -9)) - END AS swap_to_amount, - COALESCE( - s2.mint, - 'So11111111111111111111111111111111111111112' - ) AS swap_to_mint + WHEN a1.succeeded THEN a2.amt * pow( + 10,- a2.decimal + ) + ELSE 0 + END AS to_amt FROM - swaps_tmp s1 - LEFT OUTER JOIN swaps_tmp s2 - ON s1.tx_id = s2.tx_id - AND s1.rn <> s2.rn - LEFT OUTER JOIN signers_info s - ON s.tx_id = s1.tx_id - LEFT OUTER JOIN pre_balances_acct_map pre_from - ON pre_from.tx_id = s1.tx_id - AND pre_from.mint = s1.mint - AND pre_from.owner = COALESCE( - s2.owner, - s.acct - ) - LEFT OUTER JOIN pre_balances_acct_map pre_to - ON pre_to.tx_id = s1.tx_id - AND pre_to.mint = s2.mint - AND pre_to.owner = COALESCE( - s2.owner, - s.acct - ) - LEFT OUTER JOIN post_balances_acct_map post_from - ON post_from.tx_id = s1.tx_id - AND post_from.mint = s1.mint - AND post_from.owner = COALESCE( - s2.owner, - s.acct - ) - LEFT OUTER JOIN post_balances_acct_map post_to - ON post_to.tx_id = s1.tx_id - AND post_to.mint = s2.mint - AND post_to.owner = COALESCE( - s2.owner, - s.acct - ) + agg a1 + LEFT OUTER JOIN agg a2 + ON a1.tx_id = a2.tx_id + AND a1.rn <> a2.rn + AND a2.rn = a2.max_rn WHERE - s1.rn = 1 - -QUALIFY(ROW_NUMBER() over(PARTITION BY s1.block_id, s1.tx_id -ORDER BY - s1.block_timestamp DESC)) = 1 \ No newline at end of file + a1.rn = 1 diff --git a/models/solana/silver/silver_solana__swaps_jupiter_dex.yml b/models/solana/silver/silver_solana__swaps_jupiter_dex.yml index 6b1f11ab..62302e1e 100644 --- a/models/solana/silver/silver_solana__swaps_jupiter_dex.yml +++ b/models/solana/silver/silver_solana__swaps_jupiter_dex.yml @@ -30,19 +30,22 @@ models: description: Address that initiated the swap tests: - not_null - - name: SWAP_FROM_AMOUNT + - name: FROM_AMT description: Total amount of the token sent in to initiate the swap tests: - - not_null - - name: SWAP_FROM_MINT + - not_null: + enabled: FALSE + - name: FROM_MINT description: Token being sent or swapped from tests: - not_null - - name: SWAP_TO_AMOUNT + - name: TO_AMT description: Total amount of the token received in the swap tests: - - not_null - - name: SWAP_TO_MINT + - not_null: + enabled: FALSE + - name: TO_MINT description: Token being received or swapped for tests: - - not_null \ No newline at end of file + - not_null: + enabled: FALSE ## Nulls expected when tx fails \ No newline at end of file From c133c9cbe77616984a40fbecb2d82cdea416eaf4 Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Wed, 2 Mar 2022 13:16:00 -0600 Subject: [PATCH 23/35] gold swaps name change update --- models/solana/gold/solana__swaps.sql | 9 ++++----- models/solana/gold/solana__swaps.yml | 6 ++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/models/solana/gold/solana__swaps.sql b/models/solana/gold/solana__swaps.sql index 99751919..c16af3a6 100644 --- a/models/solana/gold/solana__swaps.sql +++ b/models/solana/gold/solana__swaps.sql @@ -4,15 +4,14 @@ ) }} SELECT - 'jupiter aggregator v2' AS swap_program, block_timestamp, block_id, tx_id, succeeded, swapper, - swap_from_amount, - swap_from_mint, - swap_to_amount, - swap_to_mint + from_amt AS swap_from_amount, + from_mint AS swap_from_mint, + to_amt AS swap_to_amount, + to_mint AS swap_to_mint FROM {{ ref('silver_solana__swaps_jupiter_dex') }} diff --git a/models/solana/gold/solana__swaps.yml b/models/solana/gold/solana__swaps.yml index ed423256..57996450 100644 --- a/models/solana/gold/solana__swaps.yml +++ b/models/solana/gold/solana__swaps.yml @@ -41,8 +41,10 @@ models: - name: SWAP_TO_AMOUNT description: Total amount of the token received in the swap tests: - - not_null + - not_null: + enabled: FALSE - name: SWAP_TO_MINT description: Token being received or swapped for tests: - - not_null \ No newline at end of file + - not_null: + enabled: FALSE \ No newline at end of file From f24789e80833ba3c4e425d98deb722bec158e8ae Mon Sep 17 00:00:00 2001 From: jhuhnke Date: Wed, 2 Mar 2022 13:37:58 -0600 Subject: [PATCH 24/35] updates --- models/solana/gold/solana__swaps.sql | 1 + models/solana/gold/solana__swaps.yml | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/models/solana/gold/solana__swaps.sql b/models/solana/gold/solana__swaps.sql index c16af3a6..fac19268 100644 --- a/models/solana/gold/solana__swaps.sql +++ b/models/solana/gold/solana__swaps.sql @@ -4,6 +4,7 @@ ) }} SELECT + 'jupiter aggregator v2' AS swap_program, block_timestamp, block_id, tx_id, diff --git a/models/solana/gold/solana__swaps.yml b/models/solana/gold/solana__swaps.yml index 57996450..ed423256 100644 --- a/models/solana/gold/solana__swaps.yml +++ b/models/solana/gold/solana__swaps.yml @@ -41,10 +41,8 @@ models: - name: SWAP_TO_AMOUNT description: Total amount of the token received in the swap tests: - - not_null: - enabled: FALSE + - not_null - name: SWAP_TO_MINT description: Token being received or swapped for tests: - - not_null: - enabled: FALSE \ No newline at end of file + - not_null \ No newline at end of file From 324dcdd1c5629d8669b2f5a22e44a298458b575f Mon Sep 17 00:00:00 2001 From: desmond-hui <97470747+desmond-hui@users.noreply.github.com> Date: Wed, 2 Mar 2022 13:54:09 -0800 Subject: [PATCH 25/35] remove to_amt null records (#369) --- models/solana/silver/silver_solana__swaps_jupiter_dex.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/solana/silver/silver_solana__swaps_jupiter_dex.sql b/models/solana/silver/silver_solana__swaps_jupiter_dex.sql index 79f5360d..a2a412d4 100644 --- a/models/solana/silver/silver_solana__swaps_jupiter_dex.sql +++ b/models/solana/silver/silver_solana__swaps_jupiter_dex.sql @@ -320,3 +320,4 @@ FROM AND a2.rn = a2.max_rn WHERE a1.rn = 1 + AND to_amt IS NOT NULL From 1d2d825e9068d7929ad110c258f632dc39c89d35 Mon Sep 17 00:00:00 2001 From: xiuy001 <97179309+xiuy001@users.noreply.github.com> Date: Wed, 2 Mar 2022 17:34:23 -0500 Subject: [PATCH 26/35] created astroport table and the yml testing file (#363) --- models/terra/swap/astroport__swaps.sql | 787 +++++++++++++++++++++++++ models/terra/swap/astroport__swaps.yml | 68 +++ models/terra/swap/terraswap__swaps.sql | 2 +- models/terra/swap/terraswap__swaps.yml | 4 +- 4 files changed, 857 insertions(+), 4 deletions(-) create mode 100644 models/terra/swap/astroport__swaps.sql create mode 100644 models/terra/swap/astroport__swaps.yml diff --git a/models/terra/swap/astroport__swaps.sql b/models/terra/swap/astroport__swaps.sql new file mode 100644 index 00000000..067f1179 --- /dev/null +++ b/models/terra/swap/astroport__swaps.sql @@ -0,0 +1,787 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', block_id, tx_id, msg_index, tx_index)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], + tags = ['snowflake', 'terra', 'astroport', 'swap', 'address_labels'] +) }} + +WITH prices AS ( + + SELECT + DATE_TRUNC('hour',block_timestamp) AS HOUR, + currency, + symbol, + AVG(price_usd) AS price + FROM + {{ ref('terra__oracle_prices') }} + WHERE 1=1 + {% if is_incremental() %} + AND block_timestamp :: DATE >= (SELECT MAX(block_timestamp :: DATE) FROM {{ ref('silver_terra__msgs') }}) + {% endif %} + +GROUP BY + 1, + 2, + 3 +), + +source_msgs AS ( + SELECT + * + FROM {{ ref('silver_terra__msgs') }} + WHERE 1=1 + {% if is_incremental() %} + AND block_timestamp :: DATE >= (SELECT MAX(block_timestamp :: DATE) FROM {{ ref('silver_terra__msgs') }}) + {% endif %} +), + +source_msg_events AS ( + SELECT + * + FROM {{ ref('silver_terra__msg_events') }} + WHERE 1=1 + {% if is_incremental() %} + AND block_timestamp :: DATE >= (SELECT MAX(block_timestamp :: DATE) FROM {{ ref('silver_terra__msgs') }}) + {% endif %} +), + +source_address_labels AS ( + SELECT + * + FROM {{ ref('silver_crosschain__address_labels') }} +), + +--------------- +--Single Swap-- +--------------- +astro_pairs AS ( + SELECT + event_attributes :pair_contract_addr::STRING AS contract_address + FROM source_msg_events + WHERE tx_id IN (SELECT + tx_id + FROM source_msgs + WHERE msg_value :execute_msg :create_pair IS NOT NULL + AND msg_value :contract = 'terra1fnywlw4edny3vw44x04xd67uzkdqluymgreu7g' + ) + AND event_type = 'from_contract' +), + +single_msgs_array_raw_type_0 AS ( + SELECT + blockchain, + chain_id, + block_id, + msg_index, + block_timestamp, + tx_id, + msg_value :sender :: STRING AS sender, + msg_value :execute_msg :send :contract :: STRING AS contract_address + FROM source_msgs + WHERE contract_address in (SELECT contract_address from astro_pairs) + AND tx_status = 'SUCCEEDED' +), + +single_msgs_array_raw_type_1 AS ( + SELECT + blockchain, + chain_id, + block_id, + msg_index, + block_timestamp, + tx_id, + msg_value :sender :: STRING AS sender, + msg_value :contract :: STRING AS contract_address + FROM source_msgs + WHERE msg_value :execute_msg :execute_swap_operations :operations[0] :astro_swap IS NOT NULL + AND msg_value :execute_msg :execute_swap_operations :operations[1] :native_swap IS NULL + AND msg_value :execute_msg :execute_swap_operations :operations[1] :terra_swap IS NULL + AND msg_value :execute_msg :execute_swap_operations :operations[1] :prism_swap IS NULL + AND msg_value :execute_msg :execute_swap_operations :operations[1] :astro_swap IS NULL + AND msg_value :execute_msg :execute_swap_operations :operations[1] :loop_swap IS NULL + AND msg_value :execute_msg :execute_swap_operations :operations[2] :native_swap IS NULL + AND msg_value :execute_msg :execute_swap_operations :operations[2] :terra_swap IS NULL + AND msg_value :execute_msg :execute_swap_operations :operations[2] :prism_swap IS NULL + AND msg_value :execute_msg :execute_swap_operations :operations[2] :astro_swap IS NULL + AND msg_value :execute_msg :execute_swap_operations :operations[2] :loop_swap IS NULL + AND tx_status = 'SUCCEEDED' +), + +single_msgs_array_raw_type_2 AS ( + SELECT + blockchain, + chain_id, + block_id, + msg_index, + block_timestamp, + tx_id, + COALESCE(event_attributes :"0_sender" :: STRING, event_attributes :"sender" :: STRING) AS sender, + COALESCE(event_attributes :"0_contract_address" :: STRING, event_attributes :"contract_address" :: STRING) AS contract_address + FROM source_msg_events + WHERE event_type = 'from_contract' AND event_attributes:"maker_fee_amount" IS NOT NULL AND tx_id NOT IN (SELECT tx_id FROM single_msgs_array_raw_type_1) +), + +single_msgs_array_raw AS ( + SELECT + blockchain, + chain_id, + block_id, + msg_index, + block_timestamp, + tx_id, + sender, + contract_address, + event_attributes + FROM single_msgs_array_raw_type_0 + + UNION ALL + + SELECT + blockchain, + chain_id, + block_id, + msg_index, + block_timestamp, + tx_id, + sender, + contract_address, + event_attributes + FROM single_msgs_array_raw_type_1 + + UNION ALL + + SELECT + blockchain, + chain_id, + block_id, + msg_index, + block_timestamp, + tx_id, + sender, + contract_address, + event_attributes + FROM single_msgs_array_raw_type_2 +), + +single_msgs_array_raw_events_raw AS ( + SELECT + blockchain, + chain_id, + block_id, + tx_id, + block_timestamp, + event_type, + event_attributes, + msg_index, + key, + value, + split_part(key, '_', 1) AS tx_index, + MAX(split_part(key, '_', 1)) OVER (PARTITION BY tx_id) AS max_tx_index, + SUBSTRING(key, LEN(split_part(key, '_', 1))+2, LEN(key)) AS tx_subtype + FROM source_msg_events + , lateral flatten ( input => event_attributes) + WHERE event_type = 'from_contract' +), + +single_msgs_array_raw_events_index AS ( + SELECT + tx_id, + block_timestamp, + msg_index, + tx_index AS tx_index + FROM single_msgs_array_raw_events_raw + WHERE value = 'swap' +), + +single_msgs_array_raw_events_value AS ( + SELECT + a.tx_id, + a.msg_index, + RANK() OVER (PARTITION BY a.tx_id, a.msg_index ORDER BY a.tx_index ASC) AS tx_index, + a.value::STRING AS pool_address + FROM single_msgs_array_raw_events_raw a + INNER JOIN single_msgs_array_raw_events_index b + ON a.tx_id = b.tx_id AND a.tx_index = b.tx_index AND a.msg_index = b.msg_index + WHERE tx_subtype = 'contract_address' +), + +single_msgs_array_raw_updated AS ( + SELECT + blockchain, + chain_id, + block_id, + a.msg_index, + block_timestamp, + a.tx_id, + sender, + COALESCE(b.pool_address, a.contract_address) AS contract_address + FROM single_msgs_array_raw a + LEFT JOIN single_msgs_array_raw_events_value b + ON a.tx_id = b.tx_id AND a.msg_index = b.msg_index + WHERE NOT(sender IS NULL AND b.pool_address IS NULL) +), + +single_msg_array_events AS ( + SELECT + source_msg_events.msg_index, + source_msg_events.tx_id, + source_msg_events.event_attributes :contract_address :: STRING AS pool_address + FROM source_msg_events + INNER JOIN single_msgs_array_raw_updated + ON source_msg_events.tx_id = single_msgs_array_raw_updated.tx_id + WHERE event_type = 'from_contract' +), + +single_msgs_array AS ( + SELECT + blockchain, + chain_id, + block_id, + single_msgs_array_raw_updated.msg_index, + block_timestamp, + single_msgs_array_raw_updated.tx_id, + sender, + CASE + WHEN single_msg_array_events.pool_address IS NULL + THEN single_msgs_array_raw_updated.contract_address + ELSE single_msg_array_events.pool_address + END AS pool_address + FROM single_msgs_array_raw_updated + LEFT JOIN single_msg_array_events + ON single_msgs_array_raw_updated.tx_id = single_msg_array_events.tx_id + AND single_msgs_array_raw_updated.msg_index = single_msg_array_events.msg_index +), + +msgs AS ( + SELECT + blockchain, + chain_id, + block_id, + msg_index, + block_timestamp, + tx_id, + sender, + pool_address + FROM single_msgs_array +), + +events AS ( + SELECT + msg_index, + tx_id, + event_attributes :sender :: STRING AS sender, + event_attributes :offer_amount :: numeric / pow(10,6) AS offer_amount, + event_attributes :offer_asset :: STRING AS offer_currency, + event_attributes :return_amount :: numeric / pow(10,6) AS return_amount, + event_attributes :ask_asset :: STRING AS return_currency + FROM source_msg_events + WHERE event_type = 'from_contract' + AND tx_id IN(SELECT DISTINCT tx_id FROM msgs) + AND event_attributes :offer_amount IS NOT NULL +), + +swaps AS ( + SELECT + m.blockchain, + chain_id, + block_id, + m.msg_index, + 0 AS tx_index, --Because there is always only 1 tx index here, so it should be 0 + block_timestamp, + m.tx_id, + COALESCE(m.sender, e.sender) AS sender, + offer_amount, + offer_amount * o.price AS offer_amount_usd, + offer_currency, + return_amount, + return_amount * r.price AS return_amount_usd, + return_currency, + pool_address, + l.address_name AS pool_name + FROM + msgs m + + JOIN events e + ON m.tx_id = e.tx_id + AND m.msg_index = e.msg_index + + LEFT OUTER JOIN prices o + ON DATE_TRUNC('hour',m.block_timestamp) = o.hour + AND e.offer_currency = o.currency + + LEFT OUTER JOIN prices r + ON DATE_TRUNC('hour',m.block_timestamp) = r.hour + AND e.return_currency = r.currency + + LEFT OUTER JOIN source_address_labels l + ON pool_address = l.address + AND l.blockchain = 'terra' + AND l.creator = 'flipside' +), + +----------------- +--Multiple Swap-- +----------------- +-- First type of multiple swaps, the pool address works for here +msgs_multi_swaps_raw_type_2_msg AS ( + SELECT + blockchain, + chain_id, + block_id, + msg_index, + index AS tx_index, + MAX(index) OVER (PARTITION BY tx_id) AS max_tx_index, + block_timestamp, + tx_id, + msg_value :sender :: STRING AS sender, + msg_value :contract :: STRING AS pool_address, + msg_value, + OBJECT_KEYS(value)[0]::STRING AS tx_type + FROM source_msgs + , lateral flatten ( msg_value :execute_msg :execute_swap_operations :operations) + WHERE msg_value :execute_msg :execute_swap_operations :operations IS NOT NULL + AND tx_status = 'SUCCEEDED' + AND OBJECT_KEYS(value)[0]::STRING IN ('astro_swap', 'astroswap') + + UNION ALL + + SELECT + blockchain, + chain_id, + block_id, + msg_index, + index AS tx_index, + MAX(index) OVER (PARTITION BY tx_id) AS max_tx_index, + block_timestamp, + tx_id, + msg_value :sender :: STRING AS sender, + msg_value :execute_msg :send :contract :: STRING AS pool_address, + msg_value, + OBJECT_KEYS(value)[0]::STRING AS tx_type + FROM source_msgs + , lateral flatten ( msg_value :execute_msg :send :msg :execute_swap_operations :operations) + WHERE msg_value :execute_msg :send :msg :execute_swap_operations :operations IS NOT NULL + AND tx_status = 'SUCCEEDED' + AND OBJECT_KEYS(value)[0]::STRING IN ('astro_swap', 'astroswap') +), + +msgs_multi_swaps_raw_type_2_events_raw AS ( + SELECT + tx_id, + block_timestamp, + event_type, + event_attributes, + msg_index, + key, + value, + split_part(key, '_', 1) AS tx_index, + MAX(split_part(key, '_', 1)) OVER (PARTITION BY tx_id) AS max_tx_index, + SUBSTRING(key, LEN(split_part(key, '_', 1))+2, LEN(key)) AS tx_subtype + FROM source_msg_events + , lateral flatten ( input => event_attributes) + WHERE event_type = 'from_contract' +), + +msgs_multi_swaps_raw_type_2_events_index AS ( + SELECT + tx_id, + block_timestamp, + msg_index, + tx_index AS tx_index + FROM msgs_multi_swaps_raw_type_2_events_raw + WHERE value = 'swap' +), + +msgs_multi_swaps_raw_type_2_events_value AS ( + SELECT + a.tx_id, + a.msg_index, + RANK() OVER (PARTITION BY a.tx_id, a.msg_index ORDER BY a.tx_index ASC) AS tx_index, + a.value::STRING AS pool_address + FROM msgs_multi_swaps_raw_type_2_events_raw a + INNER JOIN msgs_multi_swaps_raw_type_2_events_index b + ON a.tx_id = b.tx_id AND a.tx_index = b.tx_index AND a.msg_index = b.msg_index + WHERE tx_subtype = 'contract_address' +), + +msgs_multi_swaps_raw_type_2 AS ( + SELECT + blockchain, + chain_id, + block_id, + a.msg_index, + a.tx_index, + max_tx_index, + block_timestamp, + a.tx_id, + sender, + COALESCE(a.pool_address, b.pool_address) AS pool_address, + msg_value + FROM msgs_multi_swaps_raw_type_2_msg a + LEFT JOIN msgs_multi_swaps_raw_type_2_events_value b + ON a.tx_id = b.tx_id AND a.msg_index = b.msg_index AND a.tx_index = b.tx_index +), + +msgs_multi_swaps_raw AS ( + SELECT + blockchain, + chain_id, + block_id, + msg_index, + tx_index, + max_tx_index, + block_timestamp, + tx_id, + sender, + pool_address, + msg_value + FROM msgs_multi_swaps_raw_type_2 +), + +msgs_multi_swaps AS ( + SELECT + blockchain, + chain_id, + block_id, + msg_index, + tx_index, + block_timestamp, + tx_id, + sender, + pool_address, + msg_value + FROM msgs_multi_swaps_raw + WHERE max_tx_index > 0 +), + +events_multi_swaps_raw_type_1 AS ( + SELECT + tx_id, + event_type, + event_attributes, + msg_index, + event_index, + tx_index, + tx_subtype, + MAX(value) AS value + FROM ( + SELECT + tx_id, + event_type, + event_attributes, + msg_index, + event_index, + value, + split_part(key, '_', 1) AS tx_index, + MAX(split_part(key, '_', 1)) OVER (PARTITION BY tx_id) AS max_tx_index, + SUBSTRING(key, LEN(split_part(key, '_', 1))+2, LEN(key)) AS tx_subtype + FROM source_msg_events + , lateral flatten ( input => event_attributes) + WHERE event_type = 'from_contract' + AND event_attributes :"0_offer_amount" IS NOT NULL AND event_attributes :"1_offer_amount" IS NOT NULL + ) tbl + WHERE tx_subtype IN ('ask_asset', 'offer_amount', 'offer_asset', 'return_amount', 'contract_address') + GROUP BY 1,2,3,4,5,6,7 +), + +events_multi_swaps_type_1 AS ( + SELECT + tx_id, + msg_index, + tx_index, + event_index, + "'offer_amount'":: numeric / pow(10,6) AS offer_amount, + "'offer_asset'"::STRING AS offer_currency, + "'return_amount'":: numeric / pow(10,6) AS return_amount, + "'ask_asset'"::STRING AS return_currency, + "'contract_address'"::STRING AS contract_address + FROM events_multi_swaps_raw_type_1 + pivot (max(value) for tx_subtype IN ('ask_asset', 'offer_amount', 'offer_asset', 'return_amount', 'contract_address')) p + ORDER BY + tx_id, + tx_index, + msg_index, + event_index +), + +events_multi_swaps_raw_type_2 AS ( + SELECT + tx_id, + event_type, + event_attributes, + msg_index, + event_index, + t1.value, + CONCAT(t0.key, '_', t1.key) AS key, + -1 AS tx_index + FROM source_msg_events + , lateral flatten ( input => event_attributes) as t0 + , lateral flatten ( input => t0.value[0] , mode => 'object') t1 + WHERE event_type = 'swap' + AND t0.key IN ('offer', 'swap_coin', 'contract_address') +), + +events_multi_swaps_type_2 AS ( + SELECT + tx_id, + msg_index, + tx_index, + event_index, + offer_amount :: numeric / pow(10,6) AS offer_amount, + offer_currency, + return_amount :: numeric / pow(10,6) AS return_amount, + return_currency, + contract_address + FROM ( + SELECT + tx_id, + msg_index, + tx_index, + event_index, + "'offer_amount'" AS offer_amount, + "'offer_denom'"::STRING AS offer_currency, + "'swap_coin_amount'" AS return_amount, + "'swap_coin_denom'"::STRING AS return_currency, + event_attributes :trader ::STRING AS contract_address + FROM events_multi_swaps_raw_type_2 + pivot (max(value) for key IN ('offer_amount', 'offer_denom', 'swap_coin_amount', 'swap_coin_denom', 'contract_address')) p + ORDER BY + tx_id, + tx_index, + msg_index, + event_index + ) +), + +events_multi_swaps_raw_type_3 AS ( + SELECT + tx_id, + event_type, + event_attributes, + msg_index, + event_index, + 0 AS tx_index, + key, + MAX(value) AS value + FROM ( + SELECT + tx_id, + event_type, + event_attributes, + msg_index, + event_index, + key, + value + FROM source_msg_events + , lateral flatten ( input => event_attributes) + WHERE event_type = 'from_contract' + AND event_attributes :"offer_amount" IS NOT NULL + ) tbl + WHERE key IN ('ask_asset', 'offer_amount', 'offer_asset', 'return_amount', '2_contract_address', 'contract_address') + GROUP BY 1,2,3,4,5,6,7 +), + +events_multi_swaps_type_3 AS ( + SELECT + tx_id, + msg_index, + tx_index, + event_index, + offer_amount :: numeric / pow(10,6) AS offer_amount, + offer_currency, + return_amount :: numeric / pow(10,6) AS return_amount, + return_currency, + contract_address + FROM ( + SELECT + tx_id, + msg_index, + tx_index, + event_index, + "'offer_amount'" AS offer_amount, + "'offer_asset'"::STRING AS offer_currency, + "'return_amount'" AS return_amount, + "'ask_asset'"::STRING AS return_currency, + COALESCE("'2_contract_address'"::STRING, "'contract_address'"::STRING) AS contract_address + FROM events_multi_swaps_raw_type_3 + pivot (max(value) for key IN ('ask_asset', 'offer_amount', 'offer_asset', 'return_amount', '2_contract_address', 'contract_address')) p + ORDER BY + tx_id, + tx_index, + msg_index, + event_index + ) + WHERE offer_amount IS NOT NULL +), + +events_multi_swaps AS ( + SELECT + tx_id, + msg_index, + event_index, + (tx_index - 1) AS tx_index, + offer_amount, + offer_currency, + return_amount, + return_currency, + contract_address + FROM ( + SELECT + tx_id, + msg_index, + event_index, + RANK() OVER(PARTITION BY tx_id, msg_index ORDER BY tx_index ASC)::integer AS tx_index, + offer_amount, + offer_currency, + return_amount, + return_currency, + contract_address + FROM ( + SELECT + tx_id, + msg_index, + tx_index::integer AS tx_index, + event_index, + offer_amount, + offer_currency, + return_amount, + return_currency, + contract_address + FROM events_multi_swaps_type_1 + + UNION ALL + + SELECT + tx_id, + msg_index, + tx_index::integer AS tx_index, + event_index, + offer_amount, + offer_currency, + return_amount, + return_currency, + contract_address + FROM events_multi_swaps_type_2 + + UNION ALL + + SELECT + tx_id, + msg_index, + tx_index::integer AS tx_index, + event_index, + offer_amount, + offer_currency, + return_amount, + return_currency, + contract_address + FROM events_multi_swaps_type_3 + ) + ) + WHERE offer_amount IS NOT NULL +), + +swaps_multi_swaps AS ( + SELECT + a.blockchain, + chain_id, + block_id, + msg_index, + m_tx_index AS tx_index, + block_timestamp, + tx_id, + sender, + offer_amount, + offer_amount_usd, + offer_currency, + return_amount, + return_amount_usd, + return_currency, + pool_address, + l.address_name AS pool_name + FROM ( + SELECT + m.blockchain, + chain_id, + block_id, + m.msg_index, + m.tx_index AS m_tx_index, + e.tx_index AS e_tx_index, + block_timestamp, + m.tx_id, + sender, + offer_amount, + offer_amount * o.price AS offer_amount_usd, + offer_currency, + return_amount, + return_amount * r.price AS return_amount_usd, + return_currency, + CASE WHEN m.pool_address IS NOT NULL THEN m.pool_address ELSE e.contract_address END AS pool_address + FROM + msgs_multi_swaps m + + LEFT JOIN events_multi_swaps e + ON m.tx_id = e.tx_id + AND m.msg_index = e.msg_index + + LEFT OUTER JOIN prices o + ON DATE_TRUNC('hour',m.block_timestamp) = o.hour + AND e.offer_currency = o.currency + + LEFT OUTER JOIN prices r + ON DATE_TRUNC('hour',m.block_timestamp) = r.hour + AND e.return_currency = r.currency + ) a + LEFT OUTER JOIN source_address_labels l + ON a.pool_address = l.address + AND l.blockchain = 'terra' + AND l.creator = 'flipside' + WHERE m_tx_index = e_tx_index +) + +SELECT DISTINCT * FROM ( + SELECT + BLOCKCHAIN, + CHAIN_ID, + BLOCK_ID, + MSG_INDEX, + TX_INDEX, + BLOCK_TIMESTAMP, + TX_ID, + SENDER, + OFFER_AMOUNT::FLOAT AS OFFER_AMOUNT, + OFFER_AMOUNT_USD, + OFFER_CURRENCY, + RETURN_AMOUNT, + RETURN_AMOUNT_USD, + RETURN_CURRENCY, + POOL_ADDRESS, + POOL_NAME + FROM swaps + WHERE OFFER_CURRENCY NOT LIKE 'cw20:terra%' -- Remove PRISM contract + AND RETURN_CURRENCY NOT LIKE 'cw20:terra%' -- Remove PRISM contract + + UNION ALL + + SELECT + BLOCKCHAIN, + CHAIN_ID, + BLOCK_ID, + MSG_INDEX, + TX_INDEX, + BLOCK_TIMESTAMP, + TX_ID, + SENDER, + OFFER_AMOUNT::FLOAT AS OFFER_AMOUNT, + OFFER_AMOUNT_USD, + OFFER_CURRENCY, + RETURN_AMOUNT, + RETURN_AMOUNT_USD, + RETURN_CURRENCY, + POOL_ADDRESS, + POOL_NAME + FROM swaps_multi_swaps + WHERE OFFER_CURRENCY NOT LIKE 'cw20:terra%' -- Remove PRISM contract + AND RETURN_CURRENCY NOT LIKE 'cw20:terra%' -- Remove PRISM contract +) diff --git a/models/terra/swap/astroport__swaps.yml b/models/terra/swap/astroport__swaps.yml new file mode 100644 index 00000000..88aacd9d --- /dev/null +++ b/models/terra/swap/astroport__swaps.yml @@ -0,0 +1,68 @@ +version: 2 +models: + - name: astroport__swaps + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - CHAIN_ID + - TX_ID + - MSG_INDEX + - TX_INDEX + columns: + - name: BLOCKCHAIN + tests: + - not_null + - name: CHAIN_ID + tests: + - not_null + - name: BLOCK_ID + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: TX_ID + tests: + - not_null + - name: SENDER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: terra[0-9a-z]{39,39} + - name: OFFER_AMOUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - decimal + - float + - name: OFFER_AMOUNT_USD + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - decimal + - float + - name: OFFER_CURRENCY + tests: + - not_null + - name: RETURN_AMOUNT + tests: + - not_null + - name: RETURN_AMOUNT_USD + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - decimal + - float + - name: RETURN_CURRENCY + tests: + - not_null + - name: POOL_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: terra[0-9a-z]{39,39} \ No newline at end of file diff --git a/models/terra/swap/terraswap__swaps.sql b/models/terra/swap/terraswap__swaps.sql index ef806957..aae9f0c8 100644 --- a/models/terra/swap/terraswap__swaps.sql +++ b/models/terra/swap/terraswap__swaps.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = "CONCAT_WS('-', block_id, tx_id)", + unique_key = "CONCAT_WS('-', block_id, tx_id, msg_index, tx_index)", incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE'], tags = ['snowflake', 'terra', 'terraswap', 'swap', 'address_labels'] diff --git a/models/terra/swap/terraswap__swaps.yml b/models/terra/swap/terraswap__swaps.yml index 0aa9e007..07e55b26 100644 --- a/models/terra/swap/terraswap__swaps.yml +++ b/models/terra/swap/terraswap__swaps.yml @@ -65,6 +65,4 @@ models: tests: - not_null - dbt_expectations.expect_column_values_to_match_regex: - regex: terra[0-9a-z]{39,39} - - name: POOL_NAME - tests: + regex: terra[0-9a-z]{39,39} \ No newline at end of file From 55c4a6e768340743cd069791132c4b2cb43e7417 Mon Sep 17 00:00:00 2001 From: JAMES MISSION Date: Wed, 2 Mar 2022 15:26:21 -0800 Subject: [PATCH 27/35] make position collected fees full refresh --- .../uniswapv3_position_collected_fees.sql | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/macros/uniswapv3/uniswapv3_position_collected_fees.sql b/macros/uniswapv3/uniswapv3_position_collected_fees.sql index 193a04b0..576b7042 100644 --- a/macros/uniswapv3/uniswapv3_position_collected_fees.sql +++ b/macros/uniswapv3/uniswapv3_position_collected_fees.sql @@ -11,11 +11,7 @@ token1 FROM {{ src_pools_table }} p WHERE - {% if is_incremental() %} - p.block_timestamp >= getdate() - interval '7 days' - {% else %} p.block_timestamp >= getdate() - interval '9 months' - {% endif %} ), pool_txs as ( SELECT @@ -25,11 +21,7 @@ INNER JOIN pools p ON p.pool_address = ee.contract_address WHERE event_name = 'Collect' - {% if is_incremental() %} - AND ee.block_timestamp >= getdate() - interval '7 days' - {% else %} AND ee.block_timestamp >= getdate() - interval '9 months' - {% endif %} ), burns as ( SELECT @@ -43,11 +35,7 @@ ee.tx_id IN (SELECT tx_id FROM pool_txs) AND event_name = 'Burn' AND (ee.event_inputs:amount0 > 0 or ee.event_inputs:amount1 > 0) - {% if is_incremental() %} - AND ee.block_timestamp >= getdate() - interval '7 days' - {% else %} AND ee.block_timestamp >= getdate() - interval '9 months' - {% endif %} ), -- Get nf position info by looking at corresponding nf pos event nf_positions as ( @@ -62,11 +50,7 @@ AND contract_address NOT IN (SELECT pool_address FROM pool_txs) AND event_inputs:tokenId is not null AND event_name = 'Collect' - {% if is_incremental() %} - AND ee.block_timestamp >= getdate() - interval '7 days' - {% else %} AND ee.block_timestamp >= getdate() - interval '9 months' - {% endif %} ), lp_providers as ( SELECT * FROM ( @@ -152,11 +136,7 @@ -- in a tx, and there are burn events we want to ensure we're pairing up the burns against collect -- events with actual amounts. Otherwise we end up with negative collects. and (ee.event_inputs:amount0 > 0 or ee.event_inputs:amount1 > 0) - {% if is_incremental() %} - AND ee.block_timestamp >= getdate() - interval '7 days' - {% else %} AND ee.block_timestamp >= getdate() - interval '9 months' - {% endif %} HAVING (amount0_adjusted >= 0 or amount1_adjusted >= 0) {%- endmacro %} \ No newline at end of file From ed345c439d59d77c660a2200efae1663bcbab690 Mon Sep 17 00:00:00 2001 From: Ryan-Loofy <63126328+Ryan-Loofy@users.noreply.github.com> Date: Thu, 3 Mar 2022 10:32:05 -0500 Subject: [PATCH 28/35] An 665/tx from to terra updated (#365) * Proxy contracts added to the seed file * missing balances contracts added to seed file * Seed file moved to SILVER and NUL issue fixed * Added tx_from and tx_to to all Terra Transactions Models - tests added to all models as well * updated run_ids per model and removed select * * fixed failing tests Co-authored-by: jhuhnke --- data/silver__ethereum_contracts_backfill.csv | 2 +- models/terra/dbt/terra_dbt__transactions.sql | 13 ++++++++++--- models/terra/gold/terra__transactions.sql | 2 ++ models/terra/gold/terra__transactions.yml | 3 +++ .../secure_views/terra_sv__transactions.sql | 2 ++ .../secure_views/terra_sv__transactions.yml | 12 +++++++++--- .../silver/silver_terra__transactions.sql | 19 ++++++++++++++++++- .../silver/silver_terra__transactions.yml | 3 +++ 8 files changed, 48 insertions(+), 8 deletions(-) diff --git a/data/silver__ethereum_contracts_backfill.csv b/data/silver__ethereum_contracts_backfill.csv index 3eb0be97..5261a8bd 100644 --- a/data/silver__ethereum_contracts_backfill.csv +++ b/data/silver__ethereum_contracts_backfill.csv @@ -34870,4 +34870,4 @@ block_id,block_timestamp,creator_address,contract_address,logic_address,symbol,d 14039499,2022-01-20T01:25:54Z,,0x63cbD1858BD79DE1A06C3C26462DB360b834912D,0xfCF1aBC45A264D1172AF928ff04D3413bbC11A9f,eDOUGH,18,PProxy,,"{'MAX_VESTING_ENTRIES': 260, 'STAKE_DURATION': 36, 'decimals': 18, 'dough': '0xad32A8e6220741182940c5aBF610bDE99E737b2D', 'name': 'PieDAO Escrowed DOUGH', 'owner': '0x6458A23B020f489651f2777Bd849ddEd34DfCcd2', 'sharesTimeLock': '0x6Bd0D8c8aD8D3F1f97810d5Cc57E9296db73DC45', 'symbol': 'eDOUGH', 'totalEscrowedBalance': 9322667155262482001200519, 'totalSupply': 9322667155262482001200519}" 14039499,2022-01-20T01:25:54Z,,0x8D1ce361eb68e9E05573443C407D4A3Bed23B033,0x706F00ea85a71EB5d7C2ce2aD61DbBE62b616435,DEFI++,18,PProxyPausable,,"{'decimals': 18, 'getBPool': '0xd485e6a0389A42D75f4b00EcE91fc02340B73938', 'getCap': 10000000000000000000000000, 'getCircuitBreaker': '0x0000000000000000000000000000000000000000', 'getController': '0x6458A23B020f489651f2777Bd849ddEd34DfCcd2', 'getDenormalizedWeights': [35000000000000000000, 15000000000000000000], 'getFeeRecipient': '0x0000000000000000000000000000000000000000', 'getNewToken': ['0x0000000000000000000000000000000000000000', False, 0, 0, 0], 'getPublicSwapSetter': '0x6458A23B020f489651f2777Bd849ddEd34DfCcd2', 'getSwapFee': 10000000000000000, 'getTokenBinder': '0x6458A23B020f489651f2777Bd849ddEd34DfCcd2', 'getTokens': ['0x78F225869c08d478c34e5f645d07A87d3fe8eb78', '0xaD6A626aE2B43DCb1B39430Ce496d2FA0365BA9C'], 'name': 'PieDAO DEFI++', 'symbol': 'DEFI++', 'totalSupply': 365425429397430428382145}" 14039499,2022-01-20T01:25:54Z,,0xaD6A626aE2B43DCb1B39430Ce496d2FA0365BA9C,0x706F00ea85a71EB5d7C2ce2aD61DbBE62b616435,DEFI+S,18,PProxyPausable,,"{'decimals': 18, 'getAnnualFee': 7000000000000000, 'getBPool': '0x94743cfAa3FDC62e9693572314B5ee377EBa5d11', 'getCap': 5000000000000000000000000, 'getCircuitBreaker': '0x0000000000000000000000000000000000000000', 'getController': '0x6458A23B020f489651f2777Bd849ddEd34DfCcd2', 'getDenormalizedWeights': [6875825858394830000, 10755714301882800000, 13555533842553300000, 15000000000000000000, 1604619455069070000, 2208306542100000000], 'getFeeRecipient': '0xAF2fE0d4fe879066B2BaA68d9e56cC375DF22815', 'getNewToken': ['0x0000000000000000000000000000000000000000', False, 0, 0, 0], 'getPublicSwapSetter': '0x6458A23B020f489651f2777Bd849ddEd34DfCcd2', 'getSwapFee': 1000000000000, 'getTokenBinder': '0x6458A23B020f489651f2777Bd849ddEd34DfCcd2', 'getTokens': ['0xba100000625a3754423978a60c9317c58a424e3D', '0xBBbbCA6A901c926F240b89EacB641d8Aec7AEafD', '0x408e41876cCCDC0F92210600ef50372656052a38', '0x04Fa0d235C4abf4BcF4787aF4CF447DE572eF828', '0xec67005c4E498Ec7f55E092bd1d35cbC47C91892', '0x89Ab32156e46F46D02ade3FEcbe5Fc4243B9AAeD'], 'name': 'PieDAO DEFI Small Cap', 'symbol': 'DEFI+S', 'totalSupply': 260875387716765753838868}" -13811226,2021-12-15T17:59:33Z,,0x9BE89D2a4cd102D8Fecc6BF9dA793be995C22541,0x9F344834752cb3a8C54c3DdCd41Da4042b10D0b9,BBTC,8,AdminUpgradeabilityProxy,,"{'name': 'Binance Wrapped BTC', 'totalSupply': 990000000000, 'decimals': 8, 'owner': '0xD183F2BBF8b28d9fec8367cb06FE72B88778C86B', 'symbol': 'BBTC', 'pendingOwner': '0x0000000000000000000000000000000000000000'}" \ No newline at end of file +13811226,2021-12-15T17:59:33Z,,0x9BE89D2a4cd102D8Fecc6BF9dA793be995C22541,0x9F344834752cb3a8C54c3DdCd41Da4042b10D0b9,BBTC,8,AdminUpgradeabilityProxy,,"{'name': 'Binance Wrapped BTC', 'totalSupply': 990000000000, 'decimals': 8, 'owner': '0xD183F2BBF8b28d9fec8367cb06FE72B88778C86B', 'symbol': 'BBTC', 'pendingOwner': '0x0000000000000000000000000000000000000000'}" diff --git a/models/terra/dbt/terra_dbt__transactions.sql b/models/terra/dbt/terra_dbt__transactions.sql index 0161d5e9..fac0f8d7 100644 --- a/models/terra/dbt/terra_dbt__transactions.sql +++ b/models/terra/dbt/terra_dbt__transactions.sql @@ -16,9 +16,14 @@ WITH base_tables AS ( 'prod_terra_sink_645110886' ) }} WHERE - record_content :model :name :: STRING IN ( - 'terra_tx_model', - 'terra-5_tx_model' + ( + record_content :model :name :: STRING = 'terra_tx_model' + AND record_content:model.run_id = 'v2022.03.02.0' + ) + OR + ( + record_content :model :name :: STRING = 'terra-5_tx_model' + AND record_content:model.run_id = 'v2022.01.14.0' ) {% if is_incremental() %} @@ -46,6 +51,8 @@ SELECT -- Post Columbus-4: txhash t.value :tx_id :: STRING ) AS tx_id, + t.value :tx_from as tx_from, + t.value :tx_to as tx_to, t.value :tx_type :: STRING AS tx_type, t.value :tx_module :: STRING AS tx_module, t.value :tx_status :: STRING AS tx_status, diff --git a/models/terra/gold/terra__transactions.sql b/models/terra/gold/terra__transactions.sql index 8d9493dc..462b2237 100644 --- a/models/terra/gold/terra__transactions.sql +++ b/models/terra/gold/terra__transactions.sql @@ -9,6 +9,8 @@ SELECT blockchain, chain_id, tx_id, + tx_from, + tx_to, tx_type, tx_status, tx_status_msg, diff --git a/models/terra/gold/terra__transactions.yml b/models/terra/gold/terra__transactions.yml index b576bb67..6ce51aa7 100644 --- a/models/terra/gold/terra__transactions.yml +++ b/models/terra/gold/terra__transactions.yml @@ -42,6 +42,9 @@ models: - name: TX_ID tests: - not_null + - name: TX_FROM + tests: + - not_null - name: TX_MODULE tests: - not_null diff --git a/models/terra/secure_views/terra_sv__transactions.sql b/models/terra/secure_views/terra_sv__transactions.sql index 7eca9206..23852624 100644 --- a/models/terra/secure_views/terra_sv__transactions.sql +++ b/models/terra/secure_views/terra_sv__transactions.sql @@ -10,6 +10,8 @@ SELECT blockchain, chain_id, tx_id, + tx_from, + tx_to, tx_type, tx_status, tx_status_msg, diff --git a/models/terra/secure_views/terra_sv__transactions.yml b/models/terra/secure_views/terra_sv__transactions.yml index d7a8d1ef..d9cdf1df 100644 --- a/models/terra/secure_views/terra_sv__transactions.yml +++ b/models/terra/secure_views/terra_sv__transactions.yml @@ -24,7 +24,8 @@ models: - not_null - name: CODESPACE tests: - - not_null + - not_null: + enabled: False - name: FEE tests: - not_null @@ -36,10 +37,14 @@ models: - not_null - name: TX_CODE tests: - - not_null + - not_null: + enabled: False - name: TX_ID tests: - not_null + - name: TX_FROM + tests: + - not_null - name: TX_MODULE tests: - not_null @@ -48,7 +53,8 @@ models: - not_null - name: TX_STATUS_MSG tests: - - not_null + - not_null: + enabled: False - name: TX_TYPE tests: - not_null diff --git a/models/terra/silver/silver_terra__transactions.sql b/models/terra/silver/silver_terra__transactions.sql index a61177f6..9a19bce2 100644 --- a/models/terra/silver/silver_terra__transactions.sql +++ b/models/terra/silver/silver_terra__transactions.sql @@ -7,7 +7,24 @@ ) }} SELECT - * + system_created_at, + _inserted_timestamp, + blockchain, + block_id, + block_timestamp, + chain_id, + codespace, + tx_id, + tx_from, + tx_to, + tx_type, + tx_module, + tx_status, + tx_status_msg, + tx_code, + fee, + gas_wanted, + gas_used FROM {{ ref('terra_dbt__transactions') }} diff --git a/models/terra/silver/silver_terra__transactions.yml b/models/terra/silver/silver_terra__transactions.yml index 77c403a6..9389f6dd 100644 --- a/models/terra/silver/silver_terra__transactions.yml +++ b/models/terra/silver/silver_terra__transactions.yml @@ -46,6 +46,9 @@ models: - name: TX_ID tests: - not_null + - name: TX_FROM + tests: + - not_null - name: TX_MODULE tests: - not_null From 882965a026e2138ae0d8128678cabd56c66ea7b9 Mon Sep 17 00:00:00 2001 From: Jessica Huhnke <91915469+jhuhnke@users.noreply.github.com> Date: Thu, 3 Mar 2022 12:13:09 -0600 Subject: [PATCH 29/35] Fixes typo that allowed null program ids (#372) * Fixes typo that allowed null program ids * update to check instruction length * where clause --- models/solana/silver/silver_solana__transactions.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/models/solana/silver/silver_solana__transactions.sql b/models/solana/silver/silver_solana__transactions.sql index 7ca285c3..195378d6 100644 --- a/models/solana/silver/silver_solana__transactions.sql +++ b/models/solana/silver/silver_solana__transactions.sql @@ -38,14 +38,15 @@ WITH base_table AS ( ) > 0 AND len( tx :meta :preTokenBalances [0] - ) > 0 THEN TRUE + ) > 0 + AND len(tx: TRANSACTION :message :instructions) > 0 THEN TRUE ELSE FALSE END AS transfer_tx_flag FROM {{ ref('bronze_solana__transactions') }} WHERE - program_id IS NULL - OR program_id <> 'Vote111111111111111111111111111111111111111' + program_id IS NULL + OR program_id <> 'Vote111111111111111111111111111111111111111' {% if is_incremental() %} AND ingested_at >= getdate() - INTERVAL '2 days' From 1410814e9718adc161290b04d176206cfb16a989 Mon Sep 17 00:00:00 2001 From: desmond-hui <97470747+desmond-hui@users.noreply.github.com> Date: Thu, 3 Mar 2022 12:42:57 -0800 Subject: [PATCH 30/35] An 758/orca swaps (#373) * add orca swaps model * add orca to gold view * change to test in gold Co-authored-by: jhuhnke --- models/solana/gold/solana__swaps.sql | 14 + models/solana/gold/solana__swaps.yml | 3 +- .../silver/silver_solana__swaps_orca_dex.sql | 378 ++++++++++++++++++ .../silver/silver_solana__swaps_orca_dex.yml | 49 +++ 4 files changed, 443 insertions(+), 1 deletion(-) create mode 100644 models/solana/silver/silver_solana__swaps_orca_dex.sql create mode 100644 models/solana/silver/silver_solana__swaps_orca_dex.yml diff --git a/models/solana/gold/solana__swaps.sql b/models/solana/gold/solana__swaps.sql index fac19268..d4929386 100644 --- a/models/solana/gold/solana__swaps.sql +++ b/models/solana/gold/solana__swaps.sql @@ -16,3 +16,17 @@ SELECT to_mint AS swap_to_mint FROM {{ ref('silver_solana__swaps_jupiter_dex') }} +UNION +SELECT + 'orca' AS swap_program, + block_timestamp, + block_id, + tx_id, + succeeded, + swapper, + from_amt, + from_mint, + to_amt, + to_mint +FROM + {{ ref('silver_solana__swaps_orca_dex') }} diff --git a/models/solana/gold/solana__swaps.yml b/models/solana/gold/solana__swaps.yml index ed423256..c4aba758 100644 --- a/models/solana/gold/solana__swaps.yml +++ b/models/solana/gold/solana__swaps.yml @@ -45,4 +45,5 @@ models: - name: SWAP_TO_MINT description: Token being received or swapped for tests: - - not_null \ No newline at end of file + - not_null: + where: SUCCEEDED = TRUE \ No newline at end of file diff --git a/models/solana/silver/silver_solana__swaps_orca_dex.sql b/models/solana/silver/silver_solana__swaps_orca_dex.sql new file mode 100644 index 00000000..3b5bcf95 --- /dev/null +++ b/models/solana/silver/silver_solana__swaps_orca_dex.sql @@ -0,0 +1,378 @@ +{{ config( + materialized = 'incremental', + unique_key = "CONCAT_WS('-', block_id, tx_id)", + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::DATE'], + tags = ['snowflake', 'solana', 'silver_solana', 'solana_swaps'] +) }} + +WITH base_i AS ( + + SELECT + * + FROM + {{ ref('solana_dbt__instructions') }} + i + +{% if is_incremental() %} +WHERE + i.ingested_at :: DATE >= CURRENT_DATE - 2 +{% endif %} +), +orca_dex_txs AS ( + SELECT + DISTINCT i.block_id, + i.block_timestamp, + i.tx_id, + t.fee, + t.account_keys, + t.succeeded + FROM + base_i i + INNER JOIN {{ ref('silver_solana__transactions') }} + t + ON t.tx_id = i.tx_id + WHERE + i.value :programId :: STRING IN ( + -- unknown orca swaps version, seems related to v2 + 'MEV1HDn99aybER3U3oa9MySSXqoEZNDEQ4miAimTjaW', + -- orca swaps v2 + '9W959DqEETiGZocYWCQPaJ6sBmUzgfxXfqGeTEdp3aQP', + -- orca swaps v1 + 'DjVE6JNiYqPL2QXyCUUh8rNjHrbz9hXHNYt99MQ59qw1' + ) + +{% if is_incremental() %} +AND t.block_timestamp :: DATE >= CURRENT_DATE - 2 +{% endif %} +), +delegates_map_tmp AS ( + SELECT + i.tx_id, + VALUE :parsed :info :delegate :: STRING AS delegate, + VALUE :parsed :info :owner :: STRING AS owner + FROM + base_i i + INNER JOIN orca_dex_txs t on t.tx_id = i.tx_id + WHERE + delegate IS NOT NULL +), +delegates_map as ( + select + tx_id, + delegate, + owner + from delegates_map_tmp + group by 1,2,3 +), +signers_tmp AS ( + SELECT + t.tx_id, + A.value :pubkey :: STRING AS acct,= + A.index + FROM + orca_dex_txs t + LEFT OUTER JOIN TABLE(FLATTEN(t.account_keys)) A + WHERE + A.value :signer = TRUE +), +signers as ( + select + s.tx_id, + s.acct, + dm.owner AS delegate_owner + from signers_tmp s + LEFT OUTER JOIN delegates_map dm + ON dm.tx_id = s.tx_id + AND s.acct = dm.delegate +), +post_balances_acct_map AS ( + SELECT + t.tx_id, + t.account_keys [b.account_index] :pubkey :: STRING AS middle_acct, + b.owner, + b.mint, + b.decimal, + b.amount + FROM + {{ ref('solana_dbt__post_token_balances') }} + b + INNER JOIN orca_dex_txs t + ON t.tx_id = b.tx_id + +{% if is_incremental() %} +WHERE + b.ingested_at :: DATE >= CURRENT_DATE - 2 +{% endif %} +), +destinations AS ( + SELECT + i.block_id, + i.block_timestamp, + i.tx_id, + i.succeeded, + i.index, + ii.index AS inner_index, + ii.value :parsed :info :destination :: STRING AS destination, + ii.value :parsed :info :authority :: STRING AS authority, + ii.value :parsed :info :source :: STRING AS source, + ii.value :parsed :info :amount AS amount, + ROW_NUMBER() over ( + PARTITION BY i.tx_id + ORDER BY + i.index, + inner_index + ) AS rn + FROM + {{ ref('silver_solana__events') }} + i + INNER JOIN orca_dex_txs t + ON t.tx_id = i.tx_id + LEFT OUTER JOIN TABLE(FLATTEN(inner_instruction :instructions)) ii + WHERE + ii.value :parsed :info :destination :: STRING IS NOT NULL + AND COALESCE( + ii.value :programId :: STRING, + '' + ) <> '11111111111111111111111111111111' + AND i.instruction :programId :: STRING IN ( + -- unknown orca swaps version, seems related to v2 + 'MEV1HDn99aybER3U3oa9MySSXqoEZNDEQ4miAimTjaW', + -- orca swaps v2 + '9W959DqEETiGZocYWCQPaJ6sBmUzgfxXfqGeTEdp3aQP', + -- orca swaps v1 + 'DjVE6JNiYqPL2QXyCUUh8rNjHrbz9hXHNYt99MQ59qw1' + ) + +{% if is_incremental() %} +AND i.block_timestamp :: DATE >= CURRENT_DATE - 2 +{% endif %} +), +destination_acct_map AS ( + SELECT + tx_id, + authority, + source + FROM + destinations + GROUP BY + 1, + 2, + 3 +), +swaps_tmp_1 AS ( + SELECT + COALESCE( + s.delegate_owner, + s.acct + ) AS swapper, + -- s.acct AS swapper, + COALESCE( + p1.owner, + d2.authority + ) AS destination_owner, + COALESCE( + p1.mint, + p2.mint + ) AS mint, + COALESCE( + p1.decimal, + p2.decimal + ) AS DECIMAL, + d.* + FROM + destinations d + LEFT OUTER JOIN signers s + ON s.acct = d.authority + and s.tx_id = d.tx_id + LEFT OUTER JOIN post_balances_acct_map p1 + ON p1.middle_acct = d.destination + AND p1.tx_id = d.tx_id + LEFT OUTER JOIN post_balances_acct_map p2 + ON p2.middle_acct = d.source + AND p2.tx_id = d.tx_id + LEFT OUTER JOIN destination_acct_map d2 + ON d2.source = d.destination + AND d2.tx_id = d.tx_id +), +swapper_min_rn AS ( + SELECT + DISTINCT s.tx_id, + MIN(rn) over ( + PARTITION BY s.tx_id + ) AS min_swapper_rn + FROM + swaps_tmp_1 s + WHERE + swapper IS NOT NULL +), +swaps_tmp AS ( + SELECT + s.block_id, + s.block_timestamp, + s.tx_id, + s.succeeded, + s.swapper, + s.destination_owner, + s.mint, + s.decimal, + s.index, + s.inner_index, + s.destination, + s.authority, + s.source, + s.amount, + ROW_NUMBER() over ( + PARTITION BY s.tx_id + ORDER BY + s.index, + s.inner_index + ) AS rn + FROM + swaps_tmp_1 s + INNER JOIN swapper_min_rn m + ON s.tx_id = m.tx_id + WHERE + s.rn >= m.min_swapper_rn +), +mint_acct_map AS ( + SELECT + tx_id, + source, + mint, + DECIMAL + FROM + swaps_tmp + GROUP BY + 1, + 2, + 3, + 4 +), +swap_actions AS ( + SELECT + s1.block_id, + s1.block_timestamp, + s1.tx_id, + s1.succeeded, + s1.swapper, + s1.destination_owner, + s1.destination, + s1.source, + COALESCE( + s1.mint, + s2.mint + ) AS mint, + COALESCE( + s1.decimal, + s2.decimal + ) AS DECIMAL, + s1.amount :: bigint AS amount, + s1.rn + FROM + swaps_tmp s1 + LEFT OUTER JOIN mint_acct_map s2 + ON s1.destination = s2.source + AND s1.tx_id = s2.tx_id +), +swap_actions_with_refund AS ( + SELECT + s1.*, + s3.mint AS originating_mint, + CASE + WHEN s2.amount < s1.amount THEN s2.amount + ELSE NULL + END AS refund, + MAX(refund) over ( + PARTITION BY s1.tx_id + ) AS max_refund, + s1.amount - COALESCE( + refund, + 0 + ) AS final_amt + FROM + swap_actions s1 + LEFT OUTER JOIN swap_actions s2 + ON s1.tx_id = s2.tx_id + AND s1.swapper = s2.destination_owner + AND s1.mint = s2.mint + AND s1.rn = 1 + LEFT OUTER JOIN swap_actions s3 + ON s1.tx_id = s3.tx_id + AND s3.rn = 1 +), +swap_actions_final AS ( + SELECT + * + FROM + swap_actions_with_refund + WHERE + swapper IS NOT NULL + OR mint <> originating_mint + OR ( + originating_mint = mint + AND max_refund IS NULL + ) -- need to do this for situations where it appears the user swaps back to the same mint... +), +agg_tmp AS ( + SELECT + block_id, + block_timestamp, + tx_id, + succeeded, + swapper, + mint, + DECIMAL, + SUM(final_amt) AS amt, + MIN(rn) AS rn + FROM + swap_actions_final + GROUP BY + 1, + 2, + 3, + 4, + 5, + 6, + 7 +), +agg AS ( + SELECT + *, + MAX(rn) over ( + PARTITION BY tx_id + ) AS max_rn + FROM + agg_tmp + WHERE + amt <> 0 +) +SELECT + a1.block_id, + a1.block_timestamp, + a1.tx_id, + a1.succeeded, + a1.swapper, + a1.mint AS from_mint, + CASE + WHEN a1.succeeded THEN a1.amt * pow( + 10,- a1.decimal + ) + ELSE 0 + END AS from_amt, + a2.mint AS to_mint, + CASE + WHEN a1.succeeded THEN a2.amt * pow( + 10,- a2.decimal + ) + ELSE 0 + END AS to_amt +FROM + agg a1 + LEFT OUTER JOIN agg a2 + ON a1.tx_id = a2.tx_id + AND a1.rn <> a2.rn + AND a2.rn = a2.max_rn +WHERE + a1.rn = 1 + AND to_amt IS NOT NULL diff --git a/models/solana/silver/silver_solana__swaps_orca_dex.yml b/models/solana/silver/silver_solana__swaps_orca_dex.yml new file mode 100644 index 00000000..cd0d24bc --- /dev/null +++ b/models/solana/silver/silver_solana__swaps_orca_dex.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: silver_solana__swaps_orca_dex + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - TX_ID + columns: + - name: BLOCK_TIMESTAMP + description: The time the block began + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: Unique sequential number that identifies the current block + tests: + - not_null + - name: TX_ID + description: A unique key that identifies a transaction + tests: + - not_null + - name: SUCCEEDED + description: True when a transaction is successful, otherwise false. + tests: + - not_null + - name: SWAPPER + description: Address that initiated the swap + tests: + - not_null + - name: FROM_AMT + description: Total amount of the token sent in to initiate the swap + tests: + - not_null + - name: FROM_MINT + description: Token being sent or swapped from + tests: + - not_null + - name: TO_AMT + description: Total amount of the token received in the swap + tests: + - not_null + - name: TO_MINT + description: Token being received or swapped for + tests: + - not_null: + where: succeeded = TRUE \ No newline at end of file From 8b89510a17c10a14bb39afb5717a05fd1efd060c Mon Sep 17 00:00:00 2001 From: desmond-hui <97470747+desmond-hui@users.noreply.github.com> Date: Thu, 3 Mar 2022 13:01:19 -0800 Subject: [PATCH 31/35] An 758/orca swaps (#375) * add orca swaps model * add orca to gold view * remove randoom = sign --- models/solana/silver/silver_solana__swaps_orca_dex.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/solana/silver/silver_solana__swaps_orca_dex.sql b/models/solana/silver/silver_solana__swaps_orca_dex.sql index 3b5bcf95..4e5fb4a5 100644 --- a/models/solana/silver/silver_solana__swaps_orca_dex.sql +++ b/models/solana/silver/silver_solana__swaps_orca_dex.sql @@ -68,7 +68,7 @@ delegates_map as ( signers_tmp AS ( SELECT t.tx_id, - A.value :pubkey :: STRING AS acct,= + A.value :pubkey :: STRING AS acct, A.index FROM orca_dex_txs t From e4ae5a50bb3baa242a32c42dc23735e821b16e86 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Thu, 3 Mar 2022 14:04:22 -0800 Subject: [PATCH 32/35] - persist_docs turned on --- dbt_project.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbt_project.yml b/dbt_project.yml index 32e2a932..69985d3b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -31,6 +31,9 @@ clean-targets: # directories to be removed by `dbt clean` # using the `{{ config(...) }}` macro. models: +copy_grants: true + +persist_docs: + relation: true + columns: true sql_models: ethereum: materialized: incremental From ec316b0e99f260f9497310a93d4effedf67beba9 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Fri, 4 Mar 2022 12:28:40 -0800 Subject: [PATCH 33/35] - desired changes --- models/algorand/gold/algorand__account.sql | 1 - models/algorand/gold/algorand__account.yml | 10 +- .../algorand/gold/algorand__account_app.sql | 1 - .../algorand/gold/algorand__account_app.yml | 10 +- .../algorand/gold/algorand__account_asset.sql | 1 - .../algorand/gold/algorand__account_asset.yml | 10 +- models/algorand/gold/algorand__app.sql | 1 - models/algorand/gold/algorand__app.yml | 7 +- ...algorand__application_call_transaction.yml | 14 +- ...orand__asset_configuration_transaction.yml | 15 +- .../algorand__asset_freeze_transaction.yml | 11 +- .../algorand__asset_transfer_transaction.yml | 12 +- models/algorand/gold/algorand__block.yml | 7 +- ...algorand__key_registration_transaction.yml | 11 +- .../gold/algorand__payment_transaction.yml | 12 +- .../algorand__transaction_participation.yml | 9 +- .../algorand/gold/algorand__transactions.yml | 12 +- .../silver/silver_algorand__account.sql | 37 ++--- .../silver/silver_algorand__account.yml | 13 -- .../silver/silver_algorand__account_app.sql | 35 +++-- .../silver/silver_algorand__account_app.yml | 15 +- .../silver/silver_algorand__account_asset.sql | 30 ++-- .../silver/silver_algorand__account_asset.yml | 18 +-- .../algorand/silver/silver_algorand__app.sql | 29 ++-- .../algorand/silver/silver_algorand__app.yml | 7 +- ...algorand__application_call_transaction.sql | 91 ++---------- ...algorand__application_call_transaction.yml | 19 +-- .../silver/silver_algorand__asset.sql | 14 +- ...orand__asset_configuration_transaction.sql | 107 +++---------- ...orand__asset_configuration_transaction.yml | 15 +- ...ver_algorand__asset_freeze_transaction.sql | 107 +++---------- ...ver_algorand__asset_freeze_transaction.yml | 11 +- ...r_algorand__asset_transfer_transaction.sql | 109 +++----------- ...r_algorand__asset_transfer_transaction.yml | 17 +-- .../silver/silver_algorand__block.sql | 51 +++---- .../silver/silver_algorand__block.yml | 10 +- .../silver/silver_algorand__inner_txids.sql | 140 +++--------------- ...algorand__key_registration_transaction.sql | 110 +++----------- ...algorand__key_registration_transaction.yml | 11 +- .../silver_algorand__payment_transaction.sql | 110 +++----------- .../silver_algorand__payment_transaction.yml | 17 +-- ...er_algorand__transaction_participation.sql | 73 ++------- ...er_algorand__transaction_participation.yml | 14 +- .../silver/silver_algorand__transactions.sql | 114 +++----------- .../silver/silver_algorand__transactions.yml | 19 +-- models/sources.yml | 11 +- 46 files changed, 343 insertions(+), 1155 deletions(-) diff --git a/models/algorand/gold/algorand__account.sql b/models/algorand/gold/algorand__account.sql index 11ea6533..cd5d12c5 100644 --- a/models/algorand/gold/algorand__account.sql +++ b/models/algorand/gold/algorand__account.sql @@ -11,7 +11,6 @@ SELECT balance, closed_at, created_at, - created_at_timestamp, wallet_type, account_data FROM diff --git a/models/algorand/gold/algorand__account.yml b/models/algorand/gold/algorand__account.yml index 2c44cd51..ca643b06 100644 --- a/models/algorand/gold/algorand__account.yml +++ b/models/algorand/gold/algorand__account.yml @@ -9,8 +9,6 @@ models: - name: ADDRESS tests: - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - name: ACCOUNT_CLOSED tests: - not_null @@ -30,14 +28,8 @@ models: - name: CREATED_AT tests: - not_null - - name: CREATED_AT_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 50 - name: WALLET_TYPE tests: - accepted_values: values: ["sig", "msig", "lsig", "", null] - enabled: False # unhandled deletions causes this test to fail + enabled: False # unhandled deletions causes this test to fail \ No newline at end of file diff --git a/models/algorand/gold/algorand__account_app.sql b/models/algorand/gold/algorand__account_app.sql index bea54717..ac232670 100644 --- a/models/algorand/gold/algorand__account_app.sql +++ b/models/algorand/gold/algorand__account_app.sql @@ -9,7 +9,6 @@ SELECT app_closed, closed_at, created_at, - created_at_timestamp, app_info FROM {{ ref('silver_algorand__account_app') }} diff --git a/models/algorand/gold/algorand__account_app.yml b/models/algorand/gold/algorand__account_app.yml index ea433a68..2d6b01bb 100644 --- a/models/algorand/gold/algorand__account_app.yml +++ b/models/algorand/gold/algorand__account_app.yml @@ -10,8 +10,6 @@ models: - name: ADDRESS tests: - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - name: APP_ID tests: - not_null @@ -21,13 +19,7 @@ models: - name: CREATED_AT tests: - not_null - - name: CREATED_AT_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 50 - name: APP_INFO tests: - not_null: - where: APP_CLOSED = FALSE + where: APP_CLOSED = FALSE \ No newline at end of file diff --git a/models/algorand/gold/algorand__account_asset.sql b/models/algorand/gold/algorand__account_asset.sql index 091c2fbf..e47efdb0 100644 --- a/models/algorand/gold/algorand__account_asset.sql +++ b/models/algorand/gold/algorand__account_asset.sql @@ -9,7 +9,6 @@ SELECT asset_name, amount, asset_added_at, - added_at_timestamp, asset_last_removed, asset_closed, frozen diff --git a/models/algorand/gold/algorand__account_asset.yml b/models/algorand/gold/algorand__account_asset.yml index ca923988..05567497 100644 --- a/models/algorand/gold/algorand__account_asset.yml +++ b/models/algorand/gold/algorand__account_asset.yml @@ -10,8 +10,6 @@ models: - name: ADDRESS tests: - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - name: ASSET_ID tests: - not_null @@ -27,15 +25,9 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - name: ADDED_AT_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 50 - name: FROZEN tests: - not_null - name: ASSET_CLOSED tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/gold/algorand__app.sql b/models/algorand/gold/algorand__app.sql index a78b311c..b4bd3472 100644 --- a/models/algorand/gold/algorand__app.sql +++ b/models/algorand/gold/algorand__app.sql @@ -9,7 +9,6 @@ SELECT app_closed, closed_at, created_at, - created_at_timestamp, params FROM {{ ref('silver_algorand__app') }} diff --git a/models/algorand/gold/algorand__app.yml b/models/algorand/gold/algorand__app.yml index 80e97b93..b681eca9 100644 --- a/models/algorand/gold/algorand__app.yml +++ b/models/algorand/gold/algorand__app.yml @@ -12,14 +12,9 @@ models: - name: CREATOR_ADDRESS tests: - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - name: APP_CLOSED tests: - not_null - name: CREATED_AT tests: - - not_null - - name: CREATED_AT_TIMESTAMP - tests: - - not_null \ No newline at end of file + - not_null \ No newline at end of file diff --git a/models/algorand/gold/algorand__application_call_transaction.yml b/models/algorand/gold/algorand__application_call_transaction.yml index 5d3b8d43..4faa7686 100644 --- a/models/algorand/gold/algorand__application_call_transaction.yml +++ b/models/algorand/gold/algorand__application_call_transaction.yml @@ -9,11 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 50 + - not_null - name: block_id tests: - not_null @@ -28,11 +24,11 @@ models: - not_null - name: SENDER tests: - - not_null + - not_null - name: FEE tests: - not_null: - where: inner_tx = 'true' + where: inner_tx = 'true' - name: TX_TYPE tests: - not_null @@ -41,7 +37,7 @@ models: - not_null - name: GENESIS_HASH tests: - - not_null + - not_null - name: TX_MESSAGE tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/gold/algorand__asset_configuration_transaction.yml b/models/algorand/gold/algorand__asset_configuration_transaction.yml index 943abfd8..be222a27 100644 --- a/models/algorand/gold/algorand__asset_configuration_transaction.yml +++ b/models/algorand/gold/algorand__asset_configuration_transaction.yml @@ -9,8 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False + - not_null - name: BLOCK_ID tests: - not_null @@ -26,16 +25,14 @@ models: - name: ASSET_ID tests: - not_null: - where: inner_tx = 'false' + where: inner_tx = 'false' - name: SENDER tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null - name: FEE tests: - not_null: - where: inner_tx = 'true' + where: inner_tx = 'true' - name: TX_TYPE tests: - not_null @@ -44,7 +41,7 @@ models: - not_null - name: GENESIS_HASH tests: - - not_null + - not_null - name: TX_MESSAGE tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/gold/algorand__asset_freeze_transaction.yml b/models/algorand/gold/algorand__asset_freeze_transaction.yml index 2abde91c..08c43640 100644 --- a/models/algorand/gold/algorand__asset_freeze_transaction.yml +++ b/models/algorand/gold/algorand__asset_freeze_transaction.yml @@ -9,8 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False + - not_null - name: BLOCK_ID tests: - not_null @@ -31,12 +30,10 @@ models: - not_null - name: SENDER tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null - name: FEE tests: - - not_null + - not_null - name: TX_TYPE tests: - not_null @@ -48,4 +45,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/gold/algorand__asset_transfer_transaction.yml b/models/algorand/gold/algorand__asset_transfer_transaction.yml index 690e8b5a..5bea0073 100644 --- a/models/algorand/gold/algorand__asset_transfer_transaction.yml +++ b/models/algorand/gold/algorand__asset_transfer_transaction.yml @@ -5,11 +5,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 50 + - not_null - name: BLOCK_ID tests: - not_null @@ -27,9 +23,7 @@ models: - not_null - name: SENDER tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null - name: TX_TYPE tests: - not_null @@ -41,4 +35,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/gold/algorand__block.yml b/models/algorand/gold/algorand__block.yml index bf6800ed..3ec9cdbe 100644 --- a/models/algorand/gold/algorand__block.yml +++ b/models/algorand/gold/algorand__block.yml @@ -11,11 +11,10 @@ models: - not_null - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 50 + interval: 1 - name: REWARDSLEVEL tests: - not_null @@ -31,4 +30,4 @@ models: where: BLOCK_ID <> 0 - name: HEADER tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/gold/algorand__key_registration_transaction.yml b/models/algorand/gold/algorand__key_registration_transaction.yml index 6e7192e6..7e6ab096 100644 --- a/models/algorand/gold/algorand__key_registration_transaction.yml +++ b/models/algorand/gold/algorand__key_registration_transaction.yml @@ -9,8 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False + - not_null - name: BLOCK_ID tests: - not_null @@ -28,12 +27,10 @@ models: - not_null - name: SENDER tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null - name: FEE tests: - - not_null + - not_null - name: TX_TYPE tests: - not_null @@ -45,4 +42,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/gold/algorand__payment_transaction.yml b/models/algorand/gold/algorand__payment_transaction.yml index dd20944c..5f663a38 100644 --- a/models/algorand/gold/algorand__payment_transaction.yml +++ b/models/algorand/gold/algorand__payment_transaction.yml @@ -5,11 +5,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 50 + - not_null - name: BLOCK_ID tests: - not_null @@ -27,9 +23,7 @@ models: - not_null - name: SENDER tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null - name: TX_TYPE tests: - not_null @@ -41,4 +35,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/gold/algorand__transaction_participation.yml b/models/algorand/gold/algorand__transaction_participation.yml index bac357ad..28437337 100644 --- a/models/algorand/gold/algorand__transaction_participation.yml +++ b/models/algorand/gold/algorand__transaction_participation.yml @@ -10,11 +10,10 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 50 + interval: 1 - name: BLOCK_ID tests: - not_null @@ -23,6 +22,4 @@ models: - not_null - name: ADDRESS tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null \ No newline at end of file diff --git a/models/algorand/gold/algorand__transactions.yml b/models/algorand/gold/algorand__transactions.yml index 7b1ced03..132ffe87 100644 --- a/models/algorand/gold/algorand__transactions.yml +++ b/models/algorand/gold/algorand__transactions.yml @@ -5,11 +5,10 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 50 + interval: 1 - name: BLOCK_ID tests: - not_null @@ -22,11 +21,6 @@ models: - name: INNER_TX tests: - not_null - - name: SENDER - tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - name: TX_TYPE tests: - not_null @@ -38,4 +32,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/silver/silver_algorand__account.sql b/models/algorand/silver/silver_algorand__account.sql index c4385dae..2ba7e805 100644 --- a/models/algorand/silver/silver_algorand__account.sql +++ b/models/algorand/silver/silver_algorand__account.sql @@ -7,44 +7,47 @@ SELECT algorand_decode_hex_addr( - aa.addr :: text + addr :: text ) AS address, - aa.deleted AS account_closed, - aa.rewardsbase / pow( + deleted AS account_closed, + rewardsbase / pow( 10, 6 ) AS rewardsbase, - aa.rewards_total / pow( + rewards_total / pow( 10, 6 ) AS rewards_total, - aa.microalgos / pow( + microalgos / pow( 10, 6 ) AS balance, - aa.closed_at AS closed_at, - aa.created_at AS created_at, - ab.block_timestamp AS created_at_timestamp, - aa.keytype AS wallet_type, - aa.account_data AS account_data, - aa._FIVETRAN_SYNCED + closed_at AS closed_at, + created_at AS created_at, + keytype AS wallet_type, + account_data AS account_data, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', 'ACCOUNT' ) }} - aa - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON aa.created_at = ab.block_id WHERE 1 = 1 {% if is_incremental() %} -AND aa._FIVETRAN_SYNCED >= ( +AND DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' +) >= ( SELECT MAX( - _FIVETRAN_SYNCED + _INSERTED_TIMESTAMP ) FROM {{ this }} diff --git a/models/algorand/silver/silver_algorand__account.yml b/models/algorand/silver/silver_algorand__account.yml index 47df1936..705991b9 100644 --- a/models/algorand/silver/silver_algorand__account.yml +++ b/models/algorand/silver/silver_algorand__account.yml @@ -9,8 +9,6 @@ models: - name: ADDRESS tests: - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - name: ACCOUNT_CLOSED tests: - not_null @@ -30,19 +28,8 @@ models: - name: CREATED_AT tests: - not_null - - name: CREATED_AT_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 50 - name: WALLET_TYPE tests: - accepted_values: values: ["sig", "msig", "lsig", "", null] enabled: False # unhandled deletions causes this test to fail - - name: _FIVETRAN_SYNCED - tests: - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 diff --git a/models/algorand/silver/silver_algorand__account_app.sql b/models/algorand/silver/silver_algorand__account_app.sql index fbde5222..56b8b239 100644 --- a/models/algorand/silver/silver_algorand__account_app.sql +++ b/models/algorand/silver/silver_algorand__account_app.sql @@ -7,37 +7,40 @@ SELECT algorand_decode_hex_addr( - b.addr :: text + addr :: text ) AS address, - b.app AS app_id, - b.deleted AS app_closed, - b.closed_at AS closed_at, - b.created_at AS created_at, - ab.block_timestamp AS created_at_timestamp, - b.localstate AS app_info, + app AS app_id, + deleted AS app_closed, + closed_at AS closed_at, + created_at AS created_at, + localstate AS app_info, concat_ws( '-', - b.addr :: STRING, - b.app :: STRING + addr :: STRING, + app :: STRING ) AS _unique_key, - b._FIVETRAN_SYNCED + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', 'ACCOUNT_APP' ) }} - b - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON b.created_at = ab.block_id WHERE 1 = 1 {% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( +AND DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' +) >= ( SELECT MAX( - _FIVETRAN_SYNCED + _INSERTED_TIMESTAMP ) FROM {{ this }} diff --git a/models/algorand/silver/silver_algorand__account_app.yml b/models/algorand/silver/silver_algorand__account_app.yml index edb26e60..ec5c9c3f 100644 --- a/models/algorand/silver/silver_algorand__account_app.yml +++ b/models/algorand/silver/silver_algorand__account_app.yml @@ -10,8 +10,6 @@ models: - name: ADDRESS tests: - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - name: APP_ID tests: - not_null @@ -21,18 +19,7 @@ models: - name: CREATED_AT tests: - not_null - - name: CREATED_AT_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 50 - name: APP_INFO tests: - not_null: - where: APP_CLOSED = FALSE - - name: _FIVETRAN_SYNCED - tests: - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + where: APP_CLOSED = FALSE \ No newline at end of file diff --git a/models/algorand/silver/silver_algorand__account_asset.sql b/models/algorand/silver/silver_algorand__account_asset.sql index 5501e019..404c6796 100644 --- a/models/algorand/silver/silver_algorand__account_asset.sql +++ b/models/algorand/silver/silver_algorand__account_asset.sql @@ -20,20 +20,23 @@ SELECT algorand_decode_hex_addr( aa.addr :: text ) AS address, - aa.assetid AS asset_id, + assetid AS asset_id, an.name :: STRING AS asset_name, - aa.amount AS amount, - aa.created_at AS asset_added_at, - ab.block_timestamp AS added_at_timestamp, - aa.closed_at AS asset_last_removed, - aa.deleted AS asset_closed, - aa.frozen AS frozen, + amount :: NUMBER AS amount, + created_at AS asset_added_at, + closed_at AS asset_last_removed, + deleted AS asset_closed, + frozen AS frozen, concat_ws( '-', address :: STRING, asset_id :: STRING ) AS _unique_key, - aa._FIVETRAN_SYNCED + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', @@ -42,17 +45,18 @@ FROM aa LEFT JOIN asset_name an ON aa.assetid = an.index - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON aa.created_at = ab.block_id WHERE 1 = 1 {% if is_incremental() %} -AND aa._FIVETRAN_SYNCED >= ( +AND DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' +) >= ( SELECT MAX( - _FIVETRAN_SYNCED + _INSERTED_TIMESTAMP ) FROM {{ this }} diff --git a/models/algorand/silver/silver_algorand__account_asset.yml b/models/algorand/silver/silver_algorand__account_asset.yml index aeda4e7f..f44e7244 100644 --- a/models/algorand/silver/silver_algorand__account_asset.yml +++ b/models/algorand/silver/silver_algorand__account_asset.yml @@ -10,8 +10,6 @@ models: - name: ADDRESS tests: - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - name: ASSET_ID tests: - not_null @@ -27,20 +25,14 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - name: ADDED_AT_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 50 - name: FROZEN tests: - not_null - name: ASSET_CLOSED tests: - not_null - - name: _FIVETRAN_SYNCED - tests: - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + + + + + diff --git a/models/algorand/silver/silver_algorand__app.sql b/models/algorand/silver/silver_algorand__app.sql index bbcdc4a9..a33b9d74 100644 --- a/models/algorand/silver/silver_algorand__app.sql +++ b/models/algorand/silver/silver_algorand__app.sql @@ -8,31 +8,34 @@ SELECT INDEX AS app_id, algorand_decode_hex_addr( - aa.creator :: text + creator :: text ) AS creator_address, - aa.deleted AS app_closed, - aa.closed_at AS closed_at, - aa.created_at AS created_at, - ab.block_timestamp AS created_at_timestamp, - aa.params, - aa._FIVETRAN_SYNCED + deleted AS app_closed, + closed_at AS closed_at, + created_at AS created_at, + params, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', 'APP' ) }} - aa - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON aa.created_at = ab.block_id WHERE 1 = 1 {% if is_incremental() %} -AND aa._FIVETRAN_SYNCED >= ( +AND DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' +) >= ( SELECT MAX( - _FIVETRAN_SYNCED + _INSERTED_TIMESTAMP ) FROM {{ this }} diff --git a/models/algorand/silver/silver_algorand__app.yml b/models/algorand/silver/silver_algorand__app.yml index 7a370964..ff8324cf 100644 --- a/models/algorand/silver/silver_algorand__app.yml +++ b/models/algorand/silver/silver_algorand__app.yml @@ -12,14 +12,9 @@ models: - name: CREATOR_ADDRESS tests: - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - name: APP_CLOSED tests: - not_null - name: CREATED_AT tests: - - not_null - - name: CREATED_AT_TIMESTAMP - tests: - - not_null \ No newline at end of file + - not_null \ No newline at end of file diff --git a/models/algorand/silver/silver_algorand__application_call_transaction.sql b/models/algorand/silver/silver_algorand__application_call_transaction.sql index 81f402f0..66a275f3 100644 --- a/models/algorand/silver/silver_algorand__application_call_transaction.sql +++ b/models/algorand/silver/silver_algorand__application_call_transaction.sql @@ -5,7 +5,7 @@ tags = ['snowflake', 'algorand', 'application_call', 'silver_algorand_tx'] ) }} -WITH allTXN_fivetran AS ( +WITH allTXN AS ( SELECT ab.block_timestamp AS block_timestamp, @@ -32,7 +32,12 @@ WITH allTXN_fivetran AS ( ELSE txn :txn :gh :: STRING END AS genesis_hash, txn AS tx_message, - extra + extra, + DATEADD( + ms, + b.__HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', @@ -50,7 +55,11 @@ WITH allTXN_fivetran AS ( tx_type = 'appl' {% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( +AND DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' +) >= ( SELECT MAX( _INSERTED_TIMESTAMP @@ -59,80 +68,6 @@ AND b._FIVETRAN_SYNCED >= ( {{ this }} ) {% endif %} -), -allTXN_hevo AS ( - SELECT - ab.block_timestamp AS block_timestamp, - b.intra AS intra, - b.round AS block_id, - txn :txn :grp :: STRING AS tx_group_id, - CASE - WHEN b.txid IS NULL THEN ft.txn_txn_id :: text - ELSE b.txid :: text - END AS tx_id, - CASE - WHEN b.txid IS NULL THEN 'true' - ELSE 'false' - END AS inner_tx, - txn :txn :snd :: text AS sender, - txn :txn :fee / pow( - 10, - 6 - ) AS fee, - txn :txn :apid AS app_id, - txn :txn :type :: STRING AS tx_type, - CASE - WHEN b.txid IS NULL THEN ft.genesis_hash :: text - ELSE txn :txn :gh :: STRING - END AS genesis_hash, - txn AS tx_message, - extra - FROM - {{ source( - 'algorand', - 'TXN_MISSING' - ) }} - b - LEFT JOIN {{ ref('silver_algorand__inner_txids') }} - ft - ON b.round = ft.inner_round - AND b.intra = ft.inner_intra - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON b.round = ab.block_id - WHERE - tx_type = 'appl' - AND b.round > ( - SELECT - MAX(ROUND) - FROM - {{ source( - 'algorand', - 'TXN' - ) }} - ) - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN AS ( - SELECT - * - FROM - allTXN_fivetran - UNION - SELECT - * - FROM - allTXN_hevo ) SELECT block_timestamp, @@ -159,7 +94,7 @@ SELECT block_id :: STRING, intra :: STRING ) AS _unique_key, - SYSDATE() AS _INSERTED_TIMESTAMP + b._INSERTED_TIMESTAMP FROM allTXN b LEFT JOIN {{ ref('silver_algorand__transaction_types') }} diff --git a/models/algorand/silver/silver_algorand__application_call_transaction.yml b/models/algorand/silver/silver_algorand__application_call_transaction.yml index 377c9601..15ee477a 100644 --- a/models/algorand/silver/silver_algorand__application_call_transaction.yml +++ b/models/algorand/silver/silver_algorand__application_call_transaction.yml @@ -9,11 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False # FIXME - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: hour - interval: 50 + - not_null - name: block_id tests: - not_null @@ -28,11 +24,11 @@ models: - not_null - name: SENDER tests: - - not_null + - not_null - name: FEE tests: - not_null: - where: inner_tx = 'true' + where: inner_tx = 'true' - name: TX_TYPE tests: - not_null @@ -41,12 +37,7 @@ models: - not_null - name: GENESIS_HASH tests: - - not_null + - not_null - name: TX_MESSAGE tests: - - not_null - - name: _INSERTED_TIMESTAMP::DATE - tests: - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + - not_null \ No newline at end of file diff --git a/models/algorand/silver/silver_algorand__asset.sql b/models/algorand/silver/silver_algorand__asset.sql index ff6ec2f9..651c9042 100644 --- a/models/algorand/silver/silver_algorand__asset.sql +++ b/models/algorand/silver/silver_algorand__asset.sql @@ -17,7 +17,11 @@ SELECT deleted AS asset_deleted, closed_at AS closed_at, created_at AS created_at, - _FIVETRAN_SYNCED + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', @@ -27,10 +31,14 @@ WHERE 1 = 1 {% if is_incremental() %} -AND _FIVETRAN_SYNCED >= ( +AND DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' +) >= ( SELECT MAX( - _FIVETRAN_SYNCED + _INSERTED_TIMESTAMP ) FROM {{ this }} diff --git a/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql b/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql index 6c41725e..23c9c460 100644 --- a/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql +++ b/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql @@ -5,7 +5,7 @@ tags = ['snowflake', 'algorand', 'asset_configuration', 'silver_algorand_tx'] ) }} -WITH allTXN_fivetran AS ( +WITH allTXN AS ( SELECT ab.block_timestamp AS block_timestamp, @@ -34,7 +34,8 @@ WITH allTXN_fivetran AS ( ELSE txn :txn :gh :: STRING END AS genesis_hash, txn AS tx_message, - extra + extra, + b.__HEVO__LOADED_AT AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', @@ -50,93 +51,6 @@ WITH allTXN_fivetran AS ( ON b.round = ab.block_id WHERE tx_type = 'acfg' - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN_hevo AS ( - SELECT - ab.block_timestamp AS block_timestamp, - b.intra AS intra, - b.round AS block_id, - txn :txn :grp :: STRING AS tx_group_id, - CASE - WHEN b.txid IS NULL THEN ft.txn_txn_id :: text - ELSE b.txid :: text - END AS tx_id, - CASE - WHEN b.txid IS NULL THEN 'true' - ELSE 'false' - END AS inner_tx, - asset AS asset_id, - txn :txn :apar :t AS asset_supply, - txn :txn :snd :: text AS sender, - txn :txn :fee / pow( - 10, - 6 - ) AS fee, - txn :txn :apar AS asset_parameters, - txn :txn :type :: STRING AS tx_type, - CASE - WHEN b.txid IS NULL THEN ft.genesis_hash :: text - ELSE txn :txn :gh :: STRING - END AS genesis_hash, - txn AS tx_message, - extra - FROM - {{ source( - 'algorand', - 'TXN_MISSING' - ) }} - b - LEFT JOIN {{ ref('silver_algorand__inner_txids') }} - ft - ON b.round = ft.inner_round - AND b.intra = ft.inner_intra - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON b.round = ab.block_id - WHERE - tx_type = 'acfg' - AND b.round > ( - SELECT - MAX(ROUND) - FROM - {{ source( - 'algorand', - 'TXN' - ) }} - ) - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN AS( - SELECT - * - FROM - allTXN_fivetran - UNION - SELECT - * - FROM - allTXN_hevo ) SELECT block_timestamp, @@ -164,9 +78,22 @@ SELECT block_id :: STRING, intra :: STRING ) AS _unique_key, - SYSDATE() AS _inserted_timestamp + b._INSERTED_TIMESTAMP FROM allTXN b LEFT JOIN {{ ref('silver_algorand__transaction_types') }} csv ON b.tx_type = csv.type +WHERE + 1 = 1 + +{% if is_incremental() %} +AND b._INSERTED_TIMESTAMP >= ( + SELECT + MAX( + _INSERTED_TIMESTAMP + ) + FROM + {{ this }} +) +{% endif %} diff --git a/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml b/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml index 9a7c5ab5..e5389f2d 100644 --- a/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml +++ b/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml @@ -9,8 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False # FIXME + - not_null - name: BLOCK_ID tests: - not_null @@ -26,16 +25,14 @@ models: - name: ASSET_ID tests: - not_null: - where: inner_tx = 'false' + where: inner_tx = 'false' - name: SENDER tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null - name: FEE tests: - not_null: - where: inner_tx = 'true' + where: inner_tx = 'true' - name: TX_TYPE tests: - not_null @@ -44,7 +41,7 @@ models: - not_null - name: GENESIS_HASH tests: - - not_null + - not_null - name: TX_MESSAGE tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/silver/silver_algorand__asset_freeze_transaction.sql b/models/algorand/silver/silver_algorand__asset_freeze_transaction.sql index 717a360a..7579ab31 100644 --- a/models/algorand/silver/silver_algorand__asset_freeze_transaction.sql +++ b/models/algorand/silver/silver_algorand__asset_freeze_transaction.sql @@ -5,7 +5,7 @@ tags = ['snowflake', 'algorand', 'asset_freeze', 'silver_algorand_tx'] ) }} -WITH allTXN_fivetran AS ( +WITH allTXN AS ( SELECT ab.block_timestamp AS block_timestamp, @@ -34,7 +34,8 @@ WITH allTXN_fivetran AS ( ELSE txn :txn :gh :: STRING END AS genesis_hash, txn AS tx_message, - extra + extra, + b.__HEVO__LOADED_AT AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', @@ -50,93 +51,6 @@ WITH allTXN_fivetran AS ( ON b.round = ab.block_id WHERE tx_type = 'afrz' - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN_hevo AS ( - SELECT - ab.block_timestamp AS block_timestamp, - b.intra AS intra, - b.round AS block_id, - txn :txn :grp :: STRING AS tx_group_id, - CASE - WHEN b.txid IS NULL THEN ft.txn_txn_id :: text - ELSE b.txid :: text - END AS tx_id, - CASE - WHEN b.txid IS NULL THEN 'true' - ELSE 'false' - END AS inner_tx, - asset AS asset_id, - txn :txn :fadd :: text AS asset_address, - txn :txn :afrz AS asset_freeze, - txn :txn :snd :: text AS sender, - txn :txn :fee / pow( - 10, - 6 - ) AS fee, - txn :txn :type :: STRING AS tx_type, - CASE - WHEN b.txid IS NULL THEN ft.genesis_hash :: text - ELSE txn :txn :gh :: STRING - END AS genesis_hash, - txn AS tx_message, - extra - FROM - {{ source( - 'algorand', - 'TXN_MISSING' - ) }} - b - LEFT JOIN {{ ref('silver_algorand__inner_txids') }} - ft - ON b.round = ft.inner_round - AND b.intra = ft.inner_intra - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON b.round = ab.block_id - WHERE - tx_type = 'afrz' - AND b.round > ( - SELECT - MAX(ROUND) - FROM - {{ source( - 'algorand', - 'TXN' - ) }} - ) - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN AS( - SELECT - * - FROM - allTXN_fivetran - UNION - SELECT - * - FROM - allTXN_hevo ) SELECT block_timestamp, @@ -166,9 +80,22 @@ SELECT block_id :: STRING, intra :: STRING ) AS _unique_key, - SYSDATE() AS _inserted_timestamp + b._INSERTED_TIMESTAMP FROM allTXN b LEFT JOIN {{ ref('silver_algorand__transaction_types') }} csv ON b.tx_type = csv.type +WHERE + 1 = 1 + +{% if is_incremental() %} +AND b._INSERTED_TIMESTAMP >= ( + SELECT + MAX( + _INSERTED_TIMESTAMP + ) + FROM + {{ this }} +) +{% endif %} diff --git a/models/algorand/silver/silver_algorand__asset_freeze_transaction.yml b/models/algorand/silver/silver_algorand__asset_freeze_transaction.yml index 09d4018c..452e13ef 100644 --- a/models/algorand/silver/silver_algorand__asset_freeze_transaction.yml +++ b/models/algorand/silver/silver_algorand__asset_freeze_transaction.yml @@ -9,8 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False # FIXME + - not_null - name: BLOCK_ID tests: - not_null @@ -31,12 +30,10 @@ models: - not_null - name: SENDER tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null - name: FEE tests: - - not_null + - not_null - name: TX_TYPE tests: - not_null @@ -48,4 +45,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/silver/silver_algorand__asset_transfer_transaction.sql b/models/algorand/silver/silver_algorand__asset_transfer_transaction.sql index 8e9a43cb..7c13dc71 100644 --- a/models/algorand/silver/silver_algorand__asset_transfer_transaction.sql +++ b/models/algorand/silver/silver_algorand__asset_transfer_transaction.sql @@ -6,7 +6,7 @@ tags = ['snowflake', 'algorand', 'asset_transfer', 'silver_algorand_tx'] ) }} -WITH allTXN_fivetran AS ( +WITH allTXN AS ( SELECT ab.block_timestamp AS block_timestamp, @@ -37,7 +37,8 @@ WITH allTXN_fivetran AS ( ELSE txn :txn :gh :: STRING END AS genesis_hash, txn AS tx_message, - extra + extra, + b.__HEVO__LOADED_AT AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', @@ -53,95 +54,6 @@ WITH allTXN_fivetran AS ( ON b.round = ab.block_id WHERE tx_type = 'axfer' - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN_hevo AS ( - SELECT - ab.block_timestamp AS block_timestamp, - b.intra AS intra, - b.round AS block_id, - txn :txn :grp :: STRING AS tx_group_id, - CASE - WHEN b.txid IS NULL THEN ft.txn_txn_id :: text - ELSE b.txid :: text - END AS tx_id, - CASE - WHEN b.txid IS NULL THEN 'true' - ELSE 'false' - END AS inner_tx, - asset AS asset_id, - txn :txn :snd :: text AS sender, - txn :txn :fee / pow( - 10, - 6 - ) AS fee, - txn :txn :asnd :: text AS asset_sender, - txn :txn :arcv :: text AS asset_receiver, - txn :txn :aamt AS asset_amount, - txn :txn :xaid AS asset_transferred, - txn :txn :type :: STRING AS tx_type, - CASE - WHEN b.txid IS NULL THEN ft.genesis_hash :: text - ELSE txn :txn :gh :: STRING - END AS genesis_hash, - txn AS tx_message, - extra - FROM - {{ source( - 'algorand', - 'TXN_MISSING' - ) }} - b - LEFT JOIN {{ ref('silver_algorand__inner_txids') }} - ft - ON b.round = ft.inner_round - AND b.intra = ft.inner_intra - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON b.round = ab.block_id - WHERE - tx_type = 'axfer' - AND b.round > ( - SELECT - MAX(ROUND) - FROM - {{ source( - 'algorand', - 'TXN' - ) }} - ) - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN AS( - SELECT - * - FROM - allTXN_fivetran - UNION - SELECT - * - FROM - allTXN_hevo ) SELECT block_timestamp, @@ -175,9 +87,22 @@ SELECT block_id :: STRING, intra :: STRING ) AS _unique_key, - SYSDATE() AS _inserted_timestamp + b._INSERTED_TIMESTAMP FROM allTXN b LEFT JOIN {{ ref('silver_algorand__transaction_types') }} csv ON b.tx_type = csv.type +WHERE + 1 = 1 + +{% if is_incremental() %} +AND b._INSERTED_TIMESTAMP >= ( + SELECT + MAX( + _INSERTED_TIMESTAMP + ) + FROM + {{ this }} +) +{% endif %} diff --git a/models/algorand/silver/silver_algorand__asset_transfer_transaction.yml b/models/algorand/silver/silver_algorand__asset_transfer_transaction.yml index 5ee1af48..9bd09cb9 100644 --- a/models/algorand/silver/silver_algorand__asset_transfer_transaction.yml +++ b/models/algorand/silver/silver_algorand__asset_transfer_transaction.yml @@ -9,11 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False # FIXME - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: hour - interval: 50 + - not_null - name: BLOCK_ID tests: - not_null @@ -31,9 +27,7 @@ models: - not_null - name: SENDER tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null - name: TX_TYPE tests: - not_null @@ -45,9 +39,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null - - name: _INSERTED_TIMESTAMP::DATE - tests: - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + - not_null \ No newline at end of file diff --git a/models/algorand/silver/silver_algorand__block.sql b/models/algorand/silver/silver_algorand__block.sql index e9566a9b..772efbb7 100644 --- a/models/algorand/silver/silver_algorand__block.sql +++ b/models/algorand/silver/silver_algorand__block.sql @@ -5,44 +5,37 @@ tags = ['snowflake', 'algorand', 'block', 'silver_algorand'] ) }} -WITH allBLOCKS AS( - - SELECT - ROUND AS block_id, - realtime :: TIMESTAMP AS block_timestamp, - rewardslevel AS rewardslevel, - header :gen :: STRING AS network, - header :gh :: STRING AS genesis_hash, - header :prev :: STRING AS prev_block_hash, - header :txn :: STRING AS txn_root, - header, - _FIVETRAN_SYNCED - FROM - {{ source( - 'algorand', - 'BLOCK_HEADER' - ) }} -) SELECT - block_id, - block_timestamp, - rewardslevel, - network, - genesis_hash, - prev_block_hash, - txn_root, + ROUND AS block_id, + realtime :: TIMESTAMP AS block_timestamp, + rewardslevel AS rewardslevel, + header :gen :: STRING AS network, + header :gh :: STRING AS genesis_hash, + header :prev :: STRING AS prev_block_hash, + header :txn :: STRING AS txn_root, header, - _FIVETRAN_SYNCED + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP FROM - allBLOCKS + {{ source( + 'algorand', + 'BLOCK_HEADER' + ) }} WHERE 1 = 1 {% if is_incremental() %} -AND _FIVETRAN_SYNCED >= ( +AND DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' +) >= ( SELECT MAX( - _FIVETRAN_SYNCED + _INSERTED_TIMESTAMP ) FROM {{ this }} diff --git a/models/algorand/silver/silver_algorand__block.yml b/models/algorand/silver/silver_algorand__block.yml index bc265e79..7caa5003 100644 --- a/models/algorand/silver/silver_algorand__block.yml +++ b/models/algorand/silver/silver_algorand__block.yml @@ -11,11 +11,10 @@ models: - not_null - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 50 + interval: 1 - name: REWARDSLEVEL tests: - not_null @@ -32,8 +31,3 @@ models: - name: HEADER tests: - not_null - - name: _FIVETRAN_SYNCED - tests: - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 diff --git a/models/algorand/silver/silver_algorand__inner_txids.sql b/models/algorand/silver/silver_algorand__inner_txids.sql index 8a2c0f0a..dfb2627b 100644 --- a/models/algorand/silver/silver_algorand__inner_txids.sql +++ b/models/algorand/silver/silver_algorand__inner_txids.sql @@ -5,13 +5,14 @@ tags = ['snowflake', 'algorand', 'transactions', 'all_algorand_tx'] ) }} -WITH emptyROUNDS_fivetran AS ( +WITH emptyROUNDS AS ( SELECT ROUND, intra, txn, - extra + extra, + __HEVO__LOADED_AT FROM {{ source( 'algorand', @@ -19,64 +20,8 @@ WITH emptyROUNDS_fivetran AS ( ) }} WHERE txid IS NULL - -{% if is_incremental() %} -AND _FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} ), -emptyROUNDS_hevo AS ( - SELECT - ROUND, - intra, - txn, - extra - FROM - {{ source( - 'algorand', - 'TXN_MISSING' - ) }} - WHERE - txid IS NULL - AND ROUND > ( - SELECT - MAX(ROUND) - FROM - {{ source( - 'algorand', - 'TXN' - ) }} - ) - -{% if is_incremental() %} -AND _FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -emptyROUNDS AS( - SELECT - * - FROM - emptyROUNDS_fivetran - UNION - SELECT - * - FROM - emptyROUNDS_hevo -), -fulljson_fivetran AS ( +fulljson AS ( SELECT ROUND, txid, @@ -89,62 +34,6 @@ fulljson_fivetran AS ( ) }} WHERE txid IS NOT NULL - -{% if is_incremental() %} -AND _FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -fulljson_hevo AS ( - SELECT - ROUND, - txid, - intra, - txn :txn :gh :: STRING AS gh - FROM - {{ source( - 'algorand', - 'TXN_MISSING' - ) }} - WHERE - txid IS NOT NULL - AND ROUND > ( - SELECT - MAX(ROUND) - FROM - {{ source( - 'algorand', - 'TXN' - ) }} - ) - -{% if is_incremental() %} -AND _FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -fulljson AS( - SELECT - * - FROM - fulljson_fivetran - UNION - SELECT - * - FROM - fulljson_hevo ) SELECT f.round AS txn_round, @@ -158,7 +47,11 @@ SELECT er.round :: STRING, er.intra :: STRING ) AS _unique_key, - SYSDATE() AS _inserted_timestamp + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP FROM emptyROUNDS er LEFT JOIN fulljson f @@ -166,3 +59,18 @@ FROM AND er.round = f.round WHERE f.round IS NOT NULL + +{% if is_incremental() %} +AND DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' +) >= ( + SELECT + MAX( + _INSERTED_TIMESTAMP + ) + FROM + {{ this }} +) +{% endif %} diff --git a/models/algorand/silver/silver_algorand__key_registration_transaction.sql b/models/algorand/silver/silver_algorand__key_registration_transaction.sql index 95cc5349..150059a6 100644 --- a/models/algorand/silver/silver_algorand__key_registration_transaction.sql +++ b/models/algorand/silver/silver_algorand__key_registration_transaction.sql @@ -5,7 +5,7 @@ tags = ['snowflake', 'algorand', 'key_registration', 'silver_algorand_tx'] ) }} -WITH allTXN_fivetran AS ( +WITH allTXN AS ( SELECT ab.block_timestamp AS block_timestamp, @@ -37,7 +37,8 @@ WITH allTXN_fivetran AS ( ELSE txn :txn :gh :: STRING END AS genesis_hash, txn AS tx_message, - extra + extra, + b.__HEVO__LOADED_AT AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', @@ -53,96 +54,6 @@ WITH allTXN_fivetran AS ( ON b.round = ab.block_id WHERE tx_type = 'keyreg' - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN_hevo AS ( - SELECT - ab.block_timestamp AS block_timestamp, - b.intra AS intra, - b.round AS block_id, - txn :txn :grp :: STRING AS tx_group_id, - CASE - WHEN b.txid IS NULL THEN ft.txn_txn_id :: text - ELSE b.txid :: text - END AS tx_id, - CASE - WHEN b.txid IS NULL THEN 'true' - ELSE 'false' - END AS inner_tx, - asset AS asset_id, - txn :txn :snd :: text AS sender, - txn :txn :fee / pow( - 10, - 6 - ) AS fee, - txn :txn :votekey :: text AS participation_key, - txn :txn :selkey :: text AS vrf_public_key, - txn :txn :votefst AS vote_first, - txn :txn :votelst AS vote_last, - txn :txn :votekd AS vote_keydilution, - txn :txn :type :: STRING AS tx_type, - CASE - WHEN b.txid IS NULL THEN ft.genesis_hash :: text - ELSE txn :txn :gh :: STRING - END AS genesis_hash, - txn AS tx_message, - extra - FROM - {{ source( - 'algorand', - 'TXN_MISSING' - ) }} - b - LEFT JOIN {{ ref('silver_algorand__inner_txids') }} - ft - ON b.round = ft.inner_round - AND b.intra = ft.inner_intra - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON b.round = ab.block_id - WHERE - tx_type = 'keyreg' - AND b.round > ( - SELECT - MAX(ROUND) - FROM - {{ source( - 'algorand', - 'TXN' - ) }} - ) - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN AS( - SELECT - * - FROM - allTXN_fivetran - UNION - SELECT - * - FROM - allTXN_hevo ) SELECT block_timestamp, @@ -177,9 +88,22 @@ SELECT block_id :: STRING, intra :: STRING ) AS _unique_key, - SYSDATE() AS _inserted_timestamp + b._INSERTED_TIMESTAMP FROM allTXN b LEFT JOIN {{ ref('silver_algorand__transaction_types') }} csv ON b.tx_type = csv.type +WHERE + 1 = 1 + +{% if is_incremental() %} +AND b._INSERTED_TIMESTAMP >= ( + SELECT + MAX( + _INSERTED_TIMESTAMP + ) + FROM + {{ this }} +) +{% endif %} diff --git a/models/algorand/silver/silver_algorand__key_registration_transaction.yml b/models/algorand/silver/silver_algorand__key_registration_transaction.yml index d4259289..caf2038f 100644 --- a/models/algorand/silver/silver_algorand__key_registration_transaction.yml +++ b/models/algorand/silver/silver_algorand__key_registration_transaction.yml @@ -9,8 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False + - not_null - name: BLOCK_ID tests: - not_null @@ -28,12 +27,10 @@ models: - not_null - name: SENDER tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null - name: FEE tests: - - not_null + - not_null - name: TX_TYPE tests: - not_null @@ -45,4 +42,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null + - not_null \ No newline at end of file diff --git a/models/algorand/silver/silver_algorand__payment_transaction.sql b/models/algorand/silver/silver_algorand__payment_transaction.sql index b33fc4be..5839376b 100644 --- a/models/algorand/silver/silver_algorand__payment_transaction.sql +++ b/models/algorand/silver/silver_algorand__payment_transaction.sql @@ -6,7 +6,7 @@ tags = ['snowflake', 'algorand', 'payment', 'silver_algorand_tx'] ) }} -WITH allTXN_fivetran AS ( +WITH allTXN AS ( SELECT ab.block_timestamp AS block_timestamp, @@ -38,7 +38,8 @@ WITH allTXN_fivetran AS ( ELSE txn :txn :gh :: STRING END AS genesis_hash, txn AS tx_message, - extra + extra, + b.__HEVO__LOADED_AT AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', @@ -54,96 +55,6 @@ WITH allTXN_fivetran AS ( ON b.round = ab.block_id WHERE tx_type = 'pay' - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN_hevo AS ( - SELECT - ab.block_timestamp AS block_timestamp, - b.intra, - b.round AS block_id, - txn :txn :grp :: STRING AS tx_group_id, - CASE - WHEN b.txid IS NULL THEN ft.txn_txn_id :: text - ELSE b.txid :: text - END AS tx_id, - CASE - WHEN b.txid IS NULL THEN 'true' - ELSE 'false' - END AS inner_tx, - asset AS asset_id, - txn :txn :snd :: text AS sender, - txn :txn :rcv :: text AS receiver, - txn :txn :amt / pow( - 10, - 6 - ) AS amount, - txn :txn :fee / pow( - 10, - 6 - ) AS fee, - txn :txn :type :: STRING AS tx_type, - CASE - WHEN b.txid IS NULL THEN ft.genesis_hash :: text - ELSE txn :txn :gh :: STRING - END AS genesis_hash, - txn AS tx_message, - extra - FROM - {{ source( - 'algorand', - 'TXN_MISSING' - ) }} - b - LEFT JOIN {{ ref('silver_algorand__inner_txids') }} - ft - ON b.round = ft.inner_round - AND b.intra = ft.inner_intra - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON b.round = ab.block_id - WHERE - tx_type = 'pay' - AND b.round > ( - SELECT - MAX(ROUND) - FROM - {{ source( - 'algorand', - 'TXN' - ) }} - ) - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN AS( - SELECT - * - FROM - allTXN_fivetran - UNION - SELECT - * - FROM - allTXN_hevo ) SELECT block_timestamp, @@ -173,9 +84,22 @@ SELECT block_id :: STRING, intra :: STRING ) AS _unique_key, - SYSDATE() AS _inserted_timestamp + b._INSERTED_TIMESTAMP FROM allTXN b LEFT JOIN {{ ref('silver_algorand__transaction_types') }} csv ON b.tx_type = csv.type +WHERE + 1 = 1 + +{% if is_incremental() %} +AND b._INSERTED_TIMESTAMP >= ( + SELECT + MAX( + _INSERTED_TIMESTAMP + ) + FROM + {{ this }} +) +{% endif %} diff --git a/models/algorand/silver/silver_algorand__payment_transaction.yml b/models/algorand/silver/silver_algorand__payment_transaction.yml index 2fe3fc12..9ab686cb 100644 --- a/models/algorand/silver/silver_algorand__payment_transaction.yml +++ b/models/algorand/silver/silver_algorand__payment_transaction.yml @@ -9,11 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False # FIXME - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: hour - interval: 50 + - not_null - name: BLOCK_ID tests: - not_null @@ -31,9 +27,7 @@ models: - not_null - name: SENDER tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 + - not_null - name: TX_TYPE tests: - not_null @@ -45,9 +39,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null - - name: _INSERTED_TIMESTAMP::DATE - tests: - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + - not_null \ No newline at end of file diff --git a/models/algorand/silver/silver_algorand__transaction_participation.sql b/models/algorand/silver/silver_algorand__transaction_participation.sql index 5f7db95f..8c4e36de 100644 --- a/models/algorand/silver/silver_algorand__transaction_participation.sql +++ b/models/algorand/silver/silver_algorand__transaction_participation.sql @@ -11,9 +11,8 @@ WITH inner_tx_individual AS( SELECT ROUND AS block_id, intra, - algorand_decode_hex_addr( - addr :: text - ) AS address + addr :: text AS address, + DATEADD(ms, MAX(__HEVO__LOADED_AT), '1970-01-01') AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', @@ -22,7 +21,11 @@ WITH inner_tx_individual AS( {% if is_incremental() %} WHERE - _FIVETRAN_SYNCED >= ( + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) >= ( SELECT MAX( _INSERTED_TIMESTAMP @@ -35,71 +38,23 @@ GROUP BY block_id, intra, address -), -hevo_inner_tx_individual AS( - SELECT - ROUND AS block_id, - intra, - algorand_decode_hex_addr( - addr :: text - ) AS address - FROM - {{ source( - 'algorand', - 'TXN_PARTICIPATION_MISSING' - ) }} - WHERE - ROUND > ( - SELECT - MAX(ROUND) - FROM - {{ source( - 'algorand', - 'TXN_PARTICIPATION' - ) }} - ) - -{% if is_incremental() %} -WHERE - _FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} - ) -{% endif %} -GROUP BY - block_id, - intra, - address -), -all_inner_tx_individual AS( - SELECT - * - FROM - inner_tx_individual - UNION - SELECT - * - FROM - hevo_inner_tx_individual ) SELECT ab.block_timestamp AS block_timestamp, iti.block_id, iti.intra, - address, + algorand_decode_hex_addr( + iti.address :: text + ) AS address, concat_ws( '-', iti.block_id :: STRING, - iti.intra :: STRING, - iti.address :: STRING + intra :: STRING, + address :: STRING ) AS _unique_key, - SYSDATE() AS _INSERTED_TIMESTAMP + iti._INSERTED_TIMESTAMP FROM - all_inner_tx_individual iti + inner_tx_individual iti LEFT JOIN {{ ref('silver_algorand__block') }} ab ON iti.block_id = ab.block_id diff --git a/models/algorand/silver/silver_algorand__transaction_participation.yml b/models/algorand/silver/silver_algorand__transaction_participation.yml index 96a8c345..eaf2f13b 100644 --- a/models/algorand/silver/silver_algorand__transaction_participation.yml +++ b/models/algorand/silver/silver_algorand__transaction_participation.yml @@ -10,11 +10,10 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day - interval: 50 + interval: 1 - name: BLOCK_ID tests: - not_null @@ -23,11 +22,4 @@ models: - not_null - name: ADDRESS tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - - name: _INSERTED_TIMESTAMP::DATE - tests: - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + - not_null \ No newline at end of file diff --git a/models/algorand/silver/silver_algorand__transactions.sql b/models/algorand/silver/silver_algorand__transactions.sql index d66dc8e0..1b8d5300 100644 --- a/models/algorand/silver/silver_algorand__transactions.sql +++ b/models/algorand/silver/silver_algorand__transactions.sql @@ -6,11 +6,11 @@ tags = ['snowflake', 'algorand', 'transactions', 'silver_algorand_tx'] ) }} -WITH allTXN_fivetran AS ( +WITH allTXN AS ( SELECT ab.block_timestamp AS block_timestamp, - b.intra AS intra, + b.intra, b.round AS block_id, txn :txn :grp :: STRING AS tx_group_id, CASE @@ -36,7 +36,8 @@ WITH allTXN_fivetran AS ( ELSE txn :txn :gh :: STRING END AS genesis_hash, txn AS tx_message, - extra + extra, + b.__HEVO__LOADED_AT AS _INSERTED_TIMESTAMP FROM {{ source( 'algorand', @@ -50,102 +51,14 @@ WITH allTXN_fivetran AS ( LEFT JOIN {{ ref('silver_algorand__block') }} ab ON b.round = ab.block_id - -{% if is_incremental() %} -WHERE - b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} - ) -{% endif %} -), -allTXN_hevo AS ( - SELECT - ab.block_timestamp AS block_timestamp, - b.intra AS intra, - b.round AS block_id, - txn :txn :grp :: STRING AS tx_group_id, - CASE - WHEN b.txid IS NULL THEN ft.txn_txn_id :: text - ELSE b.txid :: text - END AS tx_id, - CASE - WHEN b.txid IS NULL THEN 'true' - ELSE 'false' - END AS inner_tx, - CASE - WHEN txn :txn :type :: STRING = 'appl' THEN NULL - ELSE asset - END AS asset_id, - txn :txn :snd :: text AS sender, - txn :txn :fee / pow( - 10, - 6 - ) AS fee, - txn :txn :type :: STRING AS tx_type, - CASE - WHEN b.txid IS NULL THEN ft.genesis_hash :: text - ELSE txn :txn :gh :: STRING - END AS genesis_hash, - txn AS tx_message, - extra - FROM - {{ source( - 'algorand', - 'TXN_MISSING' - ) }} - b - LEFT JOIN {{ ref('silver_algorand__inner_txids') }} - ft - ON b.round = ft.inner_round - AND b.intra = ft.inner_intra - LEFT JOIN {{ ref('silver_algorand__block') }} - ab - ON b.round = ab.block_id - WHERE - b.round > ( - SELECT - MAX(ROUND) - FROM - {{ source( - 'algorand', - 'TXN' - ) }} - ) - -{% if is_incremental() %} -AND b._FIVETRAN_SYNCED >= ( - SELECT - MAX( - _INSERTED_TIMESTAMP - ) - FROM - {{ this }} -) -{% endif %} -), -allTXN AS( - SELECT - * - FROM - allTXN_fivetran - UNION - SELECT - * - FROM - allTXN_hevo ) SELECT block_timestamp, intra, block_id, tx_group_id, - TRY_HEX_DECODE_STRING( - b.tx_id :: text + HEX_DECODE_STRING( + tx_id ) AS tx_id, TO_BOOLEAN(inner_tx) AS inner_tx, asset_id, @@ -163,9 +76,22 @@ SELECT block_id :: STRING, intra :: STRING ) AS _unique_key, - SYSDATE() AS _inserted_timestamp + b._INSERTED_TIMESTAMP FROM allTXN b LEFT JOIN {{ ref('silver_algorand__transaction_types') }} csv ON b.tx_type = csv.type +WHERE + 1 = 1 + +{% if is_incremental() %} +AND b._INSERTED_TIMESTAMP >= ( + SELECT + MAX( + _INSERTED_TIMESTAMP + ) + FROM + {{ this }} +) +{% endif %} diff --git a/models/algorand/silver/silver_algorand__transactions.yml b/models/algorand/silver/silver_algorand__transactions.yml index 92bdbcad..38c8b729 100644 --- a/models/algorand/silver/silver_algorand__transactions.yml +++ b/models/algorand/silver/silver_algorand__transactions.yml @@ -9,11 +9,10 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: - enabled: False # FIXME + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: hour - interval: 50 + datepart: day + interval: 1 - name: BLOCK_ID tests: - not_null @@ -26,11 +25,6 @@ models: - name: INNER_TX tests: - not_null - - name: SENDER - tests: - - not_null - - dbt_expectations.expect_column_value_lengths_to_equal: - value: 58 - name: TX_TYPE tests: - not_null @@ -42,9 +36,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null - - name: _INSERTED_TIMESTAMP::DATE - tests: - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + - not_null \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index bf95eb8c..c66c02e6 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -134,12 +134,7 @@ sources: tables: - name: daily_balances - name: algorand - schema: BRONZE_ALGORAND_2_8_0_PUBLIC - tables: - - name: TXN - - name: TXN_PARTICIPATION - - name: algorand - schema: BRONZE_ALGORAND_2_8_0__2_PUBLIC + schema: BRONZE_ALGORAND_2_8_0__HEVO_PARALLEL tables: - name: ACCOUNT - name: ACCOUNT_APP @@ -148,8 +143,8 @@ sources: - name: ASSET - name: BLOCK_HEADER - name: METASTATE - - name: TXN_PARTICIPATION_MISSING - - name: TXN_MISSING + - name: TXN + - name: TXN_PARTICIPATION - name: prod database: chainwalkers schema: prod From 8859c37fce923c4de78a73b159662c9e4b9fe288 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Fri, 4 Mar 2022 16:23:25 -0800 Subject: [PATCH 34/35] - added back missing columns that were in main --- models/algorand/gold/algorand__account.sql | 1 + models/algorand/gold/algorand__account.yml | 10 +++++++- .../algorand/gold/algorand__account_app.sql | 1 + .../algorand/gold/algorand__account_app.yml | 10 +++++++- .../algorand/gold/algorand__account_asset.sql | 1 + .../algorand/gold/algorand__account_asset.yml | 10 +++++++- models/algorand/gold/algorand__app.sql | 1 + models/algorand/gold/algorand__app.yml | 7 +++++- ...algorand__application_call_transaction.yml | 13 +++++++---- ...orand__asset_configuration_transaction.yml | 12 ++++++---- .../algorand__asset_freeze_transaction.yml | 8 ++++--- .../algorand__asset_transfer_transaction.yml | 12 +++++++--- models/algorand/gold/algorand__block.yml | 6 ++--- ...algorand__key_registration_transaction.yml | 8 ++++--- .../gold/algorand__payment_transaction.yml | 7 ++++-- .../algorand__transaction_participation.yml | 8 ++++--- .../algorand/gold/algorand__transactions.yml | 11 ++++++--- .../silver/silver_algorand__account.sql | 23 +++++++++++-------- .../silver/silver_algorand__account.yml | 13 +++++++++++ .../silver/silver_algorand__account_app.sql | 20 +++++++++------- .../silver/silver_algorand__account_app.yml | 15 +++++++++++- .../silver/silver_algorand__account_asset.sql | 16 ++++++++----- .../silver/silver_algorand__account_asset.yml | 18 +++++++++++---- .../algorand/silver/silver_algorand__app.sql | 17 ++++++++------ .../algorand/silver/silver_algorand__app.yml | 7 +++++- ...algorand__application_call_transaction.sql | 9 ++------ ...algorand__application_call_transaction.yml | 16 +++++++++---- ...orand__asset_configuration_transaction.sql | 5 ++-- ...orand__asset_configuration_transaction.yml | 12 ++++++---- 29 files changed, 207 insertions(+), 90 deletions(-) diff --git a/models/algorand/gold/algorand__account.sql b/models/algorand/gold/algorand__account.sql index cd5d12c5..11ea6533 100644 --- a/models/algorand/gold/algorand__account.sql +++ b/models/algorand/gold/algorand__account.sql @@ -11,6 +11,7 @@ SELECT balance, closed_at, created_at, + created_at_timestamp, wallet_type, account_data FROM diff --git a/models/algorand/gold/algorand__account.yml b/models/algorand/gold/algorand__account.yml index ca643b06..71420086 100644 --- a/models/algorand/gold/algorand__account.yml +++ b/models/algorand/gold/algorand__account.yml @@ -9,6 +9,8 @@ models: - name: ADDRESS tests: - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: ACCOUNT_CLOSED tests: - not_null @@ -28,8 +30,14 @@ models: - name: CREATED_AT tests: - not_null + - name: CREATED_AT_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 50 - name: WALLET_TYPE tests: - accepted_values: values: ["sig", "msig", "lsig", "", null] - enabled: False # unhandled deletions causes this test to fail \ No newline at end of file + enabled: False # unhandled deletions causes this test to fail diff --git a/models/algorand/gold/algorand__account_app.sql b/models/algorand/gold/algorand__account_app.sql index ac232670..bea54717 100644 --- a/models/algorand/gold/algorand__account_app.sql +++ b/models/algorand/gold/algorand__account_app.sql @@ -9,6 +9,7 @@ SELECT app_closed, closed_at, created_at, + created_at_timestamp, app_info FROM {{ ref('silver_algorand__account_app') }} diff --git a/models/algorand/gold/algorand__account_app.yml b/models/algorand/gold/algorand__account_app.yml index 2d6b01bb..63be8558 100644 --- a/models/algorand/gold/algorand__account_app.yml +++ b/models/algorand/gold/algorand__account_app.yml @@ -10,6 +10,8 @@ models: - name: ADDRESS tests: - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: APP_ID tests: - not_null @@ -19,7 +21,13 @@ models: - name: CREATED_AT tests: - not_null + - name: CREATED_AT_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 50 - name: APP_INFO tests: - not_null: - where: APP_CLOSED = FALSE \ No newline at end of file + where: APP_CLOSED = FALSE diff --git a/models/algorand/gold/algorand__account_asset.sql b/models/algorand/gold/algorand__account_asset.sql index e47efdb0..091c2fbf 100644 --- a/models/algorand/gold/algorand__account_asset.sql +++ b/models/algorand/gold/algorand__account_asset.sql @@ -9,6 +9,7 @@ SELECT asset_name, amount, asset_added_at, + added_at_timestamp, asset_last_removed, asset_closed, frozen diff --git a/models/algorand/gold/algorand__account_asset.yml b/models/algorand/gold/algorand__account_asset.yml index 05567497..e04f4560 100644 --- a/models/algorand/gold/algorand__account_asset.yml +++ b/models/algorand/gold/algorand__account_asset.yml @@ -10,6 +10,8 @@ models: - name: ADDRESS tests: - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: ASSET_ID tests: - not_null @@ -25,9 +27,15 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER + - name: ADDED_AT_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 50 - name: FROZEN tests: - not_null - name: ASSET_CLOSED tests: - - not_null \ No newline at end of file + - not_null diff --git a/models/algorand/gold/algorand__app.sql b/models/algorand/gold/algorand__app.sql index b4bd3472..a78b311c 100644 --- a/models/algorand/gold/algorand__app.sql +++ b/models/algorand/gold/algorand__app.sql @@ -9,6 +9,7 @@ SELECT app_closed, closed_at, created_at, + created_at_timestamp, params FROM {{ ref('silver_algorand__app') }} diff --git a/models/algorand/gold/algorand__app.yml b/models/algorand/gold/algorand__app.yml index b681eca9..fd45728c 100644 --- a/models/algorand/gold/algorand__app.yml +++ b/models/algorand/gold/algorand__app.yml @@ -12,9 +12,14 @@ models: - name: CREATOR_ADDRESS tests: - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: APP_CLOSED tests: - not_null - name: CREATED_AT tests: - - not_null \ No newline at end of file + - not_null + - name: CREATED_AT_TIMESTAMP + tests: + - not_null diff --git a/models/algorand/gold/algorand__application_call_transaction.yml b/models/algorand/gold/algorand__application_call_transaction.yml index 4faa7686..32381e89 100644 --- a/models/algorand/gold/algorand__application_call_transaction.yml +++ b/models/algorand/gold/algorand__application_call_transaction.yml @@ -9,7 +9,10 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null + - not_null: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 50 - name: block_id tests: - not_null @@ -24,11 +27,11 @@ models: - not_null - name: SENDER tests: - - not_null + - not_null - name: FEE tests: - not_null: - where: inner_tx = 'true' + where: inner_tx = 'true' - name: TX_TYPE tests: - not_null @@ -37,7 +40,7 @@ models: - not_null - name: GENESIS_HASH tests: - - not_null + - not_null - name: TX_MESSAGE tests: - - not_null \ No newline at end of file + - not_null diff --git a/models/algorand/gold/algorand__asset_configuration_transaction.yml b/models/algorand/gold/algorand__asset_configuration_transaction.yml index be222a27..9a4c5cb5 100644 --- a/models/algorand/gold/algorand__asset_configuration_transaction.yml +++ b/models/algorand/gold/algorand__asset_configuration_transaction.yml @@ -25,14 +25,16 @@ models: - name: ASSET_ID tests: - not_null: - where: inner_tx = 'false' + where: inner_tx = 'false' - name: SENDER tests: - - not_null + - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: FEE tests: - not_null: - where: inner_tx = 'true' + where: inner_tx = 'true' - name: TX_TYPE tests: - not_null @@ -41,7 +43,7 @@ models: - not_null - name: GENESIS_HASH tests: - - not_null + - not_null - name: TX_MESSAGE tests: - - not_null \ No newline at end of file + - not_null diff --git a/models/algorand/gold/algorand__asset_freeze_transaction.yml b/models/algorand/gold/algorand__asset_freeze_transaction.yml index 08c43640..75f67219 100644 --- a/models/algorand/gold/algorand__asset_freeze_transaction.yml +++ b/models/algorand/gold/algorand__asset_freeze_transaction.yml @@ -30,10 +30,12 @@ models: - not_null - name: SENDER tests: - - not_null + - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: FEE tests: - - not_null + - not_null - name: TX_TYPE tests: - not_null @@ -45,4 +47,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null \ No newline at end of file + - not_null diff --git a/models/algorand/gold/algorand__asset_transfer_transaction.yml b/models/algorand/gold/algorand__asset_transfer_transaction.yml index 5bea0073..22163750 100644 --- a/models/algorand/gold/algorand__asset_transfer_transaction.yml +++ b/models/algorand/gold/algorand__asset_transfer_transaction.yml @@ -5,7 +5,11 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null + - not_null: + enabled: False + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 50 - name: BLOCK_ID tests: - not_null @@ -23,7 +27,9 @@ models: - not_null - name: SENDER tests: - - not_null + - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: TX_TYPE tests: - not_null @@ -35,4 +41,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null \ No newline at end of file + - not_null diff --git a/models/algorand/gold/algorand__block.yml b/models/algorand/gold/algorand__block.yml index 3ec9cdbe..92383c31 100644 --- a/models/algorand/gold/algorand__block.yml +++ b/models/algorand/gold/algorand__block.yml @@ -13,8 +13,8 @@ models: tests: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + datepart: hour + interval: 50 - name: REWARDSLEVEL tests: - not_null @@ -30,4 +30,4 @@ models: where: BLOCK_ID <> 0 - name: HEADER tests: - - not_null \ No newline at end of file + - not_null diff --git a/models/algorand/gold/algorand__key_registration_transaction.yml b/models/algorand/gold/algorand__key_registration_transaction.yml index 7e6ab096..2ea89287 100644 --- a/models/algorand/gold/algorand__key_registration_transaction.yml +++ b/models/algorand/gold/algorand__key_registration_transaction.yml @@ -27,10 +27,12 @@ models: - not_null - name: SENDER tests: - - not_null + - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: FEE tests: - - not_null + - not_null - name: TX_TYPE tests: - not_null @@ -42,4 +44,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null \ No newline at end of file + - not_null diff --git a/models/algorand/gold/algorand__payment_transaction.yml b/models/algorand/gold/algorand__payment_transaction.yml index 5f663a38..ac64edd4 100644 --- a/models/algorand/gold/algorand__payment_transaction.yml +++ b/models/algorand/gold/algorand__payment_transaction.yml @@ -6,6 +6,9 @@ models: - name: BLOCK_TIMESTAMP tests: - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 50 - name: BLOCK_ID tests: - not_null @@ -23,7 +26,7 @@ models: - not_null - name: SENDER tests: - - not_null + - not_null - name: TX_TYPE tests: - not_null @@ -35,4 +38,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null \ No newline at end of file + - not_null diff --git a/models/algorand/gold/algorand__transaction_participation.yml b/models/algorand/gold/algorand__transaction_participation.yml index 28437337..8433e469 100644 --- a/models/algorand/gold/algorand__transaction_participation.yml +++ b/models/algorand/gold/algorand__transaction_participation.yml @@ -12,8 +12,8 @@ models: tests: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + datepart: hour + interval: 50 - name: BLOCK_ID tests: - not_null @@ -22,4 +22,6 @@ models: - not_null - name: ADDRESS tests: - - not_null \ No newline at end of file + - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 diff --git a/models/algorand/gold/algorand__transactions.yml b/models/algorand/gold/algorand__transactions.yml index 132ffe87..f0d0d71b 100644 --- a/models/algorand/gold/algorand__transactions.yml +++ b/models/algorand/gold/algorand__transactions.yml @@ -7,8 +7,8 @@ models: tests: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + datepart: hour + interval: 50 - name: BLOCK_ID tests: - not_null @@ -21,6 +21,11 @@ models: - name: INNER_TX tests: - not_null + - name: SENDER + tests: + - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: TX_TYPE tests: - not_null @@ -32,4 +37,4 @@ models: - not_null - name: TX_MESSAGE tests: - - not_null \ No newline at end of file + - not_null diff --git a/models/algorand/silver/silver_algorand__account.sql b/models/algorand/silver/silver_algorand__account.sql index 2ba7e805..e4850d5d 100644 --- a/models/algorand/silver/silver_algorand__account.sql +++ b/models/algorand/silver/silver_algorand__account.sql @@ -7,25 +7,26 @@ SELECT algorand_decode_hex_addr( - addr :: text + aa.addr :: text ) AS address, - deleted AS account_closed, - rewardsbase / pow( + aa.deleted AS account_closed, + aa.rewardsbase / pow( 10, 6 ) AS rewardsbase, - rewards_total / pow( + aa.rewards_total / pow( 10, 6 ) AS rewards_total, - microalgos / pow( + aa.microalgos / pow( 10, 6 ) AS balance, - closed_at AS closed_at, - created_at AS created_at, - keytype AS wallet_type, - account_data AS account_data, + aa.closed_at AS closed_at, + aa.created_at AS created_at, + ab.block_timestamp AS created_at_timestamp, + aa.keytype AS wallet_type, + aa.account_data AS account_data, DATEADD( ms, __HEVO__LOADED_AT, @@ -36,6 +37,10 @@ FROM 'algorand', 'ACCOUNT' ) }} + aa + LEFT JOIN {{ ref('silver_algorand__block') }} + ab + ON aa.created_at = ab.block_id WHERE 1 = 1 diff --git a/models/algorand/silver/silver_algorand__account.yml b/models/algorand/silver/silver_algorand__account.yml index 705991b9..3033a579 100644 --- a/models/algorand/silver/silver_algorand__account.yml +++ b/models/algorand/silver/silver_algorand__account.yml @@ -9,6 +9,8 @@ models: - name: ADDRESS tests: - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: ACCOUNT_CLOSED tests: - not_null @@ -28,8 +30,19 @@ models: - name: CREATED_AT tests: - not_null + - name: CREATED_AT_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 50 - name: WALLET_TYPE tests: - accepted_values: values: ["sig", "msig", "lsig", "", null] enabled: False # unhandled deletions causes this test to fail + - name: _INSERTED_TIMESTAMP::DATE + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 diff --git a/models/algorand/silver/silver_algorand__account_app.sql b/models/algorand/silver/silver_algorand__account_app.sql index 56b8b239..570aa40d 100644 --- a/models/algorand/silver/silver_algorand__account_app.sql +++ b/models/algorand/silver/silver_algorand__account_app.sql @@ -7,17 +7,18 @@ SELECT algorand_decode_hex_addr( - addr :: text + b.addr :: text ) AS address, - app AS app_id, - deleted AS app_closed, - closed_at AS closed_at, - created_at AS created_at, - localstate AS app_info, + b.app AS app_id, + b.deleted AS app_closed, + b.closed_at AS closed_at, + b.created_at AS created_at, + ab.block_timestamp AS created_at_timestamp, + b.localstate AS app_info, concat_ws( '-', - addr :: STRING, - app :: STRING + b.addr :: STRING, + b.app :: STRING ) AS _unique_key, DATEADD( ms, @@ -29,6 +30,9 @@ FROM 'algorand', 'ACCOUNT_APP' ) }} + LEFT JOIN {{ ref('silver_algorand__block') }} + ab + ON b.created_at = ab.block_id WHERE 1 = 1 diff --git a/models/algorand/silver/silver_algorand__account_app.yml b/models/algorand/silver/silver_algorand__account_app.yml index ec5c9c3f..133296b0 100644 --- a/models/algorand/silver/silver_algorand__account_app.yml +++ b/models/algorand/silver/silver_algorand__account_app.yml @@ -10,6 +10,8 @@ models: - name: ADDRESS tests: - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: APP_ID tests: - not_null @@ -19,7 +21,18 @@ models: - name: CREATED_AT tests: - not_null + - name: CREATED_AT_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 50 - name: APP_INFO tests: - not_null: - where: APP_CLOSED = FALSE \ No newline at end of file + where: APP_CLOSED = FALSE + - name: _INSERTED_TIMESTAMP::DATE + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 diff --git a/models/algorand/silver/silver_algorand__account_asset.sql b/models/algorand/silver/silver_algorand__account_asset.sql index 404c6796..d25091af 100644 --- a/models/algorand/silver/silver_algorand__account_asset.sql +++ b/models/algorand/silver/silver_algorand__account_asset.sql @@ -20,13 +20,14 @@ SELECT algorand_decode_hex_addr( aa.addr :: text ) AS address, - assetid AS asset_id, + aa.assetid AS asset_id, an.name :: STRING AS asset_name, - amount :: NUMBER AS amount, - created_at AS asset_added_at, - closed_at AS asset_last_removed, - deleted AS asset_closed, - frozen AS frozen, + aa.amount AS amount, + aa.created_at AS asset_added_at, + ab.block_timestamp AS added_at_timestamp, + aa.closed_at AS asset_last_removed, + aa.deleted AS asset_closed, + aa.frozen AS frozen, concat_ws( '-', address :: STRING, @@ -45,6 +46,9 @@ FROM aa LEFT JOIN asset_name an ON aa.assetid = an.index + LEFT JOIN {{ ref('silver_algorand__block') }} + ab + ON aa.created_at = ab.block_id WHERE 1 = 1 diff --git a/models/algorand/silver/silver_algorand__account_asset.yml b/models/algorand/silver/silver_algorand__account_asset.yml index f44e7244..64b2c942 100644 --- a/models/algorand/silver/silver_algorand__account_asset.yml +++ b/models/algorand/silver/silver_algorand__account_asset.yml @@ -10,6 +10,8 @@ models: - name: ADDRESS tests: - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: ASSET_ID tests: - not_null @@ -25,14 +27,20 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER + - name: ADDED_AT_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 50 - name: FROZEN tests: - not_null - name: ASSET_CLOSED tests: - not_null - - - - - + - name: _INSERTED_TIMESTAMP::DATE + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 diff --git a/models/algorand/silver/silver_algorand__app.sql b/models/algorand/silver/silver_algorand__app.sql index a33b9d74..635e30c9 100644 --- a/models/algorand/silver/silver_algorand__app.sql +++ b/models/algorand/silver/silver_algorand__app.sql @@ -8,12 +8,13 @@ SELECT INDEX AS app_id, algorand_decode_hex_addr( - creator :: text + aa.creator :: text ) AS creator_address, - deleted AS app_closed, - closed_at AS closed_at, - created_at AS created_at, - params, + aa.deleted AS app_closed, + aa.closed_at AS closed_at, + aa.created_at AS created_at, + ab.block_timestamp AS created_at_timestamp, + aa.params, DATEADD( ms, __HEVO__LOADED_AT, @@ -24,8 +25,10 @@ FROM 'algorand', 'APP' ) }} -WHERE - 1 = 1 + aa + LEFT JOIN {{ ref('silver_algorand__block') }} + ab + ON aa.created_at = ab.block_idWHERE 1 = 1 {% if is_incremental() %} AND DATEADD( diff --git a/models/algorand/silver/silver_algorand__app.yml b/models/algorand/silver/silver_algorand__app.yml index ff8324cf..e941c3dd 100644 --- a/models/algorand/silver/silver_algorand__app.yml +++ b/models/algorand/silver/silver_algorand__app.yml @@ -12,9 +12,14 @@ models: - name: CREATOR_ADDRESS tests: - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: APP_CLOSED tests: - not_null - name: CREATED_AT tests: - - not_null \ No newline at end of file + - not_null + - name: CREATED_AT_TIMESTAMP + tests: + - not_null diff --git a/models/algorand/silver/silver_algorand__application_call_transaction.sql b/models/algorand/silver/silver_algorand__application_call_transaction.sql index 66a275f3..0c32005a 100644 --- a/models/algorand/silver/silver_algorand__application_call_transaction.sql +++ b/models/algorand/silver/silver_algorand__application_call_transaction.sql @@ -32,12 +32,7 @@ WITH allTXN AS ( ELSE txn :txn :gh :: STRING END AS genesis_hash, txn AS tx_message, - extra, - DATEADD( - ms, - b.__HEVO__LOADED_AT, - '1970-01-01' - ) AS _INSERTED_TIMESTAMP + extra FROM {{ source( 'algorand', @@ -94,7 +89,7 @@ SELECT block_id :: STRING, intra :: STRING ) AS _unique_key, - b._INSERTED_TIMESTAMP + SYSDATE() AS _INSERTED_TIMESTAMP FROM allTXN b LEFT JOIN {{ ref('silver_algorand__transaction_types') }} diff --git a/models/algorand/silver/silver_algorand__application_call_transaction.yml b/models/algorand/silver/silver_algorand__application_call_transaction.yml index 15ee477a..eb3deae8 100644 --- a/models/algorand/silver/silver_algorand__application_call_transaction.yml +++ b/models/algorand/silver/silver_algorand__application_call_transaction.yml @@ -10,6 +10,9 @@ models: - name: BLOCK_TIMESTAMP tests: - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 50 - name: block_id tests: - not_null @@ -24,11 +27,11 @@ models: - not_null - name: SENDER tests: - - not_null + - not_null - name: FEE tests: - not_null: - where: inner_tx = 'true' + where: inner_tx = 'true' - name: TX_TYPE tests: - not_null @@ -37,7 +40,12 @@ models: - not_null - name: GENESIS_HASH tests: - - not_null + - not_null - name: TX_MESSAGE tests: - - not_null \ No newline at end of file + - not_null + - name: _INSERTED_TIMESTAMP::DATE + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 diff --git a/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql b/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql index 23c9c460..0f92ac47 100644 --- a/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql +++ b/models/algorand/silver/silver_algorand__asset_configuration_transaction.sql @@ -34,8 +34,7 @@ WITH allTXN AS ( ELSE txn :txn :gh :: STRING END AS genesis_hash, txn AS tx_message, - extra, - b.__HEVO__LOADED_AT AS _INSERTED_TIMESTAMP + extra FROM {{ source( 'algorand', @@ -78,7 +77,7 @@ SELECT block_id :: STRING, intra :: STRING ) AS _unique_key, - b._INSERTED_TIMESTAMP + SYSDATE() AS _inserted_timestamp FROM allTXN b LEFT JOIN {{ ref('silver_algorand__transaction_types') }} diff --git a/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml b/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml index e5389f2d..da097856 100644 --- a/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml +++ b/models/algorand/silver/silver_algorand__asset_configuration_transaction.yml @@ -25,14 +25,16 @@ models: - name: ASSET_ID tests: - not_null: - where: inner_tx = 'false' + where: inner_tx = 'false' - name: SENDER tests: - - not_null + - not_null + - dbt_expectations.expect_column_value_lengths_to_equal: + value: 58 - name: FEE tests: - not_null: - where: inner_tx = 'true' + where: inner_tx = 'true' - name: TX_TYPE tests: - not_null @@ -41,7 +43,7 @@ models: - not_null - name: GENESIS_HASH tests: - - not_null + - not_null - name: TX_MESSAGE tests: - - not_null \ No newline at end of file + - not_null From 41012070495438f7dd08b226e25f7b41f2fc6f44 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Fri, 4 Mar 2022 16:25:35 -0800 Subject: [PATCH 35/35] - removed extraneous colon --- models/algorand/gold/algorand__application_call_transaction.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/algorand/gold/algorand__application_call_transaction.yml b/models/algorand/gold/algorand__application_call_transaction.yml index 32381e89..dc946442 100644 --- a/models/algorand/gold/algorand__application_call_transaction.yml +++ b/models/algorand/gold/algorand__application_call_transaction.yml @@ -9,7 +9,7 @@ models: columns: - name: BLOCK_TIMESTAMP tests: - - not_null: + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: hour interval: 50