diff --git a/models/crosschain/crosschain__address_labels.sql b/models/crosschain/gold/crosschain__address_labels.sql similarity index 100% rename from models/crosschain/crosschain__address_labels.sql rename to models/crosschain/gold/crosschain__address_labels.sql diff --git a/models/crosschain/crosschain__address_labels.yml b/models/crosschain/gold/crosschain__address_labels.yml similarity index 100% rename from models/crosschain/crosschain__address_labels.yml rename to models/crosschain/gold/crosschain__address_labels.yml diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd.sql b/models/crosschain/gold/crosschain__f_market_stats_usd.sql new file mode 100644 index 00000000..454de506 --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = "view", + unique_key = "_unique_key", + tags = ["crosschain", "gold", "fact", "f_stats_usd"] +) }} + +SELECT + asset_id, + provider, + platform, + recorded_at, + circulating_supply, + market_cap, + price, + max_supply, + total_supply, + volume_24h, + _updated, + _unique_key +FROM + {{ ref( + "silver_crosschain__f_market_stats_usd" + ) }} diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd.yml b/models/crosschain/gold/crosschain__f_market_stats_usd.yml new file mode 100644 index 00000000..8fe57ee2 --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: crosschain__f_market_stats_usd + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - NAME + - SYMBOL + - PROVIDER + - RECORDED_AT + columns: + - name: ASSET_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: NAME + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: RECORDED_AT + tests: + - not_null + - name: CIRCULATING_SUPPLY + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: _UPDATED + tests: + - not_null + - name: _UNIQUE_KEY + tests: + - not_null diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql new file mode 100644 index 00000000..c4c92d4b --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.sql @@ -0,0 +1,107 @@ +{{ config( + materialized = "incremental", + unique_key = "_unique_key", + incremental_strategy = "merge", + tags = ["crosschain", "gold", "fact", "f_stats_usd", "aggregate", "daily"] +) }} + +SELECT + asset_id, + provider, + platform, + DATE_TRUNC( + DAY, + recorded_at + ) AS recorded_at, + COUNT(*) AS n, + AVG(circulating_supply) AS avg_circulating_supply, + AVG(market_cap) AS avg_market_cap, + AVG(max_supply) AS avg_max_supply, + AVG(price) AS avg_price, + AVG(total_supply) AS avg_total_supply, + AVG(volume_24h) AS avg_volume_24h, + MAX(circulating_supply) AS max_circulating_supply, + MAX(market_cap) AS max_market_cap, + MAX(max_supply) AS max_max_supply, + MAX(price) AS max_price, + MAX(total_supply) AS max_total_supply, + MAX(volume_24h) AS max_volume_24h, + MEDIAN(circulating_supply) AS median_circulating_supply, + MEDIAN(market_cap) AS median_market_cap, + MEDIAN(max_supply) AS median_max_supply, + MEDIAN(price) AS median_price, + MEDIAN(total_supply) AS median_total_supply, + MEDIAN(volume_24h) AS median_volume_24h, + MIN(circulating_supply) AS min_circulating_supply, + MIN(market_cap) AS min_market_cap, + MIN(max_supply) AS min_max_supply, + MIN(price) AS min_price, + MIN(total_supply) AS min_total_supply, + MIN(volume_24h) AS min_volume_24h, + ARRAY_AGG( + OBJECT_CONSTRUCT( + 'circulating_supply', + circulating_supply, + 'market_cap', + market_cap, + 'max_supply', + max_supply, + 'price', + price, + 'total_supply', + total_supply, + 'volume_24h', + volume_24h, + 'recorded_at', + recorded_at + ) + ) within GROUP ( + ORDER BY + recorded_at + ) AS DATA, + DATA [0] :market_cap AS first_market_cap, + DATA [0] :max_supply AS first_max_supply, + DATA [0] :price AS first_price, + DATA [0] :total_supply AS first_total_supply, + DATA [0] :volume_24h AS first_volume_24h, + DATA [ARRAY_SIZE(data)-1] :market_cap AS last_market_cap, + DATA [ARRAY_SIZE(data)-1] :max_supply AS last_max_supply, + DATA [ARRAY_SIZE(data)-1] :price AS last_price, + DATA [ARRAY_SIZE(data)-1] :total_supply AS last_total_supply, + DATA [ARRAY_SIZE(data)-1] :volume_24h AS last_volume_24h, + SYSDATE() AS _updated_timestamp, + concat_ws( + '-', + asset_id, + NAME, + symbol, + provider, + DATE_TRUNC( + DAY, + recorded_at + ) + ) AS _unique_key +FROM + {{ ref( + "silver_crosschain__f_market_stats_usd" + ) }} + +{% if is_incremental() %} +WHERE + recorded_at >= DATE_TRUNC( + DAY,( + SELECT + MAX(recorded_at) + FROM + {{ this }} + ) + ) - INTERVAL '2 DAYS' +{% endif %} +GROUP BY + asset_id, + provider, + platform, + DATE_TRUNC( + DAY, + recorded_at + ) diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml new file mode 100644 index 00000000..37bbd301 --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_daily.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: crosschain__f_market_stats_usd_agg_daily + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - NAME + - SYMBOL + - PROVIDER + - RECORDED_AT + columns: + - name: ASSET_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: NAME + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: RECORDED_AT + tests: + - not_null + - name: CIRCULATING_SUPPLY + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: _UPDATED + tests: + - not_null + - name: _UNIQUE_KEY + tests: + - not_null diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql new file mode 100644 index 00000000..898df4ab --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.sql @@ -0,0 +1,107 @@ +{{ config( + materialized = "incremental", + unique_key = "_unique_key", + incremental_strategy = "merge", + tags = ["crosschain", "gold", "fact", "f_stats_usd", "aggregate", "daily"] +) }} + +SELECT + asset_id, + provider, + platform, + DATE_TRUNC( + HOUR, + recorded_at + ) AS recorded_at, + COUNT(*) AS n, + AVG(circulating_supply) AS avg_circulating_supply, + AVG(market_cap) AS avg_market_cap, + AVG(max_supply) AS avg_max_supply, + AVG(price) AS avg_price, + AVG(total_supply) AS avg_total_supply, + AVG(volume_24h) AS avg_volume_24h, + MAX(circulating_supply) AS max_circulating_supply, + MAX(market_cap) AS max_market_cap, + MAX(max_supply) AS max_max_supply, + MAX(price) AS max_price, + MAX(total_supply) AS max_total_supply, + MAX(volume_24h) AS max_volume_24h, + MEDIAN(circulating_supply) AS median_circulating_supply, + MEDIAN(market_cap) AS median_market_cap, + MEDIAN(max_supply) AS median_max_supply, + MEDIAN(price) AS median_price, + MEDIAN(total_supply) AS median_total_supply, + MEDIAN(volume_24h) AS median_volume_24h, + MIN(circulating_supply) AS min_circulating_supply, + MIN(market_cap) AS min_market_cap, + MIN(max_supply) AS min_max_supply, + MIN(price) AS min_price, + MIN(total_supply) AS min_total_supply, + MIN(volume_24h) AS min_volume_24h, + ARRAY_AGG( + OBJECT_CONSTRUCT( + 'circulating_supply', + circulating_supply, + 'market_cap', + market_cap, + 'max_supply', + max_supply, + 'price', + price, + 'total_supply', + total_supply, + 'volume_24h', + volume_24h, + 'recorded_at', + recorded_at + ) + ) within GROUP ( + ORDER BY + recorded_at + ) AS DATA, + DATA [0] :market_cap AS first_market_cap, + DATA [0] :max_supply AS first_max_supply, + DATA [0] :price AS first_price, + DATA [0] :total_supply AS first_total_supply, + DATA [0] :volume_24h AS first_volume_24h, + DATA [ARRAY_SIZE(data)-1] :market_cap AS last_market_cap, + DATA [ARRAY_SIZE(data)-1] :max_supply AS last_max_supply, + DATA [ARRAY_SIZE(data)-1] :price AS last_price, + DATA [ARRAY_SIZE(data)-1] :total_supply AS last_total_supply, + DATA [ARRAY_SIZE(data)-1] :volume_24h AS last_volume_24h, + SYSDATE() AS _updated_timestamp, + concat_ws( + '-', + asset_id, + NAME, + symbol, + provider, + DATE_TRUNC( + HOUR, + recorded_at + ) + ) AS _unique_key +FROM + {{ ref( + "silver_crosschain__f_market_stats_usd" + ) }} + +{% if is_incremental() %} +WHERE + recorded_at >= DATE_TRUNC( + HOUR,( + SELECT + MAX(recorded_at) + FROM + {{ this }} + ) + ) - INTERVAL '1 hour' +{% endif %} +GROUP BY + asset_id, + provider, + platform, + DATE_TRUNC( + HOUR, + recorded_at + ) diff --git a/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml new file mode 100644 index 00000000..53a88a99 --- /dev/null +++ b/models/crosschain/gold/crosschain__f_market_stats_usd_agg_hourly.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: crosschain__f_market_stats_usd_agg_hourly + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - NAME + - SYMBOL + - PROVIDER + - RECORDED_AT + columns: + - name: ASSET_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: NAME + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: RECORDED_AT + tests: + - not_null + - name: CIRCULATING_SUPPLY + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: _UPDATED + tests: + - not_null + - name: _UNIQUE_KEY + tests: + - not_null diff --git a/models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql new file mode 100644 index 00000000..570704c6 --- /dev/null +++ b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.sql @@ -0,0 +1,65 @@ +{{ config( + materialized = "incremental", + cluster_by = "recorded_at::DATE", + unique_key = "_unique_key", + incremental_strategy = "merge", + tags = ["snowflake", "market", "stats", "gold", "fact", "f_stats_usd"] +) }} + +SELECT + COALESCE( + asset_id, + '#UNKNOWN' + ) AS asset_id, + provider, + NAME, + symbol, + recorded_at, + circulating_supply, + market_cap, + price, + max_supply, + total_supply, + volume_24h, + SYSDATE() AS _updated, + concat_ws( + '-', + NAME, + symbol, + provider, + to_varchar( + recorded_at, + 'YYYY-MM-DD HH24:MI:SS.FF9' + ) + ) AS _unique_key +FROM + {{ source( + "shared", + "prices_v2" + ) }} +WHERE + NAME IS NOT NULL + AND symbol IS NOT NULL + AND provider IS NOT NULL + AND recorded_at IS NOT NULL + +{% if is_incremental() %} +AND recorded_at >= ( + SELECT + MAX(recorded_at) + FROM + {{ this }} +) - INTERVAL '30 DAYS' +{% endif %} + +qualify ROW_NUMBER() over ( + PARTITION BY NAME, + symbol, + provider, + recorded_at + ORDER BY + to_varchar( + recorded_at, + 'YYYY-MM-DD HH24:MI:SS.FF9' + ) +) = 1 diff --git a/models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml new file mode 100644 index 00000000..a7a26b12 --- /dev/null +++ b/models/crosschain/silver/silver_crosschain__f_market_stats_usd.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: silver_crosschain__f_market_stats_usd + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - NAME + - SYMBOL + - PROVIDER + - RECORDED_AT + columns: + - name: ASSET_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: NAME + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: RECORDED_AT + tests: + - not_null + - name: CIRCULATING_SUPPLY + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: _UPDATED + tests: + - not_null + - name: _UNIQUE_KEY + tests: + - not_null diff --git a/models/ethereum/silver/silver_ethereum__contracts.sql b/models/ethereum/silver/silver_ethereum__contracts.sql index f8b1fe70..1678005e 100644 --- a/models/ethereum/silver/silver_ethereum__contracts.sql +++ b/models/ethereum/silver/silver_ethereum__contracts.sql @@ -6,60 +6,68 @@ tags = ['snowflake', 'ethereum', 'silver_ethereum','silver_ethereum__contracts'] ) }} -with base as ( +WITH base AS ( + + SELECT + system_created_at, + address, + meta, + NAME + FROM + ( + SELECT + system_created_at, + address, + meta, + NAME + FROM + {{ ref('ethereum_dbt__contracts') }} + WHERE + meta IS NOT NULL + UNION + SELECT + '2000-01-01' :: TIMESTAMP AS system_created_at, + address, + meta, + NAME + FROM + {{ source( + 'ethereum', + 'ethereum_contracts' + ) }} + UNION + SELECT + '2000-01-01' :: TIMESTAMP AS system_created_at, + contract_address AS address, + TO_OBJECT(PARSE_JSON(contract_meta)) AS meta, + NAME + FROM + {{ source( + 'ethereum', + 'ethereum_contracts_backfill' + ) }} + WHERE + CHECK_JSON(contract_meta) IS NULL + ) + WHERE + <<<<<<< head address IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY address + ORDER BY + system_created_at DESC)) = 1 ======= address IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY address + ORDER BY + system_created_at DESC)) = 1 +) SELECT system_created_at, address, meta, NAME FROM - ( - SELECT - system_created_at, - address, - meta, - NAME - FROM - {{ ref('ethereum_dbt__contracts') }} - WHERE - meta IS NOT NULL - UNION - SELECT - '2000-01-01' :: TIMESTAMP AS system_created_at, - address, - meta, - NAME - FROM - {{ source( - 'ethereum', - 'ethereum_contracts' - ) }} - UNION - SELECT - '2000-01-01' :: TIMESTAMP AS system_created_at, - contract_address AS address, - TO_OBJECT(PARSE_JSON(contract_meta)) AS meta, - NAME - FROM - {{ source( - 'ethereum', - 'ethereum_contracts_backfill' - ) }} - WHERE - CHECK_JSON(contract_meta) IS NULL - ) + base WHERE - address IS NOT NULL - qualify(ROW_NUMBER() over(PARTITION BY address -ORDER BY - system_created_at DESC)) = 1 -) - -select system_created_at, - address, - meta, - NAME -from base -where case when meta:decimals::string is not null and len(meta:decimals::string) >= 3 - then true - else false end = false \ No newline at end of file + CASE + WHEN meta :decimals :: STRING IS NOT NULL + AND len( + meta :decimals :: STRING + ) >= 3 THEN TRUE + ELSE FALSE + END = FALSE >>>>>>> main