From 5d29d53efc4e41810de341c7e384391ed25a96ff Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 27 Aug 2021 14:27:32 -0400 Subject: [PATCH 01/11] initial --- models/terra/gold/terra__blocks.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 models/terra/gold/terra__blocks.sql diff --git a/models/terra/gold/terra__blocks.sql b/models/terra/gold/terra__blocks.sql new file mode 100644 index 00000000..e69de29b From 261a36311eb588e2f6c637fa3180f7095c0cb5d7 Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 27 Aug 2021 14:30:27 -0400 Subject: [PATCH 02/11] update 2 tables --- models/terra/gold/{terra__blocks.sql => terra_address_labels.sql} | 0 models/terra/gold/terra_address_tags.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename models/terra/gold/{terra__blocks.sql => terra_address_labels.sql} (100%) create mode 100644 models/terra/gold/terra_address_tags.sql diff --git a/models/terra/gold/terra__blocks.sql b/models/terra/gold/terra_address_labels.sql similarity index 100% rename from models/terra/gold/terra__blocks.sql rename to models/terra/gold/terra_address_labels.sql diff --git a/models/terra/gold/terra_address_tags.sql b/models/terra/gold/terra_address_tags.sql new file mode 100644 index 00000000..e69de29b From 85506216dc7d2b28da3f4703f72d9d31f1eb6165 Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 27 Aug 2021 14:33:36 -0400 Subject: [PATCH 03/11] update --- models/terra/gold/terra_daily_balances.sql | 0 models/terra/gold/terra_decimals.sql | 0 models/terra/gold/terra_events.sql | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 models/terra/gold/terra_daily_balances.sql create mode 100644 models/terra/gold/terra_decimals.sql create mode 100644 models/terra/gold/terra_events.sql diff --git a/models/terra/gold/terra_daily_balances.sql b/models/terra/gold/terra_daily_balances.sql new file mode 100644 index 00000000..e69de29b diff --git a/models/terra/gold/terra_decimals.sql b/models/terra/gold/terra_decimals.sql new file mode 100644 index 00000000..e69de29b diff --git a/models/terra/gold/terra_events.sql b/models/terra/gold/terra_events.sql new file mode 100644 index 00000000..e69de29b From 88bf566eb7b427fc3393a8de63f4283f8ec69c07 Mon Sep 17 00:00:00 2001 From: Ryan-Loofy <63126328+Ryan-Loofy@users.noreply.github.com> Date: Fri, 27 Aug 2021 14:34:15 -0400 Subject: [PATCH 04/11] terra models added: prices, tax_rate, velocity, voting power --- models/terra/gold/terra_prices.sql | 68 ++++++++++++++++++++++++ models/terra/gold/terra_tax_rate.sql | 20 +++++++ models/terra/gold/terra_velocity.sql | 16 ++++++ models/terra/gold/terra_voting_power.sql | 21 ++++++++ 4 files changed, 125 insertions(+) create mode 100644 models/terra/gold/terra_prices.sql create mode 100644 models/terra/gold/terra_tax_rate.sql create mode 100644 models/terra/gold/terra_velocity.sql create mode 100644 models/terra/gold/terra_voting_power.sql diff --git a/models/terra/gold/terra_prices.sql b/models/terra/gold/terra_prices.sql new file mode 100644 index 00000000..03e15306 --- /dev/null +++ b/models/terra/gold/terra_prices.sql @@ -0,0 +1,68 @@ +{{ config( + materialized='incremental', + sort='hour', + unique_key='symbol || hour') +}} + +WITH hours AS ( + select hour + from + {{source('shared', 'hours')}} + where + {% if is_incremental() %} + hour >= getdate() - interval '3 days' and hour < getdate() + interval '1 day' + {% else %} + hour >= getdate() - interval '12 months' and hour < getdate() + interval '1 day' + {% endif %} +), +raw_prices AS ( + SELECT symbol, date_trunc('hour', recorded_at) as hour, avg(price) as price FROM {{source('shared','prices')}} WHERE asset_id IN ( + 4172, -- Luna + 5115, -- Terra KRW (KRT) + 6370 -- Terra SDT (SDR) + ) + {% if is_incremental() %} + AND recorded_at >= getdate() - interval '3 days' + {% else %} + AND recorded_at >= getdate() - interval '12 months' + {% endif %} + AND recorded_at < getdate() + interval '1 day' + GROUP BY symbol, hour + + UNION ALL + + SELECT symbol, date_trunc('hour', recorded_at) as hour, avg(price) as price + FROM {{source('shared','coingecko_prices')}} WHERE asset_id IN ('terrausd') + {% if is_incremental() %} + AND recorded_at >= getdate() - interval '3 days' + {% else %} + AND recorded_at >= getdate() - interval '12 months' + {% endif %} + AND recorded_at < getdate() + interval '1 day' + GROUP BY symbol, hour +), +assets AS ( + SELECT symbol FROM raw_prices GROUP BY 1 +), +hour_assets AS ( + SELECT + h.hour, + a.symbol + FROM + hours h + CROSS JOIN + assets a +) +SELECT +ha.hour, +ha.symbol, +-- use price if we have other, otherwise fall back to daily average +coalesce(rp.price, avg(rp.price) over(partition by date_trunc('day', ha.hour), ha.symbol)) as price +FROM +hour_assets ha +LEFT OUTER JOIN +raw_prices rp +ON +ha.symbol = rp.symbol +AND +ha.hour = rp.hour \ No newline at end of file diff --git a/models/terra/gold/terra_tax_rate.sql b/models/terra/gold/terra_tax_rate.sql new file mode 100644 index 00000000..55bccb3f --- /dev/null +++ b/models/terra/gold/terra_tax_rate.sql @@ -0,0 +1,20 @@ +{{ config( + materialized='incremental', + sort='block_timestamp', + unique_key='blockchain || block_number', + tags=['custom']) +}} + +SELECT + blockchain, + block_timestamp, + block_number, + tax_rate +FROM + {{source('terra','udm_custom_fields_terra_tax_rate')}} +WHERE +{% if is_incremental() %} + block_timestamp >= getdate() - interval '3 days' +{% else %} + block_timestamp >= getdate() - interval '12 months' +{% endif %} \ No newline at end of file diff --git a/models/terra/gold/terra_velocity.sql b/models/terra/gold/terra_velocity.sql new file mode 100644 index 00000000..5dde1330 --- /dev/null +++ b/models/terra/gold/terra_velocity.sql @@ -0,0 +1,16 @@ +{{ config( + materialized='table', + sort='start_date', + tags=['custom']) +}} +SELECT + blockchain, + start_date, + end_date, + currency, + metric, + value +FROM + {{ source('shared', 'udm_velocity')}} +WHERE + blockchain = 'terra' \ No newline at end of file diff --git a/models/terra/gold/terra_voting_power.sql b/models/terra/gold/terra_voting_power.sql new file mode 100644 index 00000000..47ba8b24 --- /dev/null +++ b/models/terra/gold/terra_voting_power.sql @@ -0,0 +1,21 @@ +{{ config( + materialized='incremental', + sort='block_timestamp', + unique_key='blockchain || block_number', + tags=['custom']) +}} + +SELECT + blockchain, + block_timestamp, + block_number, + address, + voting_power +FROM + {{source('terra','terra_validator_voting_power')}} +WHERE + {% if is_incremental() %} + block_timestamp >= getdate() - interval '3 days' + {% else %} + block_timestamp >= getdate() - interval '12 months' + {% endif %} \ No newline at end of file From 6101873349487cfe69273aa1aba3df6bac3bab0a Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 27 Aug 2021 17:02:15 -0400 Subject: [PATCH 05/11] update --- .../terra/gold/gold_terra__address_labels.sql | 19 +++++ .../terra/gold/gold_terra__address_tags.sql | 21 +++++ .../terra/gold/gold_terra__daily_balances.sql | 59 ++++++++++++++ models/terra/gold/gold_terra__decimals.sql | 13 ++++ models/terra/gold/gold_terra__events.sql | 77 +++++++++++++++++++ models/terra/gold/terra_address_labels.sql | 0 models/terra/gold/terra_address_tags.sql | 0 models/terra/gold/terra_daily_balances.sql | 0 models/terra/gold/terra_decimals.sql | 0 models/terra/gold/terra_events.sql | 0 10 files changed, 189 insertions(+) create mode 100644 models/terra/gold/gold_terra__address_labels.sql create mode 100644 models/terra/gold/gold_terra__address_tags.sql create mode 100644 models/terra/gold/gold_terra__daily_balances.sql create mode 100644 models/terra/gold/gold_terra__decimals.sql create mode 100644 models/terra/gold/gold_terra__events.sql delete mode 100644 models/terra/gold/terra_address_labels.sql delete mode 100644 models/terra/gold/terra_address_tags.sql delete mode 100644 models/terra/gold/terra_daily_balances.sql delete mode 100644 models/terra/gold/terra_decimals.sql delete mode 100644 models/terra/gold/terra_events.sql diff --git a/models/terra/gold/gold_terra__address_labels.sql b/models/terra/gold/gold_terra__address_labels.sql new file mode 100644 index 00000000..0d1406e3 --- /dev/null +++ b/models/terra/gold/gold_terra__address_labels.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='incremental', + unique_key='blockchain || address', + incremental_strategy='delete+insert', + cluster_by=['block_id, 'block_timestamp'], + tags=['snowflake', 'terra_gold', 'terra_address_labels'] + ) +}} + +SELECT + * +FROM + {{ source( + 'shared', + 'udm_address_labels_new' + ) }} +WHERE + blockchain = 'terra' diff --git a/models/terra/gold/gold_terra__address_tags.sql b/models/terra/gold/gold_terra__address_tags.sql new file mode 100644 index 00000000..e6f7e87f --- /dev/null +++ b/models/terra/gold/gold_terra__address_tags.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'table', + sort = 'address' +) }} + +SELECT + blockchain, + address, + tag_name, + tag_type, + tag_slug, + source, + start_date, + end_date +FROM + {{ source( + 'shared', + 'udm_address_tags' + ) }} +WHERE + blockchain = 'terra' diff --git a/models/terra/gold/gold_terra__daily_balances.sql b/models/terra/gold/gold_terra__daily_balances.sql new file mode 100644 index 00000000..499d2079 --- /dev/null +++ b/models/terra/gold/gold_terra__daily_balances.sql @@ -0,0 +1,59 @@ +{{ config( + materialized = 'incremental', + sort = ['date', 'currency'], + unique_key = 'date', + tags = ['balances'] +) }} + +WITH prices AS ( + + SELECT + p.symbol, + DATE_TRUNC( + 'day', + HOUR + ) AS DAY, + AVG(price) AS price + FROM + {{ ref('terra_prices') }} + p + WHERE + +{% if is_incremental() %} +HOUR >= getdate() - INTERVAL '3 days' +{% else %} + HOUR >= getdate() - INTERVAL '12 months' +{% endif %} +GROUP BY + p.symbol, + DAY +) +SELECT + DATE, + b.address, + address_labels.l1_label AS address_label_type, + address_labels.l2_label AS address_label_subtype, + address_labels.project_name AS address_label, + address_labels.address_name AS address_address_name, + balance, + balance * p.price AS balance_usd, + b.balance_type, + currency +FROM + {{ source( + 'terra', + 'udm_daily_balances_terra' + ) }} + b + LEFT OUTER JOIN prices p + ON p.symbol = b.currency + AND p.day = b.date + LEFT OUTER JOIN {{ ref('terra_address_labels') }} AS address_labels + ON b.address = address_labels.address +WHERE + +{% if is_incremental() %} +DATE >= getdate() - INTERVAL '3 days' +{% else %} + DATE >= getdate() - INTERVAL '12 months' +{% endif %} diff --git a/models/terra/gold/gold_terra__decimals.sql b/models/terra/gold/gold_terra__decimals.sql new file mode 100644 index 00000000..0778629b --- /dev/null +++ b/models/terra/gold/gold_terra__decimals.sql @@ -0,0 +1,13 @@ +{{ config( + materialized = 'table' +) }} + +SELECT + * +FROM + {{ source( + 'shared', + 'udm_decimal_adjustments' + ) }} +WHERE + blockchain = 'terra' diff --git a/models/terra/gold/gold_terra__events.sql b/models/terra/gold/gold_terra__events.sql new file mode 100644 index 00000000..e977ffac --- /dev/null +++ b/models/terra/gold/gold_terra__events.sql @@ -0,0 +1,77 @@ +{{ config( + materialized = 'incremental', + sort = 'block_timestamp', + unique_key = 'block_timestamp', + tags = ['events'] +) }} + +SELECT + e.blockchain, + block_timestamp, + block_id AS block_number, + tx_id, + tx_from, + tx_from_labels.l1_label AS tx_from_label_type, + tx_from_labels.l2_label AS tx_from_label_subtype, + tx_from_labels.project_name AS tx_from_label, + tx_from_labels.address_name AS tx_from_address_name, + tx_to, + tx_to_labels.l1_label AS tx_to_label_type, + tx_to_labels.l2_label AS tx_to_label_subtype, + tx_to_labels.project_name AS tx_to_label, + tx_to_labels.address_name AS tx_to_address_name, + tx_type, + tx_fee, + tx_fee * fee_price.price AS tx_fee_usd, + event_from, + event_from_labels.l1_label AS event_from_label_type, + event_from_labels.l2_label AS event_from_label_subtype, + event_from_labels.project_name AS event_from_label, + event_from_labels.address_name AS event_from_address_name, + event_to, + event_to_labels.l1_label AS event_to_label_type, + event_to_labels.l2_label AS event_to_label_subtype, + event_to_labels.project_name AS event_to_label, + event_to_labels.address_name AS event_to_address_name, + event_type, + event_amount, + event_amount * event_price.price AS event_amount_usd, + event_currency +FROM + {{ source( + 'terra', + 'udm_events_terra' + ) }} + e + LEFT OUTER JOIN {{ ref('terra_address_labels') }} AS tx_from_labels + ON e.tx_from = tx_from_labels.address + LEFT OUTER JOIN {{ ref('terra_address_labels') }} AS tx_to_labels + ON e.tx_to = tx_to_labels.address + LEFT OUTER JOIN {{ ref('terra_address_labels') }} AS event_from_labels + ON e.event_from = event_from_labels.address + LEFT OUTER JOIN {{ ref('terra_address_labels') }} AS event_to_labels + ON e.event_to = event_to_labels.address + LEFT OUTER JOIN {{ ref('terra_prices') }} + fee_price + ON fee_price.hour = DATE_TRUNC( + 'hour', + block_timestamp + ) + AND fee_price.symbol = COALESCE( + e.event_currency, + 'LUNA' + ) + LEFT OUTER JOIN {{ ref('terra_prices') }} + event_price + ON event_price.hour = DATE_TRUNC( + 'hour', + block_timestamp + ) + AND event_price.symbol = e.event_currency +WHERE + +{% if is_incremental() %} +block_timestamp >= getdate() - INTERVAL '3 days' +{% else %} + block_timestamp >= getdate() - INTERVAL '12 months' +{% endif %} diff --git a/models/terra/gold/terra_address_labels.sql b/models/terra/gold/terra_address_labels.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/models/terra/gold/terra_address_tags.sql b/models/terra/gold/terra_address_tags.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/models/terra/gold/terra_daily_balances.sql b/models/terra/gold/terra_daily_balances.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/models/terra/gold/terra_decimals.sql b/models/terra/gold/terra_decimals.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/models/terra/gold/terra_events.sql b/models/terra/gold/terra_events.sql deleted file mode 100644 index e69de29b..00000000 From 643ef763558741cf173c59856b102369ebdc767c Mon Sep 17 00:00:00 2001 From: Ryan-Loofy <63126328+Ryan-Loofy@users.noreply.github.com> Date: Fri, 27 Aug 2021 17:03:23 -0400 Subject: [PATCH 06/11] gold naming for tables --- .../{terra_address_labels.sql => gold_terra__address_labels.sql} | 0 .../gold/{terra_address_tags.sql => gold_terra__address_tags.sql} | 0 .../{terra_daily_balances.sql => gold_terra__daily_balances.sql} | 0 .../terra/gold/{terra_decimals.sql => gold_terra__decimals.sql} | 0 models/terra/gold/{terra_events.sql => gold_terra__events.sql} | 0 models/terra/gold/{terra_prices.sql => gold_terra__prices.sql} | 0 .../terra/gold/{terra_tax_rate.sql => gold_terra__tax_rate.sql} | 0 .../terra/gold/{terra_velocity.sql => gold_terra__velocity.sql} | 0 .../gold/{terra_voting_power.sql => gold_terra__voting_power.sql} | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename models/terra/gold/{terra_address_labels.sql => gold_terra__address_labels.sql} (100%) rename models/terra/gold/{terra_address_tags.sql => gold_terra__address_tags.sql} (100%) rename models/terra/gold/{terra_daily_balances.sql => gold_terra__daily_balances.sql} (100%) rename models/terra/gold/{terra_decimals.sql => gold_terra__decimals.sql} (100%) rename models/terra/gold/{terra_events.sql => gold_terra__events.sql} (100%) rename models/terra/gold/{terra_prices.sql => gold_terra__prices.sql} (100%) rename models/terra/gold/{terra_tax_rate.sql => gold_terra__tax_rate.sql} (100%) rename models/terra/gold/{terra_velocity.sql => gold_terra__velocity.sql} (100%) rename models/terra/gold/{terra_voting_power.sql => gold_terra__voting_power.sql} (100%) diff --git a/models/terra/gold/terra_address_labels.sql b/models/terra/gold/gold_terra__address_labels.sql similarity index 100% rename from models/terra/gold/terra_address_labels.sql rename to models/terra/gold/gold_terra__address_labels.sql diff --git a/models/terra/gold/terra_address_tags.sql b/models/terra/gold/gold_terra__address_tags.sql similarity index 100% rename from models/terra/gold/terra_address_tags.sql rename to models/terra/gold/gold_terra__address_tags.sql diff --git a/models/terra/gold/terra_daily_balances.sql b/models/terra/gold/gold_terra__daily_balances.sql similarity index 100% rename from models/terra/gold/terra_daily_balances.sql rename to models/terra/gold/gold_terra__daily_balances.sql diff --git a/models/terra/gold/terra_decimals.sql b/models/terra/gold/gold_terra__decimals.sql similarity index 100% rename from models/terra/gold/terra_decimals.sql rename to models/terra/gold/gold_terra__decimals.sql diff --git a/models/terra/gold/terra_events.sql b/models/terra/gold/gold_terra__events.sql similarity index 100% rename from models/terra/gold/terra_events.sql rename to models/terra/gold/gold_terra__events.sql diff --git a/models/terra/gold/terra_prices.sql b/models/terra/gold/gold_terra__prices.sql similarity index 100% rename from models/terra/gold/terra_prices.sql rename to models/terra/gold/gold_terra__prices.sql diff --git a/models/terra/gold/terra_tax_rate.sql b/models/terra/gold/gold_terra__tax_rate.sql similarity index 100% rename from models/terra/gold/terra_tax_rate.sql rename to models/terra/gold/gold_terra__tax_rate.sql diff --git a/models/terra/gold/terra_velocity.sql b/models/terra/gold/gold_terra__velocity.sql similarity index 100% rename from models/terra/gold/terra_velocity.sql rename to models/terra/gold/gold_terra__velocity.sql diff --git a/models/terra/gold/terra_voting_power.sql b/models/terra/gold/gold_terra__voting_power.sql similarity index 100% rename from models/terra/gold/terra_voting_power.sql rename to models/terra/gold/gold_terra__voting_power.sql From 5d7b0840c7309165d34dce8d773b42fec6878701 Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 27 Aug 2021 17:03:53 -0400 Subject: [PATCH 07/11] update --- models/terra/gold/gold_terra__daily_balances.sql | 2 +- models/terra/gold/gold_terra__decimals.sql | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/models/terra/gold/gold_terra__daily_balances.sql b/models/terra/gold/gold_terra__daily_balances.sql index 499d2079..f0c743bd 100644 --- a/models/terra/gold/gold_terra__daily_balances.sql +++ b/models/terra/gold/gold_terra__daily_balances.sql @@ -2,7 +2,7 @@ materialized = 'incremental', sort = ['date', 'currency'], unique_key = 'date', - tags = ['balances'] + tags=['snowflake', 'terra_gold', 'terra_daily_balances'] ) }} WITH prices AS ( diff --git a/models/terra/gold/gold_terra__decimals.sql b/models/terra/gold/gold_terra__decimals.sql index 0778629b..76220925 100644 --- a/models/terra/gold/gold_terra__decimals.sql +++ b/models/terra/gold/gold_terra__decimals.sql @@ -1,5 +1,8 @@ {{ config( - materialized = 'table' + materialized = 'incremental', + sort = ['date', 'currency'], + unique_key = 'date', + tags=['snowflake', 'terra_gold', 'terra_daily_balances'] ) }} SELECT From 18476324372b82494a3aa53cf654af97e5e731b3 Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 27 Aug 2021 17:06:51 -0400 Subject: [PATCH 08/11] update some schemas and columns --- .../terra/gold/gold_terra__address_labels.sql | 28 ++++++++-------- .../terra/gold/gold_terra__address_tags.sql | 33 ++++++++++--------- .../terra/gold/gold_terra__daily_balances.sql | 6 +--- models/terra/gold/gold_terra__decimals.sql | 7 ++-- models/terra/gold/gold_terra__events.sql | 4 +-- 5 files changed, 37 insertions(+), 41 deletions(-) diff --git a/models/terra/gold/gold_terra__address_labels.sql b/models/terra/gold/gold_terra__address_labels.sql index 0d1406e3..46b10b12 100644 --- a/models/terra/gold/gold_terra__address_labels.sql +++ b/models/terra/gold/gold_terra__address_labels.sql @@ -1,19 +1,17 @@ -{{ - config( - materialized='incremental', - unique_key='blockchain || address', - incremental_strategy='delete+insert', - cluster_by=['block_id, 'block_timestamp'], - tags=['snowflake', 'terra_gold', 'terra_address_labels'] - ) -}} +{{ config( + materialized = 'incremental', + unique_key = 'blockchain || address', + incremental_strategy = 'delete+insert', + cluster_by = ['block_id, 'block_timestamp'], + tags = ['snowflake', 'terra_gold', 'terra_address_labels'] +) }} SELECT - * + * FROM - {{ source( - 'shared', - 'udm_address_labels_new' - ) }} + {{ source( + 'shared', + 'udm_address_labels_new' + ) }} WHERE - blockchain = 'terra' + blockchain = 'terra' diff --git a/models/terra/gold/gold_terra__address_tags.sql b/models/terra/gold/gold_terra__address_tags.sql index e6f7e87f..d836ce2c 100644 --- a/models/terra/gold/gold_terra__address_tags.sql +++ b/models/terra/gold/gold_terra__address_tags.sql @@ -1,21 +1,24 @@ {{ config( - materialized = 'table', - sort = 'address' + materialized = 'incremental', + unique_key = 'blockchain || address', + incremental_strategy = 'delete+insert', + cluster_by = ['block_id, 'block_timestamp'], + tags = ['snowflake', 'terra_gold', 'terra_address_labels'] ) }} SELECT - blockchain, - address, - tag_name, - tag_type, - tag_slug, - source, - start_date, - end_date + blockchain, + address, + tag_name, + tag_type, + tag_slug, + source, + start_date, + end_date FROM - {{ source( - 'shared', - 'udm_address_tags' - ) }} + {{ source( + 'shared', + 'udm_address_tags' + ) }} WHERE - blockchain = 'terra' + blockchain = 'terra' diff --git a/models/terra/gold/gold_terra__daily_balances.sql b/models/terra/gold/gold_terra__daily_balances.sql index f0c743bd..d767380b 100644 --- a/models/terra/gold/gold_terra__daily_balances.sql +++ b/models/terra/gold/gold_terra__daily_balances.sql @@ -2,7 +2,7 @@ materialized = 'incremental', sort = ['date', 'currency'], unique_key = 'date', - tags=['snowflake', 'terra_gold', 'terra_daily_balances'] + tags = ['snowflake', 'terra_gold', 'terra_daily_balances'] ) }} WITH prices AS ( @@ -21,8 +21,6 @@ WITH prices AS ( {% if is_incremental() %} HOUR >= getdate() - INTERVAL '3 days' -{% else %} - HOUR >= getdate() - INTERVAL '12 months' {% endif %} GROUP BY p.symbol, @@ -54,6 +52,4 @@ WHERE {% if is_incremental() %} DATE >= getdate() - INTERVAL '3 days' -{% else %} - DATE >= getdate() - INTERVAL '12 months' {% endif %} diff --git a/models/terra/gold/gold_terra__decimals.sql b/models/terra/gold/gold_terra__decimals.sql index 76220925..13199c9f 100644 --- a/models/terra/gold/gold_terra__decimals.sql +++ b/models/terra/gold/gold_terra__decimals.sql @@ -1,8 +1,9 @@ {{ config( materialized = 'incremental', - sort = ['date', 'currency'], - unique_key = 'date', - tags=['snowflake', 'terra_gold', 'terra_daily_balances'] + unique_key = 'blockchain || asset_id', + incremental_strategy = 'delete+insert', + cluster_by = ['blockchain, 'asset_id'], + tags = ['snowflake', 'terra_gold', 'terra_decimals'] ) }} SELECT diff --git a/models/terra/gold/gold_terra__events.sql b/models/terra/gold/gold_terra__events.sql index e977ffac..58d458ee 100644 --- a/models/terra/gold/gold_terra__events.sql +++ b/models/terra/gold/gold_terra__events.sql @@ -2,7 +2,7 @@ materialized = 'incremental', sort = 'block_timestamp', unique_key = 'block_timestamp', - tags = ['events'] + tags = ['snowflake', 'terra_gold', 'terra_events'] ) }} SELECT @@ -72,6 +72,4 @@ WHERE {% if is_incremental() %} block_timestamp >= getdate() - INTERVAL '3 days' -{% else %} - block_timestamp >= getdate() - INTERVAL '12 months' {% endif %} From 4a2be86fb86e3a7f20ce4496bb8f451419ca27f7 Mon Sep 17 00:00:00 2001 From: Ryan-Loofy <63126328+Ryan-Loofy@users.noreply.github.com> Date: Fri, 27 Aug 2021 17:12:46 -0400 Subject: [PATCH 09/11] cluster and tags added --- models/terra/gold/gold_terra__prices.sql | 16 ++++++---------- models/terra/gold/gold_terra__tax_rate.sql | 11 +++++------ models/terra/gold/gold_terra__velocity.sql | 8 ++++++++ models/terra/gold/gold_terra__voting_power.sql | 13 ++++++------- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/models/terra/gold/gold_terra__prices.sql b/models/terra/gold/gold_terra__prices.sql index 03e15306..60e1beca 100644 --- a/models/terra/gold/gold_terra__prices.sql +++ b/models/terra/gold/gold_terra__prices.sql @@ -1,8 +1,10 @@ {{ config( - materialized='incremental', - sort='hour', - unique_key='symbol || hour') -}} + materialized = 'incremental', + unique_key = 'symbol || hour', + incremental_strategy = 'delete+insert', + cluster_by = ['hour, 'symbol'], + tags = ['snowflake', 'terra_gold', 'terra_prices'] +) }} WITH hours AS ( select hour @@ -11,8 +13,6 @@ WITH hours AS ( where {% if is_incremental() %} hour >= getdate() - interval '3 days' and hour < getdate() + interval '1 day' - {% else %} - hour >= getdate() - interval '12 months' and hour < getdate() + interval '1 day' {% endif %} ), raw_prices AS ( @@ -23,8 +23,6 @@ raw_prices AS ( ) {% if is_incremental() %} AND recorded_at >= getdate() - interval '3 days' - {% else %} - AND recorded_at >= getdate() - interval '12 months' {% endif %} AND recorded_at < getdate() + interval '1 day' GROUP BY symbol, hour @@ -35,8 +33,6 @@ raw_prices AS ( FROM {{source('shared','coingecko_prices')}} WHERE asset_id IN ('terrausd') {% if is_incremental() %} AND recorded_at >= getdate() - interval '3 days' - {% else %} - AND recorded_at >= getdate() - interval '12 months' {% endif %} AND recorded_at < getdate() + interval '1 day' GROUP BY symbol, hour diff --git a/models/terra/gold/gold_terra__tax_rate.sql b/models/terra/gold/gold_terra__tax_rate.sql index 55bccb3f..e3072001 100644 --- a/models/terra/gold/gold_terra__tax_rate.sql +++ b/models/terra/gold/gold_terra__tax_rate.sql @@ -1,9 +1,10 @@ {{ config( - materialized='incremental', - sort='block_timestamp', + materialized = 'incremental', unique_key='blockchain || block_number', - tags=['custom']) -}} + incremental_strategy = 'delete+insert', + cluster_by = ['blockchain, 'block_number'], + tags = ['snowflake', 'terra_gold', 'terra_tax_rate'] +) }} SELECT blockchain, @@ -15,6 +16,4 @@ FROM WHERE {% if is_incremental() %} block_timestamp >= getdate() - interval '3 days' -{% else %} - block_timestamp >= getdate() - interval '12 months' {% endif %} \ No newline at end of file diff --git a/models/terra/gold/gold_terra__velocity.sql b/models/terra/gold/gold_terra__velocity.sql index 5dde1330..975998dc 100644 --- a/models/terra/gold/gold_terra__velocity.sql +++ b/models/terra/gold/gold_terra__velocity.sql @@ -1,3 +1,11 @@ +{{ config( + materialized = 'incremental', + unique_key='start_Date', + incremental_strategy = 'delete+insert', + cluster_by = ['start_date], + tags = ['snowflake', 'terra_gold', 'terra_velocity'] +) }} + {{ config( materialized='table', sort='start_date', diff --git a/models/terra/gold/gold_terra__voting_power.sql b/models/terra/gold/gold_terra__voting_power.sql index 47ba8b24..1e1b2225 100644 --- a/models/terra/gold/gold_terra__voting_power.sql +++ b/models/terra/gold/gold_terra__voting_power.sql @@ -1,9 +1,10 @@ {{ config( - materialized='incremental', - sort='block_timestamp', - unique_key='blockchain || block_number', - tags=['custom']) -}} + materialized = 'incremental', + unique_key = 'blockchain || address', + incremental_strategy = 'delete+insert', + cluster_by = ['block_id, 'block_timestamp'], + tags = ['snowflake', 'terra_gold', 'terra_voting_power'] +) }} SELECT blockchain, @@ -16,6 +17,4 @@ FROM WHERE {% if is_incremental() %} block_timestamp >= getdate() - interval '3 days' - {% else %} - block_timestamp >= getdate() - interval '12 months' {% endif %} \ No newline at end of file From 37bad9f99ddd4823e09b56ed2c517cf4e1a87f4a Mon Sep 17 00:00:00 2001 From: yulike Date: Fri, 27 Aug 2021 17:17:20 -0400 Subject: [PATCH 10/11] updated --- models/sources.yml | 1 + models/terra/gold/gold_terra__address_labels.sql | 2 +- models/terra/gold/gold_terra__address_tags.sql | 4 ++-- models/terra/gold/gold_terra__daily_balances.sql | 4 ++-- models/terra/gold/gold_terra__decimals.sql | 2 +- models/terra/gold/gold_terra__events.sql | 14 +++++++------- models/terra/gold/gold_terra__prices.sql | 2 +- models/terra/gold/gold_terra__tax_rate.sql | 2 +- models/terra/gold/gold_terra__velocity.sql | 2 +- models/terra/gold/gold_terra__voting_power.sql | 2 +- 10 files changed, 18 insertions(+), 17 deletions(-) diff --git a/models/sources.yml b/models/sources.yml index 36e5916a..52522330 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -12,6 +12,7 @@ sources: - name: hours - name: udm_address_labels - name: udm_address_labels_new + - name: udm_events_terra - name: udm_address_tags - name: udm_balances - name: udm_decimal_adjustments diff --git a/models/terra/gold/gold_terra__address_labels.sql b/models/terra/gold/gold_terra__address_labels.sql index 46b10b12..c1f8a5ea 100644 --- a/models/terra/gold/gold_terra__address_labels.sql +++ b/models/terra/gold/gold_terra__address_labels.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = 'blockchain || address', incremental_strategy = 'delete+insert', - cluster_by = ['block_id, 'block_timestamp'], + cluster_by = ['address'], tags = ['snowflake', 'terra_gold', 'terra_address_labels'] ) }} diff --git a/models/terra/gold/gold_terra__address_tags.sql b/models/terra/gold/gold_terra__address_tags.sql index d836ce2c..a71eb150 100644 --- a/models/terra/gold/gold_terra__address_tags.sql +++ b/models/terra/gold/gold_terra__address_tags.sql @@ -2,8 +2,8 @@ materialized = 'incremental', unique_key = 'blockchain || address', incremental_strategy = 'delete+insert', - cluster_by = ['block_id, 'block_timestamp'], - tags = ['snowflake', 'terra_gold', 'terra_address_labels'] + cluster_by = ['blockchain'], + tags = ['snowflake', 'terra_gold', 'terra_address_tags'] ) }} SELECT diff --git a/models/terra/gold/gold_terra__daily_balances.sql b/models/terra/gold/gold_terra__daily_balances.sql index d767380b..1252cba6 100644 --- a/models/terra/gold/gold_terra__daily_balances.sql +++ b/models/terra/gold/gold_terra__daily_balances.sql @@ -15,7 +15,7 @@ WITH prices AS ( ) AS DAY, AVG(price) AS price FROM - {{ ref('terra_prices') }} + {{ ref('gold_terra__prices') }} p WHERE @@ -46,7 +46,7 @@ FROM LEFT OUTER JOIN prices p ON p.symbol = b.currency AND p.day = b.date - LEFT OUTER JOIN {{ ref('terra_address_labels') }} AS address_labels + LEFT OUTER JOIN {{ ref('gold_terra__address_labels') }} AS address_labels ON b.address = address_labels.address WHERE diff --git a/models/terra/gold/gold_terra__decimals.sql b/models/terra/gold/gold_terra__decimals.sql index 13199c9f..ff3842e6 100644 --- a/models/terra/gold/gold_terra__decimals.sql +++ b/models/terra/gold/gold_terra__decimals.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = 'blockchain || asset_id', incremental_strategy = 'delete+insert', - cluster_by = ['blockchain, 'asset_id'], + cluster_by = ['blockchain'], tags = ['snowflake', 'terra_gold', 'terra_decimals'] ) }} diff --git a/models/terra/gold/gold_terra__events.sql b/models/terra/gold/gold_terra__events.sql index 58d458ee..75529488 100644 --- a/models/terra/gold/gold_terra__events.sql +++ b/models/terra/gold/gold_terra__events.sql @@ -39,19 +39,19 @@ SELECT event_currency FROM {{ source( - 'terra', + 'shared', 'udm_events_terra' ) }} e - LEFT OUTER JOIN {{ ref('terra_address_labels') }} AS tx_from_labels + LEFT OUTER JOIN {{ ref('gold_terra__address_labels') }} AS tx_from_labels ON e.tx_from = tx_from_labels.address - LEFT OUTER JOIN {{ ref('terra_address_labels') }} AS tx_to_labels + LEFT OUTER JOIN {{ ref('gold_terra__address_labels') }} AS tx_to_labels ON e.tx_to = tx_to_labels.address - LEFT OUTER JOIN {{ ref('terra_address_labels') }} AS event_from_labels + LEFT OUTER JOIN {{ ref('gold_terra__address_labels') }} AS event_from_labels ON e.event_from = event_from_labels.address - LEFT OUTER JOIN {{ ref('terra_address_labels') }} AS event_to_labels + LEFT OUTER JOIN {{ ref('gold_terra__address_labels') }} AS event_to_labels ON e.event_to = event_to_labels.address - LEFT OUTER JOIN {{ ref('terra_prices') }} + LEFT OUTER JOIN {{ ref('gold_terra__prices') }} fee_price ON fee_price.hour = DATE_TRUNC( 'hour', @@ -61,7 +61,7 @@ FROM e.event_currency, 'LUNA' ) - LEFT OUTER JOIN {{ ref('terra_prices') }} + LEFT OUTER JOIN {{ ref('gold_terra__prices') }} event_price ON event_price.hour = DATE_TRUNC( 'hour', diff --git a/models/terra/gold/gold_terra__prices.sql b/models/terra/gold/gold_terra__prices.sql index 60e1beca..6b44533b 100644 --- a/models/terra/gold/gold_terra__prices.sql +++ b/models/terra/gold/gold_terra__prices.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = 'symbol || hour', incremental_strategy = 'delete+insert', - cluster_by = ['hour, 'symbol'], + cluster_by = ['hour'], tags = ['snowflake', 'terra_gold', 'terra_prices'] ) }} diff --git a/models/terra/gold/gold_terra__tax_rate.sql b/models/terra/gold/gold_terra__tax_rate.sql index e3072001..62be3ff6 100644 --- a/models/terra/gold/gold_terra__tax_rate.sql +++ b/models/terra/gold/gold_terra__tax_rate.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key='blockchain || block_number', incremental_strategy = 'delete+insert', - cluster_by = ['blockchain, 'block_number'], + cluster_by = ['blockchain', 'block_number'], tags = ['snowflake', 'terra_gold', 'terra_tax_rate'] ) }} diff --git a/models/terra/gold/gold_terra__velocity.sql b/models/terra/gold/gold_terra__velocity.sql index 975998dc..63002896 100644 --- a/models/terra/gold/gold_terra__velocity.sql +++ b/models/terra/gold/gold_terra__velocity.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key='start_Date', incremental_strategy = 'delete+insert', - cluster_by = ['start_date], + cluster_by = ['start_date'], tags = ['snowflake', 'terra_gold', 'terra_velocity'] ) }} diff --git a/models/terra/gold/gold_terra__voting_power.sql b/models/terra/gold/gold_terra__voting_power.sql index 1e1b2225..8060acf1 100644 --- a/models/terra/gold/gold_terra__voting_power.sql +++ b/models/terra/gold/gold_terra__voting_power.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = 'blockchain || address', incremental_strategy = 'delete+insert', - cluster_by = ['block_id, 'block_timestamp'], + cluster_by = ['block_id', 'block_timestamp'], tags = ['snowflake', 'terra_gold', 'terra_voting_power'] ) }} From 7e2fcd0f311d0610fc0516a93f8a41a8b178551d Mon Sep 17 00:00:00 2001 From: yulike Date: Mon, 30 Aug 2021 09:03:26 -0400 Subject: [PATCH 11/11] updated the terra tables --- .../terra/gold/gold_terra__daily_balances.sql | 9 +- models/terra/gold/gold_terra__events.sql | 3 +- models/terra/gold/gold_terra__prices.sql | 124 ++++++++++++------ models/terra/gold/gold_terra__tax_rate.sql | 13 +- models/terra/gold/gold_terra__velocity.sql | 31 +++-- .../terra/gold/gold_terra__voting_power.sql | 15 ++- 6 files changed, 126 insertions(+), 69 deletions(-) diff --git a/models/terra/gold/gold_terra__daily_balances.sql b/models/terra/gold/gold_terra__daily_balances.sql index 1252cba6..4313b5f9 100644 --- a/models/terra/gold/gold_terra__daily_balances.sql +++ b/models/terra/gold/gold_terra__daily_balances.sql @@ -18,9 +18,10 @@ WITH prices AS ( {{ ref('gold_terra__prices') }} p WHERE + TRUE {% if is_incremental() %} -HOUR >= getdate() - INTERVAL '3 days' +AND HOUR >= getdate() - INTERVAL '3 days' {% endif %} GROUP BY p.symbol, @@ -46,10 +47,12 @@ FROM LEFT OUTER JOIN prices p ON p.symbol = b.currency AND p.day = b.date - LEFT OUTER JOIN {{ ref('gold_terra__address_labels') }} AS address_labels + LEFT OUTER JOIN {{ ref('gold_terra__address_labels') }} + address_labels ON b.address = address_labels.address WHERE + TRUE {% if is_incremental() %} -DATE >= getdate() - INTERVAL '3 days' +AND DATE >= getdate() - INTERVAL '3 days' {% endif %} diff --git a/models/terra/gold/gold_terra__events.sql b/models/terra/gold/gold_terra__events.sql index 75529488..bff2a197 100644 --- a/models/terra/gold/gold_terra__events.sql +++ b/models/terra/gold/gold_terra__events.sql @@ -69,7 +69,8 @@ FROM ) AND event_price.symbol = e.event_currency WHERE + TRUE {% if is_incremental() %} -block_timestamp >= getdate() - INTERVAL '3 days' +AND block_timestamp >= getdate() - INTERVAL '3 days' {% endif %} diff --git a/models/terra/gold/gold_terra__prices.sql b/models/terra/gold/gold_terra__prices.sql index 6b44533b..a7d63cae 100644 --- a/models/terra/gold/gold_terra__prices.sql +++ b/models/terra/gold/gold_terra__prices.sql @@ -7,58 +7,98 @@ ) }} WITH hours AS ( - select hour - from - {{source('shared', 'hours')}} - where - {% if is_incremental() %} - hour >= getdate() - interval '3 days' and hour < getdate() + interval '1 day' - {% endif %} + + SELECT + HOUR + FROM + {{ source( + 'shared', + 'hours' + ) }} + WHERE + TRUE + +{% if is_incremental() %} +AND HOUR >= getdate() - INTERVAL '3 days' +AND HOUR < getdate() + INTERVAL '1 day' +{% endif %} ), raw_prices AS ( - SELECT symbol, date_trunc('hour', recorded_at) as hour, avg(price) as price FROM {{source('shared','prices')}} WHERE asset_id IN ( - 4172, -- Luna - 5115, -- Terra KRW (KRT) - 6370 -- Terra SDT (SDR) - ) - {% if is_incremental() %} - AND recorded_at >= getdate() - interval '3 days' - {% endif %} - AND recorded_at < getdate() + interval '1 day' - GROUP BY symbol, hour + SELECT + symbol, + DATE_TRUNC( + 'hour', + recorded_at + ) AS HOUR, + AVG(price) AS price + FROM + {{ source( + 'shared', + 'prices' + ) }} + WHERE + asset_id IN ( + 4172, + -- Luna + 5115, + -- Terra KRW (KRT) + 6370 -- Terra SDT (SDR) + ) - UNION ALL +{% if is_incremental() %} +AND recorded_at >= getdate() - INTERVAL '3 days' +{% endif %} +AND recorded_at < getdate() + INTERVAL '1 day' +GROUP BY + symbol, + HOUR +UNION ALL +SELECT + symbol, + DATE_TRUNC( + 'hour', + recorded_at + ) AS HOUR, + AVG(price) AS price +FROM + {{ source( + 'shared', + 'coingecko_prices' + ) }} +WHERE + asset_id IN ('terrausd') - SELECT symbol, date_trunc('hour', recorded_at) as hour, avg(price) as price - FROM {{source('shared','coingecko_prices')}} WHERE asset_id IN ('terrausd') - {% if is_incremental() %} - AND recorded_at >= getdate() - interval '3 days' - {% endif %} - AND recorded_at < getdate() + interval '1 day' - GROUP BY symbol, hour +{% if is_incremental() %} +AND recorded_at >= getdate() - INTERVAL '3 days' +{% endif %} +AND recorded_at < getdate() + INTERVAL '1 day' +GROUP BY + symbol, + HOUR ), assets AS ( - SELECT symbol FROM raw_prices GROUP BY 1 + SELECT + symbol + FROM + raw_prices + GROUP BY + 1 ), hour_assets AS ( SELECT - h.hour, - a.symbol + h.hour, + A.symbol FROM - hours h - CROSS JOIN - assets a + hours h + CROSS JOIN assets A ) SELECT -ha.hour, -ha.symbol, --- use price if we have other, otherwise fall back to daily average -coalesce(rp.price, avg(rp.price) over(partition by date_trunc('day', ha.hour), ha.symbol)) as price + ha.hour, + ha.symbol, + -- use price if we have other, otherwise fall back to daily average + COALESCE(rp.price, AVG(rp.price) over(PARTITION BY DATE_TRUNC('day', ha.hour), ha.symbol)) AS price FROM -hour_assets ha -LEFT OUTER JOIN -raw_prices rp -ON -ha.symbol = rp.symbol -AND -ha.hour = rp.hour \ No newline at end of file + hour_assets ha + LEFT OUTER JOIN raw_prices rp + ON ha.symbol = rp.symbol + AND ha.hour = rp.hour diff --git a/models/terra/gold/gold_terra__tax_rate.sql b/models/terra/gold/gold_terra__tax_rate.sql index 62be3ff6..691507b1 100644 --- a/models/terra/gold/gold_terra__tax_rate.sql +++ b/models/terra/gold/gold_terra__tax_rate.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key='blockchain || block_number', + unique_key = 'blockchain || block_number', incremental_strategy = 'delete+insert', cluster_by = ['blockchain', 'block_number'], tags = ['snowflake', 'terra_gold', 'terra_tax_rate'] @@ -12,8 +12,13 @@ SELECT block_number, tax_rate FROM - {{source('terra','udm_custom_fields_terra_tax_rate')}} + {{ source( + 'terra', + 'udm_custom_fields_terra_tax_rate' + ) }} WHERE + TRUE + {% if is_incremental() %} - block_timestamp >= getdate() - interval '3 days' -{% endif %} \ No newline at end of file +AND block_timestamp >= getdate() - INTERVAL '3 days' +{% endif %} diff --git a/models/terra/gold/gold_terra__velocity.sql b/models/terra/gold/gold_terra__velocity.sql index 63002896..6ee6451a 100644 --- a/models/terra/gold/gold_terra__velocity.sql +++ b/models/terra/gold/gold_terra__velocity.sql @@ -1,24 +1,27 @@ {{ config( materialized = 'incremental', - unique_key='start_Date', + unique_key = 'start_Date', incremental_strategy = 'delete+insert', cluster_by = ['start_date'], tags = ['snowflake', 'terra_gold', 'terra_velocity'] ) }} - {{ config( - materialized='table', - sort='start_date', - tags=['custom']) -}} + materialized = 'table', + sort = 'start_date', + tags = ['custom'] +) }} + SELECT - blockchain, - start_date, - end_date, - currency, - metric, - value + blockchain, + start_date, + end_date, + currency, + metric, + VALUE FROM - {{ source('shared', 'udm_velocity')}} + {{ source( + 'shared', + 'udm_velocity' + ) }} WHERE - blockchain = 'terra' \ No newline at end of file + blockchain = 'terra' diff --git a/models/terra/gold/gold_terra__voting_power.sql b/models/terra/gold/gold_terra__voting_power.sql index 8060acf1..d3bbc7eb 100644 --- a/models/terra/gold/gold_terra__voting_power.sql +++ b/models/terra/gold/gold_terra__voting_power.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = 'blockchain || address', incremental_strategy = 'delete+insert', - cluster_by = ['block_id', 'block_timestamp'], + cluster_by = ['block_number', 'block_timestamp'], tags = ['snowflake', 'terra_gold', 'terra_voting_power'] ) }} @@ -13,8 +13,13 @@ SELECT address, voting_power FROM - {{source('terra','terra_validator_voting_power')}} + {{ source( + 'terra', + 'terra_validator_voting_power' + ) }} WHERE - {% if is_incremental() %} - block_timestamp >= getdate() - interval '3 days' - {% endif %} \ No newline at end of file + TRUE + +{% if is_incremental() %} +AND block_timestamp >= getdate() - INTERVAL '3 days' +{% endif %}