From e69d0bbf484754649f0bb5d44d58e20cfd10244e Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Fri, 15 Dec 2023 14:47:41 -0800
Subject: [PATCH 01/48] union data stuff for zendesk

---
 macros/union_data.sql      |  4 +--
 macros/union_relations.sql | 54 ++++++++++++++++++++++++++++++++++----
 2 files changed, 51 insertions(+), 7 deletions(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index a67fadb..e942eed 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -51,7 +51,7 @@
     {%- endfor -%}
     
     {%- if relations != [] -%}
-        {{ dbt_utils.union_relations(relations) }}
+        {{ fivetran_utils.union_relations(relations) }}
     {%- else -%}
     {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%}
     {{ exceptions.warn("\n\nPlease be aware: The " ~ table_identifier|upper ~ " table was not found in your " ~ default_schema|upper ~ " schema(s). The Fivetran dbt package will create a completely empty " ~ table_identifier|upper ~ " staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }}
@@ -81,7 +81,7 @@
     {%- endfor -%}
 
     {%- if relations != [] -%}
-        {{ dbt_utils.union_relations(relations) }}
+        {{ fivetran_utils.union_relations(relations) }}
     {%- else -%}
     {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%}
     {{ exceptions.warn("\n\nPlease be aware: The " ~ table_identifier|upper ~ " table was not found in your " ~ default_schema|upper ~ " schema(s). The Fivetran dbt package will create a completely empty " ~ table_identifier|upper ~ " staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }}
diff --git a/macros/union_relations.sql b/macros/union_relations.sql
index 56122e6..0ada820 100644
--- a/macros/union_relations.sql
+++ b/macros/union_relations.sql
@@ -1,4 +1,8 @@
-{%- macro union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name=none) -%}
+{%- macro union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%}
+    {{ return(adapter.dispatch('union_relations', 'fivetran_utils')(relations, aliases, column_override, include, exclude, source_column_name, where)) }}
+{% endmacro %}
+
+{%- macro default__union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%}
 
     {%- if exclude and include -%}
         {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }}
@@ -10,24 +14,38 @@
     {% endif -%}
 
     {%- set column_override = column_override if column_override is not none else {} -%}
-    {%- set source_column_name = source_column_name if source_column_name is not none else '_dbt_source_relation' -%}
 
     {%- set relation_columns = {} -%}
     {%- set column_superset = {} -%}
+    {%- set all_excludes = [] -%}
+    {%- set all_includes = [] -%}
+
+    {%- if exclude -%}
+        {%- for exc in exclude -%}
+            {%- do all_excludes.append(exc | lower) -%}
+        {%- endfor -%}
+    {%- endif -%}
+
+    {%- if include -%}
+        {%- for inc in include -%}
+            {%- do all_includes.append(inc | lower) -%}
+        {%- endfor -%}
+    {%- endif -%}
 
     {%- for relation in relations -%}
 
         {%- do relation_columns.update({relation: []}) -%}
 
         {%- do dbt_utils._is_relation(relation, 'union_relations') -%}
+        {%- do dbt_utils._is_ephemeral(relation, 'union_relations') -%}
         {%- set cols = adapter.get_columns_in_relation(relation) -%}
         {%- for col in cols -%}
 
         {#- If an exclude list was provided and the column is in the list, do nothing -#}
-        {%- if exclude and col.column in exclude -%}
+        {%- if exclude and col.column | lower in all_excludes -%}
 
         {#- If an include list was provided and the column is not in the list, do nothing -#}
-        {%- elif include and col.column not in include -%}
+        {%- elif include and col.column | lower not in all_includes -%}
 
         {#- Otherwise add the column to the column superset -#}
         {%- else -%}
@@ -56,13 +74,35 @@
     {%- endfor -%}
 
     {%- set ordered_column_names = column_superset.keys() -%}
+    {%- set dbt_command = flags.WHICH -%}
+
+
+    {% if dbt_command in ['run', 'build'] %}
+    {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %}
+        {%- set relations_string -%}
+            {%- for relation in relations -%}
+                {{ relation.name }}
+            {%- if not loop.last %}, {% endif -%}
+            {%- endfor -%}
+        {%- endset -%}
+
+        {%- set error_message -%}
+            There were no columns found to union for relations {{ relations_string }}
+        {%- endset -%}
+
+        {{ exceptions.raise_compiler_error(error_message) }}
+    {%- endif -%}
+    {%- endif -%}
 
     {%- for relation in relations %}
 
         (
             select
 
+                {%- if source_column_name is not none %}
                 cast({{ dbt.string_literal(relation) }} as {{ dbt.type_string() }}) as {{ source_column_name }},
+                {%- endif %}
+
                 {% for col_name in ordered_column_names -%}
 
                     {%- set col = column_superset[col_name] %}
@@ -72,7 +112,11 @@
 
                 {%- endfor %}
 
-            from {{ aliases[loop.index0] if aliases else relation }}
+            from {{ aliases[loop.index0] if aliases else relation }} as unioned_relation_{{ loop.index }}
+
+            {% if where -%}
+            where {{ where }}
+            {%- endif %}
         )
 
         {% if not loop.last -%}

From 48bd242d99133ae26c5500b9082928747456beea Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 19 Dec 2023 10:11:33 -0800
Subject: [PATCH 02/48] change macro name

---
 macros/fivetran_union_relations.sql | 128 ++++++++++++++++++++++++++++
 macros/union_relations.sql          |  54 ++----------
 2 files changed, 134 insertions(+), 48 deletions(-)
 create mode 100644 macros/fivetran_union_relations.sql

diff --git a/macros/fivetran_union_relations.sql b/macros/fivetran_union_relations.sql
new file mode 100644
index 0000000..75c77c1
--- /dev/null
+++ b/macros/fivetran_union_relations.sql
@@ -0,0 +1,128 @@
+{%- macro fivetran_union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%}
+    {{ return(adapter.dispatch('fivetran_union_relations', 'fivetran_utils')(relations, aliases, column_override, include, exclude, source_column_name, where)) }}
+{% endmacro %}
+
+{%- macro default__fivetran_union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%}
+
+    {%- if exclude and include -%}
+        {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }}
+    {%- endif -%}
+
+    {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#}
+    {%- if not execute %}
+        {{ return('') }}
+    {% endif -%}
+
+    {%- set column_override = column_override if column_override is not none else {} -%}
+
+    {%- set relation_columns = {} -%}
+    {%- set column_superset = {} -%}
+    {%- set all_excludes = [] -%}
+    {%- set all_includes = [] -%}
+
+    {%- if exclude -%}
+        {%- for exc in exclude -%}
+            {%- do all_excludes.append(exc | lower) -%}
+        {%- endfor -%}
+    {%- endif -%}
+
+    {%- if include -%}
+        {%- for inc in include -%}
+            {%- do all_includes.append(inc | lower) -%}
+        {%- endfor -%}
+    {%- endif -%}
+
+    {%- for relation in relations -%}
+
+        {%- do relation_columns.update({relation: []}) -%}
+
+        {%- do dbt_utils._is_relation(relation, 'fivetran_union_relations') -%}
+        {%- do dbt_utils._is_ephemeral(relation, 'fivetran_union_relations') -%}
+        {%- set cols = adapter.get_columns_in_relation(relation) -%}
+        {%- for col in cols -%}
+
+        {#- If an exclude list was provided and the column is in the list, do nothing -#}
+        {%- if exclude and col.column | lower in all_excludes -%}
+
+        {#- If an include list was provided and the column is not in the list, do nothing -#}
+        {%- elif include and col.column | lower not in all_includes -%}
+
+        {#- Otherwise add the column to the column superset -#}
+        {%- else -%}
+
+            {#- update the list of columns in this relation -#}
+            {%- do relation_columns[relation].append(col.column) -%}
+
+            {%- if col.column in column_superset -%}
+
+                {%- set stored = column_superset[col.column] -%}
+                {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%}
+
+                    {%- do column_superset.update({col.column: col}) -%}
+
+                {%- endif %}
+
+            {%- else -%}
+
+                {%- do column_superset.update({col.column: col}) -%}
+
+            {%- endif -%}
+
+        {%- endif -%}
+
+        {%- endfor -%}
+    {%- endfor -%}
+
+    {%- set ordered_column_names = column_superset.keys() -%}
+    {%- set dbt_command = flags.WHICH -%}
+
+
+    {% if dbt_command in ['run', 'build'] %}
+    {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %}
+        {%- set relations_string -%}
+            {%- for relation in relations -%}
+                {{ relation.name }}
+            {%- if not loop.last %}, {% endif -%}
+            {%- endfor -%}
+        {%- endset -%}
+
+        {%- set error_message -%}
+            There were no columns found to union for relations {{ relations_string }}
+        {%- endset -%}
+
+        {{ exceptions.raise_compiler_error(error_message) }}
+    {%- endif -%}
+    {%- endif -%}
+
+    {%- for relation in relations %}
+
+        (
+            select
+
+                {%- if source_column_name is not none %}
+                cast({{ dbt.string_literal(relation) }} as {{ dbt.type_string() }}) as {{ source_column_name }},
+                {%- endif %}
+
+                {% for col_name in ordered_column_names -%}
+
+                    {%- set col = column_superset[col_name] %}
+                    {%- set col_type = column_override.get(col.column, col.data_type) %}
+                    {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %}
+                    cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%}
+
+                {%- endfor %}
+
+            from {{ aliases[loop.index0] if aliases else relation }} as unioned_relation_{{ loop.index }}
+
+            {% if where -%}
+            where {{ where }}
+            {%- endif %}
+        )
+
+        {% if not loop.last -%}
+            union all
+        {% endif -%}
+
+    {%- endfor -%}
+
+{%- endmacro -%}
\ No newline at end of file
diff --git a/macros/union_relations.sql b/macros/union_relations.sql
index 0ada820..7076c61 100644
--- a/macros/union_relations.sql
+++ b/macros/union_relations.sql
@@ -1,8 +1,6 @@
-{%- macro union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%}
-    {{ return(adapter.dispatch('union_relations', 'fivetran_utils')(relations, aliases, column_override, include, exclude, source_column_name, where)) }}
-{% endmacro %}
+-- to be deprecated in favor of fivetran_union_relations
 
-{%- macro default__union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%}
+{%- macro union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name=none) -%}
 
     {%- if exclude and include -%}
         {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }}
@@ -14,38 +12,24 @@
     {% endif -%}
 
     {%- set column_override = column_override if column_override is not none else {} -%}
+    {%- set source_column_name = source_column_name if source_column_name is not none else '_dbt_source_relation' -%}
 
     {%- set relation_columns = {} -%}
     {%- set column_superset = {} -%}
-    {%- set all_excludes = [] -%}
-    {%- set all_includes = [] -%}
-
-    {%- if exclude -%}
-        {%- for exc in exclude -%}
-            {%- do all_excludes.append(exc | lower) -%}
-        {%- endfor -%}
-    {%- endif -%}
-
-    {%- if include -%}
-        {%- for inc in include -%}
-            {%- do all_includes.append(inc | lower) -%}
-        {%- endfor -%}
-    {%- endif -%}
 
     {%- for relation in relations -%}
 
         {%- do relation_columns.update({relation: []}) -%}
 
         {%- do dbt_utils._is_relation(relation, 'union_relations') -%}
-        {%- do dbt_utils._is_ephemeral(relation, 'union_relations') -%}
         {%- set cols = adapter.get_columns_in_relation(relation) -%}
         {%- for col in cols -%}
 
         {#- If an exclude list was provided and the column is in the list, do nothing -#}
-        {%- if exclude and col.column | lower in all_excludes -%}
+        {%- if exclude and col.column in exclude -%}
 
         {#- If an include list was provided and the column is not in the list, do nothing -#}
-        {%- elif include and col.column | lower not in all_includes -%}
+        {%- elif include and col.column not in include -%}
 
         {#- Otherwise add the column to the column superset -#}
         {%- else -%}
@@ -74,35 +58,13 @@
     {%- endfor -%}
 
     {%- set ordered_column_names = column_superset.keys() -%}
-    {%- set dbt_command = flags.WHICH -%}
-
-
-    {% if dbt_command in ['run', 'build'] %}
-    {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %}
-        {%- set relations_string -%}
-            {%- for relation in relations -%}
-                {{ relation.name }}
-            {%- if not loop.last %}, {% endif -%}
-            {%- endfor -%}
-        {%- endset -%}
-
-        {%- set error_message -%}
-            There were no columns found to union for relations {{ relations_string }}
-        {%- endset -%}
-
-        {{ exceptions.raise_compiler_error(error_message) }}
-    {%- endif -%}
-    {%- endif -%}
 
     {%- for relation in relations %}
 
         (
             select
 
-                {%- if source_column_name is not none %}
                 cast({{ dbt.string_literal(relation) }} as {{ dbt.type_string() }}) as {{ source_column_name }},
-                {%- endif %}
-
                 {% for col_name in ordered_column_names -%}
 
                     {%- set col = column_superset[col_name] %}
@@ -112,11 +74,7 @@
 
                 {%- endfor %}
 
-            from {{ aliases[loop.index0] if aliases else relation }} as unioned_relation_{{ loop.index }}
-
-            {% if where -%}
-            where {{ where }}
-            {%- endif %}
+            from {{ aliases[loop.index0] if aliases else relation }}
         )
 
         {% if not loop.last -%}

From 26727c4ec1d8d480418669d894a9ca8d0bb1f4e8 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 19 Dec 2023 10:26:32 -0800
Subject: [PATCH 03/48] refer to new macro in union_data

---
 macros/union_data.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index e942eed..09476ea 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -51,7 +51,7 @@
     {%- endfor -%}
     
     {%- if relations != [] -%}
-        {{ fivetran_utils.union_relations(relations) }}
+        {{ fivetran_utils.fivetran_union_relations(relations) }}
     {%- else -%}
     {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%}
     {{ exceptions.warn("\n\nPlease be aware: The " ~ table_identifier|upper ~ " table was not found in your " ~ default_schema|upper ~ " schema(s). The Fivetran dbt package will create a completely empty " ~ table_identifier|upper ~ " staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }}
@@ -81,7 +81,7 @@
     {%- endfor -%}
 
     {%- if relations != [] -%}
-        {{ fivetran_utils.union_relations(relations) }}
+        {{ fivetran_utils.fivetran_union_relations(relations) }}
     {%- else -%}
     {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%}
     {{ exceptions.warn("\n\nPlease be aware: The " ~ table_identifier|upper ~ " table was not found in your " ~ default_schema|upper ~ " schema(s). The Fivetran dbt package will create a completely empty " ~ table_identifier|upper ~ " staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }}

From 77d9a209c72fa2510734489042b8d2c4dadc9449 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 26 Dec 2023 15:55:09 -0800
Subject: [PATCH 04/48] docs

---
 CHANGELOG.md                      |  4 ++++
 README.md                         | 30 +++++++++++++++++++++++++++++-
 dbt_project.yml                   |  2 +-
 integration_tests/dbt_project.yml |  2 +-
 4 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d42bd17..aab849b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+# dbt_fivetran_utils v0.4.11
+
+to be filled in...
+
 # dbt_fivetran_utils v0.4.10
 
 ## Bug Fix
diff --git a/README.md b/README.md
index 4588ad3..ff5a21d 100644
--- a/README.md
+++ b/README.md
@@ -84,6 +84,7 @@ dispatch:
     - [source\_relation (source)](#source_relation-source)
     - [union\_data (source)](#union_data-source)
       - [Union Data Defined Sources Configuration](#union-data-defined-sources-configuration)
+    - [fivetran\_union\_relations (source)](#fivetran_union_relations-source)
     - [union\_relations (source)](#union_relations-source)
   - [Variable Checks](#variable-checks)
     - [empty\_variable\_warning (source)](#empty_variable_warning-source)
@@ -577,8 +578,35 @@ sources:
       - name: customer 
       ...
 ```
+----
+### fivetran_union_relations ([source](macros/union_relations.sql))
+Heavily adapted from [dbt_utils.union_relations()](https://github.com/dbt-labs/dbt-utils?tab=readme-ov-file#union_relations-source).
+
+This macro combines via a `union all` of an array of [Relations](https://docs.getdbt.com/reference/dbt-classes#relation), even when columns have differing orders in each Relation, and/or some columns are missing from some relations. Any columns exclusive to a subset of these relations will be filled with `null` where not present. A new column (`_dbt_source_relation`) is also added to indicate the source for each record.
+
+**Usage:**
+```sql
+{{ fivetran_utils.fivetran_union_relations(
+    relations=[ref('my_model'), source('my_source', 'my_table')],
+    exclude=["_loaded_at"],
+    aliases=['my_model_cte', 'my_source_table_cte']
+) }}
+```
+**Args:**
+* `relations`          (required): An array of [Relations](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation).
+* `aliases`            (optional): An override of the relation identifier. This argument should be populated with the overwritten alias for each relation (ie if we are selecting from a CTE). If not populated `relations` will be the default. This argument is not included in the `dbt_utils` version of this macro.
+* `exclude`            (optional): A list of column names that should be excluded from the final query.
+* `include`            (optional): A list of column names that should be included in the final query. Note the `include` and `exclude` parameters are mutually exclusive.
+* `column_override`    (optional): A dictionary of explicit column type overrides, e.g. `{"some_field": "varchar(100)"}`.``
+* `source_column_name` (optional, `default="_dbt_source_relation"`): The name of the column that records the source of this row. Pass `None` to omit this column from the results.
+* `where` (optional): Filter conditions to include in the `where` clause.
+
 ----
 ### union_relations ([source](macros/union_relations.sql))
+> TO BE DEPRECATED IN FAVOR OF `fivetran_union_relations`.
+>
+> Currenlty only used in Marketo transform package.
+
 This macro unions together an array of [Relations](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation),
 even when columns have differing orders in each Relation, and/or some columns are
 missing from some relations. Any columns exclusive to a subset of these
@@ -587,7 +615,7 @@ relations will be filled with `null` where not present. An new column
 
 **Usage:**
 ```sql
-{{ dbt_utils.union_relations(
+{{ fivetran_utils.union_relations(
     relations=[ref('my_model'), source('my_source', 'my_table')],
     exclude=["_loaded_at"]
 ) }}
diff --git a/dbt_project.yml b/dbt_project.yml
index d366876..d024d9c 100644
--- a/dbt_project.yml
+++ b/dbt_project.yml
@@ -1,4 +1,4 @@
 name: 'fivetran_utils'
-version: '0.4.10'
+version: '0.4.11'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
index 36507f1..70461e8 100644
--- a/integration_tests/dbt_project.yml
+++ b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'fivetran_utils_integration_tests'
-version: '0.4.10'
+version: '0.4.11'
 config-version: 2
 profile: 'integration_tests'
 

From bc23cb5e1b12da46f0f6f96c016b7bf05f373aca Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 26 Dec 2023 16:07:16 -0800
Subject: [PATCH 05/48] changelog

---
 CHANGELOG.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index aab849b..8175b1d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,12 @@
 # dbt_fivetran_utils v0.4.11
 
-to be filled in...
+## Feature Updates
+[PR #139](https://github.com/fivetran/dbt_fivetran_utils/pull/139) includes the following update:
+- Added the new `fivetran_union_relations()` macro.
+  - This was necessary for adding `union_data()` support to Zendesk the package, which works with some tables that have reserved-keywords for names (ie timezone).
+  - We heavily leveraged `dbt_utils.union_relations()` but adjusted it to work with these problematically named source tables.
+> Note: We already do have a macro calles `union_relations()` in this package, but as we learned [recently](https://github.com/fivetran/dbt_fivetran_utils/releases/tag/v0.4.10), using the same exact name as a macro in `dbt_utils` is not a good idea. Thus, `fivetran_utils.union_relations()` will be deprecated in favor of `fivetran_utils.fivetran_union_relations()` in the future.
+
 
 # dbt_fivetran_utils v0.4.10
 

From 05e364ca8ab56653b48338937b6b390a5ab75347 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 26 Dec 2023 16:10:12 -0800
Subject: [PATCH 06/48] more docs

---
 CHANGELOG.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8175b1d..e5346f7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,8 +5,9 @@
 - Added the new `fivetran_union_relations()` macro.
   - This was necessary for adding `union_data()` support to Zendesk the package, which works with some tables that have reserved-keywords for names (ie timezone).
   - We heavily leveraged `dbt_utils.union_relations()` but adjusted it to work with these problematically named source tables.
-> Note: We already do have a macro calles `union_relations()` in this package, but as we learned [recently](https://github.com/fivetran/dbt_fivetran_utils/releases/tag/v0.4.10), using the same exact name as a macro in `dbt_utils` is not a good idea. Thus, `fivetran_utils.union_relations()` will be deprecated in favor of `fivetran_utils.fivetran_union_relations()` in the future.
+- Adjusted the `union_data()` macro to reference `fivetran_union_relations()` instead of `dbt_utils.union_relations()`.
 
+> Note: We already do have a macro calles `union_relations()` in this package, but as we learned [recently](https://github.com/fivetran/dbt_fivetran_utils/releases/tag/v0.4.10), using the same exact name as a macro in `dbt_utils` is not a good idea. Thus, `fivetran_utils.union_relations()` will be deprecated in favor of `fivetran_utils.fivetran_union_relations()` in the future. Currently it is used in only one package (Marketo).
 
 # dbt_fivetran_utils v0.4.10
 

From ea643e2f341b47402b40ab043fdedc88665df5b4 Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 28 Dec 2023 18:17:45 +0100
Subject: [PATCH 07/48] bug/redshift-constant-exp

---
 macros/source_relation.sql |  2 +-
 macros/union_data.sql      | 20 +++++++++++++++++---
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/macros/source_relation.sql b/macros/source_relation.sql
index 7ca9810..e0022cf 100644
--- a/macros/source_relation.sql
+++ b/macros/source_relation.sql
@@ -19,7 +19,7 @@
     {% endfor %}
   end as source_relation
 {% else %}
-, cast('' as {{ dbt.type_string() }}) as source_relation
+, _dbt_source_relation as source_relation
 {% endif %}
 
 {% endmacro %}
diff --git a/macros/union_data.sql b/macros/union_data.sql
index a67fadb..9edf0f6 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -122,12 +122,26 @@
 {%- set table_exists=relation.value is not none -%}
 
 {%- if table_exists -%}
-    select * 
+    {%- set columns = adapter.get_columns_in_relation(relation.value) -%}
+    {%- set col_dbt_source_relation_exists = false -%}
+    {%- for col in columns -%}
+        {%- if col.name == '_dbt_source_relation' -%}
+            {%- set col_dbt_source_relation_exists = true -%}
+        {%- endif -%}
+    {%- endfor -%}
+
+    select
+        {% if not col_dbt_source_relation_exists -%}
+        cast(null as {{ dbt.type_string() }}) as _dbt_source_relation,
+        {%- endif %}
+
+        {{ dbt_utils.star(from=relation.value) }}
+
     from {{ relation.value }}
 {%- else -%}
-    {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%}
+    {%- if execute and not var('fivetran__remove_empty_table_warnings', false) -%}
     {{ exceptions.warn("\n\nPlease be aware: The " ~ table_identifier|upper ~ " table was not found in your " ~ default_schema|upper ~ " schema(s). The Fivetran dbt package will create a completely empty " ~ table_identifier|upper ~ " staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }}
-    {% endif -%}
+    {%- endif -%}
     select 
         cast(null as {{ dbt.type_string() }}) as _dbt_source_relation
     limit 0

From c08f3b935ef9103e82591332cdb9da3024e9e032 Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 28 Dec 2023 18:41:05 +0100
Subject: [PATCH 08/48] bug/redshift-constant-exp

---
 macros/union_data.sql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 9edf0f6..6c7ba3b 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -132,7 +132,8 @@
 
     select
         {% if not col_dbt_source_relation_exists -%}
-        cast(null as {{ dbt.type_string() }}) as _dbt_source_relation,
+        {# cast(null as {{ dbt.type_string() }}) as _dbt_source_relation, #}
+        'placeholder' as _dbt_source_relation,
         {%- endif %}
 
         {{ dbt_utils.star(from=relation.value) }}

From 66661de008380dd9778068d0901e96d298313eeb Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 28 Dec 2023 19:02:43 +0100
Subject: [PATCH 09/48] bug/redshift-constant-exp

---
 macros/union_data.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 6c7ba3b..e815a16 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -132,8 +132,8 @@
 
     select
         {% if not col_dbt_source_relation_exists -%}
-        {# cast(null as {{ dbt.type_string() }}) as _dbt_source_relation, #}
-        'placeholder' as _dbt_source_relation,
+        cast(null as {{ dbt.type_string() }}) as _dbt_source_relation,
+        {# 'placeholder' as _dbt_source_relation, #}
         {%- endif %}
 
         {{ dbt_utils.star(from=relation.value) }}

From 9e20e57dd22f6a42384caaea388b7d38bc382518 Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 28 Dec 2023 19:14:47 +0100
Subject: [PATCH 10/48] bug/redshift-constant-exp

---
 macros/union_data.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index e815a16..6c7ba3b 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -132,8 +132,8 @@
 
     select
         {% if not col_dbt_source_relation_exists -%}
-        cast(null as {{ dbt.type_string() }}) as _dbt_source_relation,
-        {# 'placeholder' as _dbt_source_relation, #}
+        {# cast(null as {{ dbt.type_string() }}) as _dbt_source_relation, #}
+        'placeholder' as _dbt_source_relation,
         {%- endif %}
 
         {{ dbt_utils.star(from=relation.value) }}

From 9d7029297513e4013b9b8420b7df802ac4484300 Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 28 Dec 2023 19:17:37 +0100
Subject: [PATCH 11/48] bug/redshift-constant-exp

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 6c7ba3b..df55c38 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -133,7 +133,7 @@
     select
         {% if not col_dbt_source_relation_exists -%}
         {# cast(null as {{ dbt.type_string() }}) as _dbt_source_relation, #}
-        'placeholder' as _dbt_source_relation,
+        {{ relation.schema }} as _dbt_source_relation,
         {%- endif %}
 
         {{ dbt_utils.star(from=relation.value) }}

From 5c71f515f084b72a59bb256e870d6383748be800 Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 28 Dec 2023 19:54:38 +0100
Subject: [PATCH 12/48] bug/redshift-constant-exp

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index df55c38..bed0ba1 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -133,7 +133,7 @@
     select
         {% if not col_dbt_source_relation_exists -%}
         {# cast(null as {{ dbt.type_string() }}) as _dbt_source_relation, #}
-        {{ relation.schema }} as _dbt_source_relation,
+        {{ "'" ~ relation.value.schema ~ "'" }} as _dbt_source_relation,
         {%- endif %}
 
         {{ dbt_utils.star(from=relation.value) }}

From 08c4365b7b373375af2f5e808b0ea2135255d86c Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 28 Dec 2023 20:09:06 +0100
Subject: [PATCH 13/48] bug/redshift-constant-exp

---
 macros/union_data.sql | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index bed0ba1..baf5221 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -132,7 +132,6 @@
 
     select
         {% if not col_dbt_source_relation_exists -%}
-        {# cast(null as {{ dbt.type_string() }}) as _dbt_source_relation, #}
         {{ "'" ~ relation.value.schema ~ "'" }} as _dbt_source_relation,
         {%- endif %}
 
@@ -140,9 +139,9 @@
 
     from {{ relation.value }}
 {%- else -%}
-    {%- if execute and not var('fivetran__remove_empty_table_warnings', false) -%}
+    {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%}
     {{ exceptions.warn("\n\nPlease be aware: The " ~ table_identifier|upper ~ " table was not found in your " ~ default_schema|upper ~ " schema(s). The Fivetran dbt package will create a completely empty " ~ table_identifier|upper ~ " staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }}
-    {%- endif -%}
+    {% endif -%}
     select 
         cast(null as {{ dbt.type_string() }}) as _dbt_source_relation
     limit 0

From c7cbe9885a89fb172b4ad84fa9a3142118499c4f Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 28 Dec 2023 20:21:13 +0100
Subject: [PATCH 14/48] bug/redshift-constant-exp

---
 macros/union_data.sql | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index baf5221..29385ea 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -124,10 +124,8 @@
 {%- if table_exists -%}
     {%- set columns = adapter.get_columns_in_relation(relation.value) -%}
     {%- set col_dbt_source_relation_exists = false -%}
-    {%- for col in columns -%}
-        {%- if col.name == '_dbt_source_relation' -%}
-            {%- set col_dbt_source_relation_exists = true -%}
-        {%- endif -%}
+    {%- for col in columns if col.name == '_dbt_source_relation' -%}
+        {%- set col_dbt_source_relation_exists = true -%}
     {%- endfor -%}
 
     select

From 6603273e795ff057c5aa7572a46c957f135ed4a0 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 16 Jan 2024 16:55:09 -0300
Subject: [PATCH 15/48] add connector_table_name_override aargument to
 union_data in case actual table identifiers <> source table names

---
 README.md             |  1 +
 macros/union_data.sql | 16 +++++++++-------
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index ff5a21d..d7fc607 100644
--- a/README.md
+++ b/README.md
@@ -542,6 +542,7 @@ If the source table is not found in any of the provided schemas/databases, `unio
 * `default_variable`: The name of the variable that users should populate when they want to pass one specific relation to this model (mostly used for CI)
 * `union_schema_variable` (optional): The name of the union schema variable. By default the macro will look for `union_schemas`.
 * `union_database_variable` (optional): The name of the union database variable. By default the macro will look for `union_databases`.
+* `connector_table_name_override` (optional): The _actual_ name/identifier of the table as it appears in the connector schema. Currently, this is only used in our Netsuite2 data models, whose actual table names and defined `source` tables differ slightly.
 
 #### Union Data Defined Sources Configuration
 ```yml
diff --git a/macros/union_data.sql b/macros/union_data.sql
index 0cc139e..d34900d 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -1,4 +1,4 @@
-{%- macro union_data(table_identifier, database_variable, schema_variable, default_database, default_schema, default_variable, union_schema_variable='union_schemas', union_database_variable='union_databases') -%}
+{%- macro union_data(table_identifier, database_variable, schema_variable, default_database, default_schema, default_variable, union_schema_variable='union_schemas', union_database_variable='union_databases', connector_table_name_override=None) -%}
 
 {{ adapter.dispatch('union_data', 'fivetran_utils') (
     table_identifier, 
@@ -8,7 +8,8 @@
     default_schema, 
     default_variable,
     union_schema_variable,
-    union_database_variable
+    union_database_variable,
+    connector_table_name_override
     ) }}
 
 {%- endmacro -%}
@@ -21,7 +22,8 @@
     default_schema, 
     default_variable,
     union_schema_variable,
-    union_database_variable
+    union_database_variable,
+    connector_table_name_override
     ) -%}
 
 {%- if var(union_schema_variable, none) -%}
@@ -39,7 +41,7 @@
     {%- set relation=adapter.get_relation(
         database=source(schema, table_identifier).database if var('has_defined_sources', false) else var(database_variable, default_database),
         schema=source(schema, table_identifier).schema if var('has_defined_sources', false) else schema,
-        identifier=source(schema, table_identifier).identifier if var('has_defined_sources', false) else table_identifier
+        identifier=connector_table_name_override if connector_table_name_override else (source(schema, table_identifier).identifier if var('has_defined_sources', false) else table_identifier)
     ) -%}
     
     {%- set relation_exists=relation is not none -%}
@@ -69,7 +71,7 @@
     {%- set relation=adapter.get_relation(
         database=source(schema, table_identifier).database if var('has_defined_sources', false) else database,
         schema=source(schema, table_identifier).schema if var('has_defined_sources', false) else var(schema_variable, default_schema),
-        identifier=source(schema, table_identifier).identifier if var('has_defined_sources', false) else table_identifier
+        identifier=connector_table_name_override if connector_table_name_override else (source(schema, table_identifier).identifier if var('has_defined_sources', false) else table_identifier)
     ) -%}
 
     {%- set relation_exists=relation is not none -%}
@@ -102,7 +104,7 @@
                 {%- set relation.value=adapter.get_relation(
                     database=source(corrected_schema_name[1], table_identifier).database,
                     schema=source(corrected_schema_name[1], table_identifier).schema,
-                    identifier=var(identifier_var, table_identifier)
+                    identifier=connector_table_name_override if connector_table_name_override else var(identifier_var, table_identifier)
                 ) -%}
             {% endif %}
         {% endfor %}
@@ -116,7 +118,7 @@
         {%- set relation.value=adapter.get_relation(
             database=source(default_schema, table_identifier).database,
             schema=source(default_schema, table_identifier).schema,
-            identifier=var(identifier_var, table_identifier)
+            identifier=connector_table_name_override if connector_table_name_override else var(identifier_var, table_identifier)
         ) -%}
     {% endif %}
 {%- set table_exists=relation.value is not none -%}

From f93a157b06d134941c9daeed1286c93b2e9fa0c5 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Wed, 17 Jan 2024 09:32:51 -0300
Subject: [PATCH 16/48] revert catherine's changs

---
 macros/source_relation.sql |  2 +-
 macros/union_data.sql      | 11 -----------
 2 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/macros/source_relation.sql b/macros/source_relation.sql
index e0022cf..7ca9810 100644
--- a/macros/source_relation.sql
+++ b/macros/source_relation.sql
@@ -19,7 +19,7 @@
     {% endfor %}
   end as source_relation
 {% else %}
-, _dbt_source_relation as source_relation
+, cast('' as {{ dbt.type_string() }}) as source_relation
 {% endif %}
 
 {% endmacro %}
diff --git a/macros/union_data.sql b/macros/union_data.sql
index d34900d..da92ba3 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -124,19 +124,8 @@
 {%- set table_exists=relation.value is not none -%}
 
 {%- if table_exists -%}
-    {%- set columns = adapter.get_columns_in_relation(relation.value) -%}
-    {%- set col_dbt_source_relation_exists = false -%}
-    {%- for col in columns if col.name == '_dbt_source_relation' -%}
-        {%- set col_dbt_source_relation_exists = true -%}
-    {%- endfor -%}
-
     select
-        {% if not col_dbt_source_relation_exists -%}
-        {{ "'" ~ relation.value.schema ~ "'" }} as _dbt_source_relation,
-        {%- endif %}
-
         {{ dbt_utils.star(from=relation.value) }}
-
     from {{ relation.value }}
 {%- else -%}
     {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%}

From ea317eccf1fa41a4e6b7542e46716da011d1c5c9 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Thu, 25 Jan 2024 21:35:20 -0300
Subject: [PATCH 17/48] try this out for netsuite integration tests w union
 data

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index da92ba3..07d7901 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -118,7 +118,7 @@
         {%- set relation.value=adapter.get_relation(
             database=source(default_schema, table_identifier).database,
             schema=source(default_schema, table_identifier).schema,
-            identifier=connector_table_name_override if connector_table_name_override else var(identifier_var, table_identifier)
+            identifier=connector_table_name_override if connector_table_name_override and not var(integration_tests_seed_identifer_override, false) else var(identifier_var, table_identifier)
         ) -%}
     {% endif %}
 {%- set table_exists=relation.value is not none -%}

From afa73671ebebf1f6652a16c2c67c6029eaa9c8c5 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Mon, 29 Jan 2024 13:03:23 -0400
Subject: [PATCH 18/48] log message

---
 macros/union_data.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 07d7901..c65f4fb 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -115,6 +115,7 @@
         {% if var(identifier_var, none) is none %} 
             {% set identifier_var = default_schema + "_" + table_identifier + "_identifer"  %}
         {% endif %}
+        {{ log('\nidentifier is ' ~ var(identifier_var), info=true )}}
         {%- set relation.value=adapter.get_relation(
             database=source(default_schema, table_identifier).database,
             schema=source(default_schema, table_identifier).schema,

From 1065a1e58f1a91183eed2d7538d60d87c2ea2b82 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Mon, 29 Jan 2024 13:52:19 -0400
Subject: [PATCH 19/48] testing

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index c65f4fb..4eff4a2 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -115,7 +115,7 @@
         {% if var(identifier_var, none) is none %} 
             {% set identifier_var = default_schema + "_" + table_identifier + "_identifer"  %}
         {% endif %}
-        {{ log('\nidentifier is ' ~ var(identifier_var), info=true )}}
+        {{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
         {%- set relation.value=adapter.get_relation(
             database=source(default_schema, table_identifier).database,
             schema=source(default_schema, table_identifier).schema,

From 7c335d3e1f2950cbd7051d1da3fcaee4cc1b3ddc Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Mon, 29 Jan 2024 15:25:59 -0400
Subject: [PATCH 20/48] more logging

---
 macros/union_data.sql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 4eff4a2..0c555ae 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -116,10 +116,11 @@
             {% set identifier_var = default_schema + "_" + table_identifier + "_identifer"  %}
         {% endif %}
         {{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
+        {{ '\nthe source database is ' ~ source(default_schema, table_identifier).database ~ ' and the schema is ' ~ source(default_schema, table_identifier).schema }}
         {%- set relation.value=adapter.get_relation(
             database=source(default_schema, table_identifier).database,
             schema=source(default_schema, table_identifier).schema,
-            identifier=connector_table_name_override if connector_table_name_override and not var(integration_tests_seed_identifer_override, false) else var(identifier_var, table_identifier)
+            identifier=connector_table_name_override if (connector_table_name_override and not var(integration_tests_seed_identifer_override, false)) else var(identifier_var, table_identifier)
         ) -%}
     {% endif %}
 {%- set table_exists=relation.value is not none -%}

From 471da8328e5d83c2074bcf48eeb4db771996d804 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Mon, 29 Jan 2024 16:14:06 -0400
Subject: [PATCH 21/48] fix the logging

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 0c555ae..3377611 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -116,7 +116,7 @@
             {% set identifier_var = default_schema + "_" + table_identifier + "_identifer"  %}
         {% endif %}
         {{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
-        {{ '\nthe source database is ' ~ source(default_schema, table_identifier).database ~ ' and the schema is ' ~ source(default_schema, table_identifier).schema }}
+        {{ log('\nthe source database is ' ~ source(default_schema, table_identifier).database ~ ' and the schema is ' ~ source(default_schema, table_identifier).schema, info=true) }}
         {%- set relation.value=adapter.get_relation(
             database=source(default_schema, table_identifier).database,
             schema=source(default_schema, table_identifier).schema,

From 764a3d21c0b6675de556287170b5e43459f0fd1d Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 30 Jan 2024 09:11:27 -0400
Subject: [PATCH 22/48] more more logging

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 3377611..5eb2965 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -124,7 +124,7 @@
         ) -%}
     {% endif %}
 {%- set table_exists=relation.value is not none -%}
-
+{{ log('does the' ~ relation.value ~ 'table exist? ' ~ table_exists, info=true)}}
 {%- if table_exists -%}
     select
         {{ dbt_utils.star(from=relation.value) }}

From 78b97628dc1117c017ad53db14067a9eca8a8399 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 30 Jan 2024 10:39:53 -0400
Subject: [PATCH 23/48] more derailed logging

---
 macros/union_data.sql | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 5eb2965..1c51ee4 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -115,8 +115,7 @@
         {% if var(identifier_var, none) is none %} 
             {% set identifier_var = default_schema + "_" + table_identifier + "_identifer"  %}
         {% endif %}
-        {{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
-        {{ log('\nthe source database is ' ~ source(default_schema, table_identifier).database ~ ' and the schema is ' ~ source(default_schema, table_identifier).schema, info=true) }}
+        {# {{ log('\nthe source database is ' ~ source(default_schema, table_identifier).database ~ ' and the schema is ' ~ source(default_schema, table_identifier).schema, info=true) }} #}
         {%- set relation.value=adapter.get_relation(
             database=source(default_schema, table_identifier).database,
             schema=source(default_schema, table_identifier).schema,
@@ -124,7 +123,8 @@
         ) -%}
     {% endif %}
 {%- set table_exists=relation.value is not none -%}
-{{ log('does the' ~ relation.value ~ 'table exist? ' ~ table_exists, info=true)}}
+{{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
+{{ log('does the' ~ table_identifier ~ 'table exist? ' ~ table_exists, info=true)}}
 {%- if table_exists -%}
     select
         {{ dbt_utils.star(from=relation.value) }}

From d7a973cbc7a8f89fa46555d1bbbdebe417b0138e Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 30 Jan 2024 13:23:38 -0400
Subject: [PATCH 24/48] extra specific logging

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 1c51ee4..c057198 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -124,7 +124,7 @@
     {% endif %}
 {%- set table_exists=relation.value is not none -%}
 {{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
-{{ log('does the' ~ table_identifier ~ 'table exist? ' ~ table_exists, info=true)}}
+{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier ' ~ (connector_table_name_override if (connector_table_name_override and not var(integration_tests_seed_identifer_override, false)) else var(identifier_var, table_identifier) ), info=true)}}
 {%- if table_exists -%}
     select
         {{ dbt_utils.star(from=relation.value) }}

From c93734da8bb7fe12baa2572e7e1e93104c3d8e05 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 30 Jan 2024 13:52:51 -0400
Subject: [PATCH 25/48] figuring it out...

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index c057198..7824b97 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -124,7 +124,7 @@
     {% endif %}
 {%- set table_exists=relation.value is not none -%}
 {{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
-{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier ' ~ (connector_table_name_override if (connector_table_name_override and not var(integration_tests_seed_identifer_override, false)) else var(identifier_var, table_identifier) ), info=true)}}
+{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier: ' ~ (connector_table_name_override if (connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, false)) else var(identifier_var, table_identifier) ) ~ ' and not integration_tests_seed_identifer_override is ' ~ not var(integration_tests_seed_identifer_override, false), info=true) }}
 {%- if table_exists -%}
     select
         {{ dbt_utils.star(from=relation.value) }}

From 8fc8c8cb8d7f311ac5ee2123336dce4047d6873b Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 30 Jan 2024 14:06:03 -0400
Subject: [PATCH 26/48] parentheses

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 7824b97..dd53d2b 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -124,7 +124,7 @@
     {% endif %}
 {%- set table_exists=relation.value is not none -%}
 {{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
-{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier: ' ~ (connector_table_name_override if (connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, false)) else var(identifier_var, table_identifier) ) ~ ' and not integration_tests_seed_identifer_override is ' ~ not var(integration_tests_seed_identifer_override, false), info=true) }}
+{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier: ' ~ connector_table_name_override if (connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, false)) else var(identifier_var, table_identifier) ) ~ ' and not integration_tests_seed_identifer_override is ' ~ not var(integration_tests_seed_identifer_override, false), info=true) }}
 {%- if table_exists -%}
     select
         {{ dbt_utils.star(from=relation.value) }}

From 5dfbc07db251aa0a161edb5f36ac1d7d3581f7ea Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 30 Jan 2024 15:22:24 -0400
Subject: [PATCH 27/48] fix logs

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index dd53d2b..bb8d1fc 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -124,7 +124,7 @@
     {% endif %}
 {%- set table_exists=relation.value is not none -%}
 {{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
-{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier: ' ~ connector_table_name_override if (connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, false)) else var(identifier_var, table_identifier) ) ~ ' and not integration_tests_seed_identifer_override is ' ~ not var(integration_tests_seed_identifer_override, false), info=true) }}
+{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier: ' ~ connector_table_name_override if connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, fals) else var(identifier_var, table_identifier) ~ ' and not integration_tests_seed_identifer_override is ' ~ not var(integration_tests_seed_identifer_override, false), info=true) }}
 {%- if table_exists -%}
     select
         {{ dbt_utils.star(from=relation.value) }}

From f43d93d029a7fca0fb0dd3784401f076873ae61c Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 30 Jan 2024 15:26:56 -0400
Subject: [PATCH 28/48] ok now should run

---
 macros/union_data.sql | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index bb8d1fc..42c0620 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -124,7 +124,9 @@
     {% endif %}
 {%- set table_exists=relation.value is not none -%}
 {{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
-{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier: ' ~ connector_table_name_override if connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, fals) else var(identifier_var, table_identifier) ~ ' and not integration_tests_seed_identifer_override is ' ~ not var(integration_tests_seed_identifer_override, false), info=true) }}
+{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ '. Well this is the identifier: ' ~ (connector_table_name_override if connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, false) else var(identifier_var, table_identifier)) ~ ' and integration_tests_seed_identifer_override is ' ~ var(integration_tests_seed_identifer_override, false), info=true) }}
+
+{# {{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier: ' ~ connector_table_name_override if connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, false) else var(identifier_var, table_identifier) ~ ' and not integration_tests_seed_identifer_override is ' ~ not var(integration_tests_seed_identifer_override, false), info=true) }} #}
 {%- if table_exists -%}
     select
         {{ dbt_utils.star(from=relation.value) }}

From 115718316f670e512daffa5cfdcd0c653f0062bb Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Wed, 31 Jan 2024 09:11:25 -0400
Subject: [PATCH 29/48] add quotes around var

---
 macros/union_data.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 42c0620..bca58d0 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -119,12 +119,12 @@
         {%- set relation.value=adapter.get_relation(
             database=source(default_schema, table_identifier).database,
             schema=source(default_schema, table_identifier).schema,
-            identifier=connector_table_name_override if (connector_table_name_override and not var(integration_tests_seed_identifer_override, false)) else var(identifier_var, table_identifier)
+            identifier=connector_table_name_override if (connector_table_name_override and not var('integration_tests_seed_identifer_override', false)) else var(identifier_var, table_identifier)
         ) -%}
     {% endif %}
 {%- set table_exists=relation.value is not none -%}
 {{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
-{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ '. Well this is the identifier: ' ~ (connector_table_name_override if connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, false) else var(identifier_var, table_identifier)) ~ ' and integration_tests_seed_identifer_override is ' ~ var(integration_tests_seed_identifer_override, false), info=true) }}
+{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ '. Well this is the identifier: ' ~ (connector_table_name_override if connector_table_name_override is not none and not var('integration_tests_seed_identifer_override', false) else var(identifier_var, table_identifier)) ~ ' and integration_tests_seed_identifer_override is ' ~ var('integration_tests_seed_identifer_override', false), info=true) }}
 
 {# {{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier: ' ~ connector_table_name_override if connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, false) else var(identifier_var, table_identifier) ~ ' and not integration_tests_seed_identifer_override is ' ~ not var(integration_tests_seed_identifer_override, false), info=true) }} #}
 {%- if table_exists -%}

From 7ef4d1cd04d5066c3a3f9feef19a76234d267492 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Wed, 31 Jan 2024 09:39:40 -0400
Subject: [PATCH 30/48] working woop woop

---
 README.md             | 10 ++++++++++
 macros/union_data.sql |  4 ----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index d7fc607..2723d31 100644
--- a/README.md
+++ b/README.md
@@ -84,6 +84,7 @@ dispatch:
     - [source\_relation (source)](#source_relation-source)
     - [union\_data (source)](#union_data-source)
       - [Union Data Defined Sources Configuration](#union-data-defined-sources-configuration)
+      - [Using identifiers for seeds in Buildkite in packages with `union_data`](#using-identifiers-for-seeds-in-buildkite-in-packages-with-union_data)
     - [fivetran\_union\_relations (source)](#fivetran_union_relations-source)
     - [union\_relations (source)](#union_relations-source)
   - [Variable Checks](#variable-checks)
@@ -579,6 +580,15 @@ sources:
       - name: customer 
       ...
 ```
+
+#### Using identifiers for seeds in Buildkite in packages with `union_data`
+In our integration tests, we often have seed files that do not have the default names for their respective tables. They might have a `_data` suffix or something along those lines. In these cases, we make use of our identifier variables in the `integration_tests/dbt_project.yml` file. To ensure that the `union_data` macro picks these custom name configs up, set the following variable to `true` in the package's `integration_tests/dbt_project.yml` file:
+
+```yml
+vars:
+  integration_tests_seed_identifer_override: true
+```
+
 ----
 ### fivetran_union_relations ([source](macros/union_relations.sql))
 Heavily adapted from [dbt_utils.union_relations()](https://github.com/dbt-labs/dbt-utils?tab=readme-ov-file#union_relations-source).
diff --git a/macros/union_data.sql b/macros/union_data.sql
index bca58d0..63ee026 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -115,7 +115,6 @@
         {% if var(identifier_var, none) is none %} 
             {% set identifier_var = default_schema + "_" + table_identifier + "_identifer"  %}
         {% endif %}
-        {# {{ log('\nthe source database is ' ~ source(default_schema, table_identifier).database ~ ' and the schema is ' ~ source(default_schema, table_identifier).schema, info=true) }} #}
         {%- set relation.value=adapter.get_relation(
             database=source(default_schema, table_identifier).database,
             schema=source(default_schema, table_identifier).schema,
@@ -123,10 +122,7 @@
         ) -%}
     {% endif %}
 {%- set table_exists=relation.value is not none -%}
-{{ log('\nthe ' ~ identifier_var ~ ' identifier for ' ~ table_identifier ~ ' is ' ~ var(identifier_var, table_identifier), info=true )}}
-{{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ '. Well this is the identifier: ' ~ (connector_table_name_override if connector_table_name_override is not none and not var('integration_tests_seed_identifer_override', false) else var(identifier_var, table_identifier)) ~ ' and integration_tests_seed_identifer_override is ' ~ var('integration_tests_seed_identifer_override', false), info=true) }}
 
-{# {{ log('does the ' ~ table_identifier ~ ' table exist? ' ~ table_exists ~ ' well this is the identifier: ' ~ connector_table_name_override if connector_table_name_override is not none and not var(integration_tests_seed_identifer_override, false) else var(identifier_var, table_identifier) ~ ' and not integration_tests_seed_identifer_override is ' ~ not var(integration_tests_seed_identifer_override, false), info=true) }} #}
 {%- if table_exists -%}
     select
         {{ dbt_utils.star(from=relation.value) }}

From 6716cb629cc35f8b823edb0d82dbfef6e81019af Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Wed, 31 Jan 2024 10:23:13 -0400
Subject: [PATCH 31/48] docs and change var name

---
 README.md             | 8 +++++---
 macros/union_data.sql | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 2723d31..84bf507 100644
--- a/README.md
+++ b/README.md
@@ -84,7 +84,7 @@ dispatch:
     - [source\_relation (source)](#source_relation-source)
     - [union\_data (source)](#union_data-source)
       - [Union Data Defined Sources Configuration](#union-data-defined-sources-configuration)
-      - [Using identifiers for seeds in Buildkite in packages with `union_data`](#using-identifiers-for-seeds-in-buildkite-in-packages-with-union_data)
+      - [Using identifiers for seeds in Buildkite (or for customers to use with single-connector runs) in packages with `union_data`](#using-identifiers-for-seeds-in-buildkite-or-for-customers-to-use-with-single-connector-runs-in-packages-with-union_data)
     - [fivetran\_union\_relations (source)](#fivetran_union_relations-source)
     - [union\_relations (source)](#union_relations-source)
   - [Variable Checks](#variable-checks)
@@ -581,14 +581,16 @@ sources:
       ...
 ```
 
-#### Using identifiers for seeds in Buildkite in packages with `union_data`
+#### Using identifiers for seeds in Buildkite (or for customers to use with single-connector runs) in packages with `union_data`
 In our integration tests, we often have seed files that do not have the default names for their respective tables. They might have a `_data` suffix or something along those lines. In these cases, we make use of our identifier variables in the `integration_tests/dbt_project.yml` file. To ensure that the `union_data` macro picks these custom name configs up, set the following variable to `true` in the package's `integration_tests/dbt_project.yml` file:
 
 ```yml
 vars:
-  integration_tests_seed_identifer_override: true
+  use_table_name_identifer_override: true
 ```
 
+This can also be used by customers to utilize identifier variables in packages that offer unioning capability (but for single-connector runs only). Identifiers cannot currently be used when unioning multiple connectors.
+
 ----
 ### fivetran_union_relations ([source](macros/union_relations.sql))
 Heavily adapted from [dbt_utils.union_relations()](https://github.com/dbt-labs/dbt-utils?tab=readme-ov-file#union_relations-source).
diff --git a/macros/union_data.sql b/macros/union_data.sql
index 63ee026..291cc00 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -118,7 +118,7 @@
         {%- set relation.value=adapter.get_relation(
             database=source(default_schema, table_identifier).database,
             schema=source(default_schema, table_identifier).schema,
-            identifier=connector_table_name_override if (connector_table_name_override and not var('integration_tests_seed_identifer_override', false)) else var(identifier_var, table_identifier)
+            identifier=connector_table_name_override if (connector_table_name_override and not var('use_table_name_identifer_override', false)) else var(identifier_var, connector_table_name_override if connector_table_name_override else table_identifier)
         ) -%}
     {% endif %}
 {%- set table_exists=relation.value is not none -%}

From c8ace80ff5d59a458821a5b078dc745093eba175 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Wed, 31 Jan 2024 15:15:19 -0400
Subject: [PATCH 32/48] document

---
 CHANGELOG.md |  5 ++++-
 README.md    | 37 ++++++++++++++++++++++---------------
 2 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e5346f7..48cfa66 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,10 @@
 - Added the new `fivetran_union_relations()` macro.
   - This was necessary for adding `union_data()` support to Zendesk the package, which works with some tables that have reserved-keywords for names (ie timezone).
   - We heavily leveraged `dbt_utils.union_relations()` but adjusted it to work with these problematically named source tables.
-- Adjusted the `union_data()` macro to reference `fivetran_union_relations()` instead of `dbt_utils.union_relations()`.
+- Adjusted the `union_data()` macro:
+  - Now references `fivetran_union_relations()` instead of `dbt_utils.union_relations()`.
+  - Now includes a new optonal argument, `connector_table_name_override`, which is the _actual_ name/identifier of the table as it appears in the connector schema. Currently, this is only used in our Netsuite2 data models, whose actual table names and defined `source` table names differ slightly due to our support of both the Netsuite1 and Netsuite2 endpoints.
+  - Includes a reference to a new `use_table_name_identifer_override` global variable. This variable was introduced to accommodate the use of `identifier` variables in packages where `connector_table_name_override` is used (again, just Netsuite for now).
 
 > Note: We already do have a macro calles `union_relations()` in this package, but as we learned [recently](https://github.com/fivetran/dbt_fivetran_utils/releases/tag/v0.4.10), using the same exact name as a macro in `dbt_utils` is not a good idea. Thus, `fivetran_utils.union_relations()` will be deprecated in favor of `fivetran_utils.fivetran_union_relations()` in the future. Currently it is used in only one package (Marketo).
 
diff --git a/README.md b/README.md
index 84bf507..4c1ad0d 100644
--- a/README.md
+++ b/README.md
@@ -83,8 +83,8 @@ dispatch:
     - [remove\_prefix\_from\_columns (source)](#remove_prefix_from_columns-source)
     - [source\_relation (source)](#source_relation-source)
     - [union\_data (source)](#union_data-source)
+      - [Using identifiers in packages that have `union_data` and include the `connector_table_name_override` argument (currently only Netsuite2)](#using-identifiers-in-packages-that-have-union_data-and-include-the-connector_table_name_override-argument-currently-only-netsuite2)
       - [Union Data Defined Sources Configuration](#union-data-defined-sources-configuration)
-      - [Using identifiers for seeds in Buildkite (or for customers to use with single-connector runs) in packages with `union_data`](#using-identifiers-for-seeds-in-buildkite-or-for-customers-to-use-with-single-connector-runs-in-packages-with-union_data)
     - [fivetran\_union\_relations (source)](#fivetran_union_relations-source)
     - [union\_relations (source)](#union_relations-source)
   - [Variable Checks](#variable-checks)
@@ -520,6 +520,12 @@ To create dependencies between the unioned model and its *sources*, you **must d
 
 If the source table is not found in any of the provided schemas/databases, `union_data` will return a **completely** empty table (ie `limit 0`) with just one string column (`_dbt_source_relation`). A compiler warning message will be output, highlighting that the expected source table was not found and its respective staging model is empty. The compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True`.
 
+```yml
+# in root dbt_project.yml file
+vars:
+  fivetran__remove_empty_table_warnings: true # false by default
+```
+
 **Usage:**
 ```sql
 -- in model.sql file
@@ -543,16 +549,27 @@ If the source table is not found in any of the provided schemas/databases, `unio
 * `default_variable`: The name of the variable that users should populate when they want to pass one specific relation to this model (mostly used for CI)
 * `union_schema_variable` (optional): The name of the union schema variable. By default the macro will look for `union_schemas`.
 * `union_database_variable` (optional): The name of the union database variable. By default the macro will look for `union_databases`.
-* `connector_table_name_override` (optional): The _actual_ name/identifier of the table as it appears in the connector schema. Currently, this is only used in our Netsuite2 data models, whose actual table names and defined `source` tables differ slightly.
+* `connector_table_name_override` (optional): The _actual_ name/identifier of the table as it appears in the connector schema. Currently, this is only used in our Netsuite2 data models, whose actual table names and defined `source` table names differ slightly due to our support of both the Netsuite1 and Netsuite2 endpoints.
+  * If this argument is used, refer to the below subsection.
+
+#### Using identifiers in packages that have `union_data` and include the `connector_table_name_override` argument (currently only Netsuite2)
+In our integration tests, we often have seed files that do not have the default names for their respective tables. They might have a `_data` suffix or something along those lines. In these cases, we make use of our identifier variables in the `integration_tests/dbt_project.yml` file. To ensure that the `union_data` macro picks these custom name configs up and does not use the `connector_table_name_override` (only a thing in Netsuite), set the following variable to `true` in the package's `integration_tests/dbt_project.yml` file:
+
+```yml
+vars:
+  use_table_name_identifer_override: true
+```
+
+This can also be used by customers, but is **only relevant to Netsuite2** because the `source` table names != actual connector table names.
 
 #### Union Data Defined Sources Configuration
+To create dependencies between the unioned model and its *sources*, you **must define** the source tables in a `.yml` file in your project and set the `has_defined_sources` variable (scoped to the source package in which the macro is being called) to `True` in your `dbt_project.yml` file. If you set `has_defined_sources` to true and do not define sources (at least adding the `name` of each table in the source), dbt will throw an error.
+
 ```yml
 # in root dbt_project.yml file
 vars:
-  shopify_source:
+  shopify_source: # or whatever source package
     has_defined_sources: true
-  
-  fivetran__remove_empty_table_warnings: true # false by default
 ```
 
 ```yml
@@ -581,16 +598,6 @@ sources:
       ...
 ```
 
-#### Using identifiers for seeds in Buildkite (or for customers to use with single-connector runs) in packages with `union_data`
-In our integration tests, we often have seed files that do not have the default names for their respective tables. They might have a `_data` suffix or something along those lines. In these cases, we make use of our identifier variables in the `integration_tests/dbt_project.yml` file. To ensure that the `union_data` macro picks these custom name configs up, set the following variable to `true` in the package's `integration_tests/dbt_project.yml` file:
-
-```yml
-vars:
-  use_table_name_identifer_override: true
-```
-
-This can also be used by customers to utilize identifier variables in packages that offer unioning capability (but for single-connector runs only). Identifiers cannot currently be used when unioning multiple connectors.
-
 ----
 ### fivetran_union_relations ([source](macros/union_relations.sql))
 Heavily adapted from [dbt_utils.union_relations()](https://github.com/dbt-labs/dbt-utils?tab=readme-ov-file#union_relations-source).

From 3a9bf1712b0b5fa005ff14076baee8075a6c4dba Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Mon, 5 Feb 2024 10:26:10 -0500
Subject: [PATCH 33/48] alias source table in case its a reserved keyword -
 single connector use case

---
 macros/union_data.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/union_data.sql b/macros/union_data.sql
index 291cc00..c450a80 100644
--- a/macros/union_data.sql
+++ b/macros/union_data.sql
@@ -126,7 +126,7 @@
 {%- if table_exists -%}
     select
         {{ dbt_utils.star(from=relation.value) }}
-    from {{ relation.value }}
+    from {{ relation.value }} as {{ table_identifier }}_table
 {%- else -%}
     {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%}
     {{ exceptions.warn("\n\nPlease be aware: The " ~ table_identifier|upper ~ " table was not found in your " ~ default_schema|upper ~ " schema(s). The Fivetran dbt package will create a completely empty " ~ table_identifier|upper ~ " staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }}

From f6ba3553962cca8d93874516fbe4b31cb113e15f Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Wed, 7 Feb 2024 05:57:55 -0800
Subject: [PATCH 34/48] Update CHANGELOG.md

Co-authored-by: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com>
---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 48cfa66..d1f3858 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,7 @@
   - Now includes a new optonal argument, `connector_table_name_override`, which is the _actual_ name/identifier of the table as it appears in the connector schema. Currently, this is only used in our Netsuite2 data models, whose actual table names and defined `source` table names differ slightly due to our support of both the Netsuite1 and Netsuite2 endpoints.
   - Includes a reference to a new `use_table_name_identifer_override` global variable. This variable was introduced to accommodate the use of `identifier` variables in packages where `connector_table_name_override` is used (again, just Netsuite for now).
 
-> Note: We already do have a macro calles `union_relations()` in this package, but as we learned [recently](https://github.com/fivetran/dbt_fivetran_utils/releases/tag/v0.4.10), using the same exact name as a macro in `dbt_utils` is not a good idea. Thus, `fivetran_utils.union_relations()` will be deprecated in favor of `fivetran_utils.fivetran_union_relations()` in the future. Currently it is used in only one package (Marketo).
+> Note: We already do have a macro called `union_relations()` in this package, but as we learned [recently](https://github.com/fivetran/dbt_fivetran_utils/releases/tag/v0.4.10), using the same exact name as a macro in `dbt_utils` is not a good idea. Thus, `fivetran_utils.union_relations()` will be deprecated in favor of `fivetran_utils.fivetran_union_relations()` in the future. Currently it is used in only one package (Marketo).
 
 # dbt_fivetran_utils v0.4.10
 

From 735194b80fac2efb63dd54243807ede8c7efd168 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Wed, 7 Feb 2024 05:58:03 -0800
Subject: [PATCH 35/48] Update CHANGELOG.md

Co-authored-by: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com>
---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d1f3858..eab80ea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,7 +7,7 @@
   - We heavily leveraged `dbt_utils.union_relations()` but adjusted it to work with these problematically named source tables.
 - Adjusted the `union_data()` macro:
   - Now references `fivetran_union_relations()` instead of `dbt_utils.union_relations()`.
-  - Now includes a new optonal argument, `connector_table_name_override`, which is the _actual_ name/identifier of the table as it appears in the connector schema. Currently, this is only used in our Netsuite2 data models, whose actual table names and defined `source` table names differ slightly due to our support of both the Netsuite1 and Netsuite2 endpoints.
+  - Now includes a new optional argument, `connector_table_name_override`, which is the _actual_ name/identifier of the table as it appears in the connector schema. Currently, this is only used in our Netsuite2 data models, whose actual table names and defined `source` table names differ slightly due to our support of both the Netsuite1 and Netsuite2 endpoints.
   - Includes a reference to a new `use_table_name_identifer_override` global variable. This variable was introduced to accommodate the use of `identifier` variables in packages where `connector_table_name_override` is used (again, just Netsuite for now).
 
 > Note: We already do have a macro called `union_relations()` in this package, but as we learned [recently](https://github.com/fivetran/dbt_fivetran_utils/releases/tag/v0.4.10), using the same exact name as a macro in `dbt_utils` is not a good idea. Thus, `fivetran_utils.union_relations()` will be deprecated in favor of `fivetran_utils.fivetran_union_relations()` in the future. Currently it is used in only one package (Marketo).

From 43dae723d0ea2a148c1f16231e6b322f6032ab71 Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 18 Apr 2024 16:01:35 -0500
Subject: [PATCH 36/48] update changelog

---
 CHANGELOG.md                 |  3 +++
 README.md                    | 23 +++++++++++++++++++++++
 macros/fivetran_lookback.sql | 18 ++++++++++++++++++
 3 files changed, 44 insertions(+)
 create mode 100644 macros/fivetran_lookback.sql

diff --git a/CHANGELOG.md b/CHANGELOG.md
index eab80ea..e573b62 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,9 @@
 
 > Note: We already do have a macro called `union_relations()` in this package, but as we learned [recently](https://github.com/fivetran/dbt_fivetran_utils/releases/tag/v0.4.10), using the same exact name as a macro in `dbt_utils` is not a good idea. Thus, `fivetran_utils.union_relations()` will be deprecated in favor of `fivetran_utils.fivetran_union_relations()` in the future. Currently it is used in only one package (Marketo).
 
+[PR #145](https://github.com/fivetran/dbt_fivetran_utils/pull/145) includes the following update:
+- Added the new `fivetran_lookback()` macro. This generates SQL to simplify adding lookback windows to incremental models. See [README](https://github.com/fivetran/dbt_fivetran_utils/blob/releases/v0.4.latest/README.md#fivetran_lookback-source).
+
 # dbt_fivetran_utils v0.4.10
 
 ## Bug Fix
diff --git a/README.md b/README.md
index 4c1ad0d..0d9ad33 100644
--- a/README.md
+++ b/README.md
@@ -79,6 +79,7 @@ dispatch:
     - [dummy\_coalesce\_value (source)](#dummy_coalesce_value-source)
     - [fill\_pass\_through\_columns (source)](#fill_pass_through_columns-source)
     - [fill\_staging\_columns (source)](#fill_staging_columns-source)
+    - [fivetran\_lookback (source)](#fivetran_lookback-source)
     - [persist\_pass\_through\_columns (source)](#persist_pass_through_columns-source)
     - [remove\_prefix\_from\_columns (source)](#remove_prefix_from_columns-source)
     - [source\_relation (source)](#source_relation-source)
@@ -466,6 +467,28 @@ from source
 * `source_columns`  (required): Will call the [get_columns_in_relation](https://docs.getdbt.com/reference/dbt-jinja-functions/adapter/#get_columns_in_relation) macro as well requires a `ref()` or `source()` argument for the staging models within the `_tmp` directory.
 * `staging_columns` (required): Created as a result of running the [generate_columns_macro](https://github.com/fivetran/dbt_fivetran_utils#generate_columns_macro-source) for the respective table.
 
+----
+### fivetran_lookback ([source](macros/fivetran_lookback.sql))
+This macro takes a date expression and uses `dbt.dateadd` and backdates it by the specified interval. This is intended for use in incremental blocks to look backwards and catch late arriving records. Any date aggregate can be used as an input, but the typical usage is with the max date of the present model.
+
+**Usage:**
+```sql
+{% if is_incremental() %}
+  where date_day >= 
+    {{ netsuite.netsuite_lookback(
+      from_date='max(date_day)',
+      interval=3,
+      datepart='day', 
+      safety_date='2010-01-01')
+    }}
+{% endif %}
+```
+**Args:**
+* `from_date`  (required): String to run against the current model. The expression used should be expected to return a single result.
+* `interval` (required): The number of units to look backwards.
+* `datepart` (required): The grain of the interval. 
+* `safety_date` (optional): This date will be used in the rare case that the `from_date` expression returns a null value. This only needs to be specified if you want to change the default value or '2010-01-01'.
+
 ----
 ### persist_pass_through_columns ([source](macros/persist_pass_through_columns.sql))
 This macro is used to persist pass through columns from the staging model to the **transform** package. This is particularly helpful when a `select *` is not feasible.
diff --git a/macros/fivetran_lookback.sql b/macros/fivetran_lookback.sql
new file mode 100644
index 0000000..df7c6ca
--- /dev/null
+++ b/macros/fivetran_lookback.sql
@@ -0,0 +1,18 @@
+{% macro fivetran_lookback(from_date, datepart, interval, safety_date='2010-01-01') %}
+
+{{ adapter.dispatch('fivetran_lookback', 'fivetran') (from_date, datepart, interval, safety_date='2010-01-01') }}
+
+{%- endmacro %}
+
+{% macro default__fivetran_lookback(from_date, datepart, interval, safety_date='2010-01-01')  %}
+
+    {% set sql_statement %}
+        select coalesce({{ from_date }}, {{ "'" ~ safety_date ~ "'" }})
+        from {{ this }}
+    {%- endset -%}
+
+    {%- set result = dbt_utils.get_single_value(sql_statement) %}
+
+    {{ dbt.dateadd(datepart=datepart, interval=-interval, from_date_or_timestamp="cast('" ~ result ~ "' as date)") }}
+
+{% endmacro %}
\ No newline at end of file

From 19e5a985c2b5f3472eed03e9cef348dd763fdf0f Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 18 Apr 2024 16:19:31 -0500
Subject: [PATCH 37/48] update packages

---
 integration_tests/packages.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml
index 1f95659..5403ade 100644
--- a/integration_tests/packages.yml
+++ b/integration_tests/packages.yml
@@ -2,8 +2,10 @@ packages:
   - package: fivetran/ad_reporting
     version: [">=1.0.0", "<2.0.0"]
 
-  - package: fivetran/shopify_holistic_reporting
-    version: [">=0.1.0", "<1.0.0"]
+  # - package: fivetran/shopify_holistic_reporting
+  #   version: [">=0.1.0", "<1.0.0"]
+  - git: https://github.com/fivetran/dbt_shopify_holistic_reporting.git
+    revision: feature/bump-packages
 
   - package: fivetran/social_media_reporting
     version: [">=0.1.0", "<1.0.0"]

From c282e59ede95fa800b4367a4e50afa905884dc34 Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Thu, 18 Apr 2024 16:55:28 -0500
Subject: [PATCH 38/48] update name

---
 macros/fivetran_lookback.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/fivetran_lookback.sql b/macros/fivetran_lookback.sql
index df7c6ca..b42f237 100644
--- a/macros/fivetran_lookback.sql
+++ b/macros/fivetran_lookback.sql
@@ -1,6 +1,6 @@
 {% macro fivetran_lookback(from_date, datepart, interval, safety_date='2010-01-01') %}
 
-{{ adapter.dispatch('fivetran_lookback', 'fivetran') (from_date, datepart, interval, safety_date='2010-01-01') }}
+{{ adapter.dispatch('fivetran_lookback', 'fivetran_utils') (from_date, datepart, interval, safety_date='2010-01-01') }}
 
 {%- endmacro %}
 

From 3fa1483caa2a342392a1e69f4aad3696a66efd3f Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Fri, 19 Apr 2024 12:28:54 -0500
Subject: [PATCH 39/48] add is_databricks_sql_warehouse

---
 README.md                              | 12 ++++++++++++
 macros/is_databricks_sql_warehouse.sql | 15 +++++++++++++++
 2 files changed, 27 insertions(+)
 create mode 100644 macros/is_databricks_sql_warehouse.sql

diff --git a/README.md b/README.md
index 0d9ad33..a93332c 100644
--- a/README.md
+++ b/README.md
@@ -53,6 +53,7 @@ dispatch:
 - [📋 Contents](#-contents)
   - [Tests and helpers](#tests-and-helpers)
     - [collect\_freshness (source)](#collect_freshness-source)
+    - [is\_databricks\_sql\_warehouse (source)](#is-databricks-sql-warehouse-source)
     - [seed\_data\_helper (source)](#seed_data_helper-source)
     - [snowflake\_seed\_data (source)](#snowflake_seed_data-source)
   - [Cross-database compatibility](#cross-database-compatibility)
@@ -121,6 +122,17 @@ sources:
 **Args (sorta):**
 * `meta.is_enabled` (optional): The variable(s) you would like to reference to determine if dbt should include this table in freshness tests.
 
+----
+### is_databricks_sql_warehouse ([source](macros/is_databricks_sql_warehouse.sql))
+For Databricks destinations, this macro returns `true` if the Databricks target indicates it is a SQL Warehouse. It will return return `false` if it is an All-Purpose Cluster. 
+
+***Usage:**
+```yml
+    fivetran_utils.is_databricks_sql_warehouse(target)
+```
+**Args:**
+* `target` (required): Always set this equal to "target". This will pass all the current target information to the macro.
+
 ----
 ### seed_data_helper ([source](macros/seed_data_helper.sql))
 This macro is intended to be used when a source table column is a reserved keyword in a warehouse, and Circle CI is throwing a fit.
diff --git a/macros/is_databricks_sql_warehouse.sql b/macros/is_databricks_sql_warehouse.sql
new file mode 100644
index 0000000..354c682
--- /dev/null
+++ b/macros/is_databricks_sql_warehouse.sql
@@ -0,0 +1,15 @@
+{% macro is_databricks_sql_warehouse(target) %}
+    {% if target.type in ('databricks') %}
+        {% set re = modules.re %}
+        {% set path_match = target.http_path %}
+        {% set regex_pattern = "sql/.+/warehouses/" %}
+        {% set match_result = re.search(regex_pattern, path_match) %}
+        {% if match_result %}
+            {{ return(True) }}
+        {% else %}
+            {{ return(False) }}
+        {% endif %}
+    {% else %}
+        {{ return(False) }}
+    {% endif %}
+{% endmacro %}
\ No newline at end of file

From 7d41cb9fbf7790cefd5cfbeac0f3d4e18d7eb62b Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Tue, 23 Apr 2024 10:46:57 -0500
Subject: [PATCH 40/48] update changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e573b62..88442c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@
 
 [PR #145](https://github.com/fivetran/dbt_fivetran_utils/pull/145) includes the following update:
 - Added the new `fivetran_lookback()` macro. This generates SQL to simplify adding lookback windows to incremental models. See [README](https://github.com/fivetran/dbt_fivetran_utils/blob/releases/v0.4.latest/README.md#fivetran_lookback-source).
+- Added the new `is_databricks_sql_warehouse()` macro. This determines if Databricks destination is a SQL Warehouse or All Purpose Cluster. See [README](https://github.com/fivetran/dbt_fivetran_utils/blob/releases/v0.4.latest/README.md#is-databricks-sql-warehouse-source).
 
 # dbt_fivetran_utils v0.4.10
 

From 7e4186a191eb3f5b35805d1ce31b44811887b534 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 23 Apr 2024 13:43:10 -0700
Subject: [PATCH 41/48] readme tweak

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 4c1ad0d..06dd419 100644
--- a/README.md
+++ b/README.md
@@ -600,7 +600,7 @@ sources:
 
 ----
 ### fivetran_union_relations ([source](macros/union_relations.sql))
-Heavily adapted from [dbt_utils.union_relations()](https://github.com/dbt-labs/dbt-utils?tab=readme-ov-file#union_relations-source).
+Heavily adapted from [dbt_utils.union_relations()](https://github.com/dbt-labs/dbt-utils?tab=readme-ov-file#union_relations-source). The primary difference is that our version of the macro supports unioning relations that have reserved keywords as names (ex: table called `timezone`).
 
 This macro combines via a `union all` of an array of [Relations](https://docs.getdbt.com/reference/dbt-classes#relation), even when columns have differing orders in each Relation, and/or some columns are missing from some relations. Any columns exclusive to a subset of these relations will be filled with `null` where not present. A new column (`_dbt_source_relation`) is also added to indicate the source for each record.
 
@@ -625,7 +625,7 @@ This macro combines via a `union all` of an array of [Relations](https://docs.ge
 ### union_relations ([source](macros/union_relations.sql))
 > TO BE DEPRECATED IN FAVOR OF `fivetran_union_relations`.
 >
-> Currenlty only used in Marketo transform package.
+> Currenlty only used in [Marketo](https://github.com/fivetran/dbt_marketo/blob/main/models/intermediate/marketo__change_data_scd.sql#L47) transform package.
 
 This macro unions together an array of [Relations](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation),
 even when columns have differing orders in each Relation, and/or some columns are

From d93ffb0305981a77181b24df741fa867e765cc1d Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Tue, 23 Apr 2024 14:18:52 -0700
Subject: [PATCH 42/48] shopify ref

---
 integration_tests/packages.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml
index 1f95659..5403ade 100644
--- a/integration_tests/packages.yml
+++ b/integration_tests/packages.yml
@@ -2,8 +2,10 @@ packages:
   - package: fivetran/ad_reporting
     version: [">=1.0.0", "<2.0.0"]
 
-  - package: fivetran/shopify_holistic_reporting
-    version: [">=0.1.0", "<1.0.0"]
+  # - package: fivetran/shopify_holistic_reporting
+  #   version: [">=0.1.0", "<1.0.0"]
+  - git: https://github.com/fivetran/dbt_shopify_holistic_reporting.git
+    revision: feature/bump-packages
 
   - package: fivetran/social_media_reporting
     version: [">=0.1.0", "<1.0.0"]

From 73495b5f83ae91cea02dbd15aad56fba11084d98 Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Tue, 23 Apr 2024 16:43:47 -0500
Subject: [PATCH 43/48] Apply suggestions from code review

Co-authored-by: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
---
 README.md                              | 6 +++---
 macros/is_databricks_sql_warehouse.sql | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index a93332c..db7ee9e 100644
--- a/README.md
+++ b/README.md
@@ -487,7 +487,7 @@ This macro takes a date expression and uses `dbt.dateadd` and backdates it by th
 ```sql
 {% if is_incremental() %}
   where date_day >= 
-    {{ netsuite.netsuite_lookback(
+    {{ fivetran_utils.fivetran_lookback(
       from_date='max(date_day)',
       interval=3,
       datepart='day', 
@@ -497,9 +497,9 @@ This macro takes a date expression and uses `dbt.dateadd` and backdates it by th
 ```
 **Args:**
 * `from_date`  (required): String to run against the current model. The expression used should be expected to return a single result.
-* `interval` (required): The number of units to look backwards.
 * `datepart` (required): The grain of the interval. 
-* `safety_date` (optional): This date will be used in the rare case that the `from_date` expression returns a null value. This only needs to be specified if you want to change the default value or '2010-01-01'.
+* `interval` (required): The number of units to look backwards.
+* `safety_date` (optional): This date will be used in the rare case that the `from_date` expression returns a null value. This only needs to be specified if you want to change the default value from '2010-01-01'.
 
 ----
 ### persist_pass_through_columns ([source](macros/persist_pass_through_columns.sql))
diff --git a/macros/is_databricks_sql_warehouse.sql b/macros/is_databricks_sql_warehouse.sql
index 354c682..48fd75d 100644
--- a/macros/is_databricks_sql_warehouse.sql
+++ b/macros/is_databricks_sql_warehouse.sql
@@ -1,4 +1,4 @@
-{% macro is_databricks_sql_warehouse(target) %}
+{% macro fivetran_is_databricks_sql_warehouse(target) %}
     {% if target.type in ('databricks') %}
         {% set re = modules.re %}
         {% set path_match = target.http_path %}

From 2b953b0ea879e3f0339e3648b7904d0070c7ba51 Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Tue, 23 Apr 2024 16:46:09 -0500
Subject: [PATCH 44/48] add fivetran prefix to macro names

---
 CHANGELOG.md                                                | 2 +-
 README.md                                                   | 6 +++---
 ...rehouse.sql => fivetran_is_databricks_sql_warehouse.sql} | 0
 3 files changed, 4 insertions(+), 4 deletions(-)
 rename macros/{is_databricks_sql_warehouse.sql => fivetran_is_databricks_sql_warehouse.sql} (100%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 88442c9..e3b8351 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,7 +14,7 @@
 
 [PR #145](https://github.com/fivetran/dbt_fivetran_utils/pull/145) includes the following update:
 - Added the new `fivetran_lookback()` macro. This generates SQL to simplify adding lookback windows to incremental models. See [README](https://github.com/fivetran/dbt_fivetran_utils/blob/releases/v0.4.latest/README.md#fivetran_lookback-source).
-- Added the new `is_databricks_sql_warehouse()` macro. This determines if Databricks destination is a SQL Warehouse or All Purpose Cluster. See [README](https://github.com/fivetran/dbt_fivetran_utils/blob/releases/v0.4.latest/README.md#is-databricks-sql-warehouse-source).
+- Added the new `fivetran_is_databricks_sql_warehouse()` macro. This determines if Databricks destination is a SQL Warehouse or All Purpose Cluster. See [README](https://github.com/fivetran/dbt_fivetran_utils/blob/releases/v0.4.latest/README.md#fivetran-is-databricks-sql-warehouse-source).
 
 # dbt_fivetran_utils v0.4.10
 
diff --git a/README.md b/README.md
index db7ee9e..99904e4 100644
--- a/README.md
+++ b/README.md
@@ -53,7 +53,7 @@ dispatch:
 - [📋 Contents](#-contents)
   - [Tests and helpers](#tests-and-helpers)
     - [collect\_freshness (source)](#collect_freshness-source)
-    - [is\_databricks\_sql\_warehouse (source)](#is-databricks-sql-warehouse-source)
+    - [fivetran\_is\_databricks\_sql\_warehouse (source)](#fivetran-is-databricks-sql-warehouse-source)
     - [seed\_data\_helper (source)](#seed_data_helper-source)
     - [snowflake\_seed\_data (source)](#snowflake_seed_data-source)
   - [Cross-database compatibility](#cross-database-compatibility)
@@ -123,12 +123,12 @@ sources:
 * `meta.is_enabled` (optional): The variable(s) you would like to reference to determine if dbt should include this table in freshness tests.
 
 ----
-### is_databricks_sql_warehouse ([source](macros/is_databricks_sql_warehouse.sql))
+### fivetran_is_databricks_sql_warehouse ([source](macros/fivetran_is_databricks_sql_warehouse.sql))
 For Databricks destinations, this macro returns `true` if the Databricks target indicates it is a SQL Warehouse. It will return return `false` if it is an All-Purpose Cluster. 
 
 ***Usage:**
 ```yml
-    fivetran_utils.is_databricks_sql_warehouse(target)
+    fivetran_utils.fivetran_is_databricks_sql_warehouse(target)
 ```
 **Args:**
 * `target` (required): Always set this equal to "target". This will pass all the current target information to the macro.
diff --git a/macros/is_databricks_sql_warehouse.sql b/macros/fivetran_is_databricks_sql_warehouse.sql
similarity index 100%
rename from macros/is_databricks_sql_warehouse.sql
rename to macros/fivetran_is_databricks_sql_warehouse.sql

From 4d85633b526ebc32d968027b3ce38fc06fd50c7b Mon Sep 17 00:00:00 2001
From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
Date: Tue, 23 Apr 2024 16:55:59 -0500
Subject: [PATCH 45/48] update args

---
 README.md                                       | 4 ++--
 macros/fivetran_is_databricks_sql_warehouse.sql | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 99904e4..9ff23cb 100644
--- a/README.md
+++ b/README.md
@@ -128,10 +128,10 @@ For Databricks destinations, this macro returns `true` if the Databricks target
 
 ***Usage:**
 ```yml
-    fivetran_utils.fivetran_is_databricks_sql_warehouse(target)
+    fivetran_utils.fivetran_is_databricks_sql_warehouse()
 ```
 **Args:**
-* `target` (required): Always set this equal to "target". This will pass all the current target information to the macro.
+* none
 
 ----
 ### seed_data_helper ([source](macros/seed_data_helper.sql))
diff --git a/macros/fivetran_is_databricks_sql_warehouse.sql b/macros/fivetran_is_databricks_sql_warehouse.sql
index 48fd75d..052b4f0 100644
--- a/macros/fivetran_is_databricks_sql_warehouse.sql
+++ b/macros/fivetran_is_databricks_sql_warehouse.sql
@@ -1,4 +1,4 @@
-{% macro fivetran_is_databricks_sql_warehouse(target) %}
+{% macro fivetran_is_databricks_sql_warehouse() %}
     {% if target.type in ('databricks') %}
         {% set re = modules.re %}
         {% set path_match = target.http_path %}

From 15961f9e6eefae2ce58efad0518e9bea67824c96 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Thu, 25 Apr 2024 11:14:10 -0400
Subject: [PATCH 46/48] updated github stuff

---
 .../maintainer_pull_request_template.md       | 51 ++++++++++++++++++
 .github/pull_request_template.md              | 52 ++++++-------------
 .github/workflows/auto-release.yml            | 13 +++++
 3 files changed, 80 insertions(+), 36 deletions(-)
 create mode 100644 .github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md
 create mode 100644 .github/workflows/auto-release.yml

diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md
new file mode 100644
index 0000000..768ac3f
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md
@@ -0,0 +1,51 @@
+## PR Overview
+**This PR will address the following Issue/Feature:**
+
+**This PR will result in the following new package version:**
+<!--- Please add details around your decision for breaking vs non-breaking version upgrade. If this is a breaking change, were backwards-compatible options explored? -->
+
+**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:**
+
+## PR Checklist
+### Basic Validation
+Please acknowledge that you have successfully performed the following commands locally:
+- [ ] dbt compile
+- [ ] dbt run –full-refresh
+- [ ] dbt run
+- [ ] dbt test
+- [ ] dbt run –vars (if applicable)
+
+Before marking this PR as "ready for review" the following have been applied:
+- [ ] The appropriate issue has been linked and tagged
+- [ ] You are assigned to the corresponding issue and this PR
+- [ ] BuildKite integration tests are passing
+
+### Detailed Validation
+Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review":
+- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature.
+- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages.
+- [ ] You have provided details below around the validation steps performed to gain confidence in these changes.
+<!--- Provide the steps you took to validate your changes below. -->
+
+### Standard Updates
+Please acknowledge that your PR contains the following standard updates:
+- Package versioning has been appropriately indexed in the following locations:
+    - [ ] indexed within dbt_project.yml
+    - [ ] indexed within integration_tests/dbt_project.yml
+- [ ] CHANGELOG has individual entries for each respective change in this PR
+    <!--- If there is a parallel upstream change, remember to reference the corresponding CHANGELOG as an individual entry.  -->
+- [ ] README updates have been applied (if applicable)
+    <!--- Remember to check the following README locations for common updates. →
+        <!--- Suggested install range (needed for breaking changes) →
+        <!--- Dependency matrix is appropriately updated (if applicable) →
+        <!--- New variable documentation (if applicable) -->
+- [ ] DECISIONLOG updates have been updated (if applicable)
+- [ ] Appropriate yml documentation has been added (if applicable)
+
+### dbt Docs
+Please acknowledge that after the above were all completed the below were applied to your branch:
+- [ ] docs were regenerated (unless this PR does not include any code or yml updates)
+
+### If you had to summarize this PR in an emoji, which would it be?
+<!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
+:dancer:
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 1b75bc7..b4e7e8e 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,49 +1,24 @@
-Pull Request
-**Are you a current Fivetran customer?** 
-<!--- Please tell us your name, title and company -->
+**Please provide your name and company**
 
-**What change(s) does this PR introduce?** 
-<!--- Describe what changes your PR introduces to the package and how to leverage this new feature. -->
+**Link the issue/feature request which this PR is meant to address**
+<!--- If an issue was not created, please create one first so we may discuss the PR prior to opening one. -->
 
-**Did you update the CHANGELOG?** 
+**Detail what changes this PR introduces and how this addresses the issue/feature request linked above.**
+
+**How did you validate the changes introduced within this PR?**
+
+**Which warehouse did you use to develop these changes?**
+
+**Did you update the CHANGELOG?**
 <!--- Please update the new package version’s CHANGELOG entry detailing the changes included in this PR. -->
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
 - [ ] Yes
 
-**Does this PR introduce a breaking change?**
-<!--- Does this PR introduce changes that will cause current package users' jobs to fail or require a `--full-refresh`? -->
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Yes (please provide breaking change details below.)
-- [ ] No  (please provide an explanation as to how the change is non-breaking below.)
-
-**Did you update the dbt_project.yml files with the version upgrade (please leverage standard semantic versioning)? (In both your main project and integration_tests)** 
+**Did you update the dbt_project.yml files with the version upgrade (please leverage standard semantic versioning)? (In both your main project and integration_tests)**
 <!--- The dbt_project.yml and the integration_tests/dbt_project.yml files contain the version number. Be sure to upgrade it accordingly -->
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
 - [ ] Yes
 
-**Is this PR in response to a previously created Bug or Feature Request**
-<!--- If an Issue was created it is helpful to track the progress by linking it in the PR. -->
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Yes, Issue/Feature [link bug/feature number here]
-- [ ] No 
-
-**How did you test the PR changes?** 
-<!--- Proof of testing is required in order for the PR to be approved. -->
-<!--- To check a box, remove the space and insert an x in the box (eg. [x] Buildkite). --> 
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] Buildkite <!--- Buildkite testing is only applicable to Fivetran employees. --> 
-- [ ] Local (please provide additional testing details below)
-
-**Select which warehouse(s) were used to test the PR**
-<!--- To check a warehouse remove the space and insert an x in the box (eg. [x] Bigquery). --> 
-<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] BigQuery
-- [ ] Redshift
-- [ ] Snowflake
-- [ ] Postgres
-- [ ] Databricks
-- [ ] Other (provide details below)
-
 **Provide an emoji that best describes your current mood**
 <!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
 :dancer:
@@ -51,3 +26,8 @@ Pull Request
 **Feedback**
 
 We are so excited you decided to contribute to the Fivetran community dbt package! We continue to work to improve the packages and would greatly appreciate your [feedback](https://www.surveymonkey.com/r/DQ7K7WW) on our existing dbt packages or what you'd like to see next.
+
+**PR Template** 
+- [Community Pull Request Template](?expand=1&template=pull_request_template.md) (default)
+
+- [Maintainer Pull Request Template](?expand=1&template=maintainer_pull_request_template.md) (to be used by maintainers)
diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml
new file mode 100644
index 0000000..8ed5853
--- /dev/null
+++ b/.github/workflows/auto-release.yml
@@ -0,0 +1,13 @@
+name: 'auto release'
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  call-workflow-passing-data:
+    if: github.event.pull_request.merged
+    uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
+    secrets: inherit
\ No newline at end of file

From 54328b6002e826d35253547cb7a6f10e1c9a0541 Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Thu, 25 Apr 2024 11:14:46 -0400
Subject: [PATCH 47/48] updated github stuff

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e3b8351..21e16ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,11 @@
 - Added the new `fivetran_lookback()` macro. This generates SQL to simplify adding lookback windows to incremental models. See [README](https://github.com/fivetran/dbt_fivetran_utils/blob/releases/v0.4.latest/README.md#fivetran_lookback-source).
 - Added the new `fivetran_is_databricks_sql_warehouse()` macro. This determines if Databricks destination is a SQL Warehouse or All Purpose Cluster. See [README](https://github.com/fivetran/dbt_fivetran_utils/blob/releases/v0.4.latest/README.md#fivetran-is-databricks-sql-warehouse-source).
 
+## Under the Hood
+[PR #139](https://github.com/fivetran/dbt_fivetran_utils/pull/139) includes the following update:
+- Updated the pull request [templates](https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest/.github) to align with our current format.
+- Included auto-releaser GitHub Actions workflow to automate future releases.
+
 # dbt_fivetran_utils v0.4.10
 
 ## Bug Fix

From 22cb80bac11465dccf608cdd5291c017939bfb6e Mon Sep 17 00:00:00 2001
From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com>
Date: Thu, 25 Apr 2024 12:45:14 -0400
Subject: [PATCH 48/48] remove workflow

---
 .github/workflows/auto-release.yml | 13 -------------
 CHANGELOG.md                       |  1 -
 2 files changed, 14 deletions(-)
 delete mode 100644 .github/workflows/auto-release.yml

diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml
deleted file mode 100644
index 8ed5853..0000000
--- a/.github/workflows/auto-release.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-name: 'auto release'
-on:
-  pull_request:
-    types:
-      - closed
-    branches:
-      - main
-
-jobs:
-  call-workflow-passing-data:
-    if: github.event.pull_request.merged
-    uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
-    secrets: inherit
\ No newline at end of file
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 21e16ba..9dae9a4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,7 +19,6 @@
 ## Under the Hood
 [PR #139](https://github.com/fivetran/dbt_fivetran_utils/pull/139) includes the following update:
 - Updated the pull request [templates](https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest/.github) to align with our current format.
-- Included auto-releaser GitHub Actions workflow to automate future releases.
 
 # dbt_fivetran_utils v0.4.10