This is an automated email from the ASF dual-hosted git repository.
zykkk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1110ff49f3 [feature-wip](dbt) exchange table temp to target table
atomically (#21931)
1110ff49f3 is described below
commit 1110ff49f3e123870462b1acec1745030dc2b822
Author: catpineapple <[email protected]>
AuthorDate: Wed Jul 19 10:20:50 2023 +0800
[feature-wip](dbt) exchange table temp to target table atomically (#21931)
exchange table temp to target table atomically
---
.../dbt/include/doris/macros/adapters/relation.sql | 34 ++++++++++++++++++----
.../materializations/incremental/incremental.sql | 26 +++++++++++------
2 files changed, 46 insertions(+), 14 deletions(-)
diff --git a/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql
b/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql
index 7c9f2ba57b..03990efff3 100644
--- a/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql
+++ b/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql
@@ -154,7 +154,31 @@
{% endcall %}
{% endif %}
- {%- endmacro %}
+{%- endmacro %}
+
+
+{% macro exchange_relation(relation1, relation2, is_drop_r1=false) -%}
+
+ {% if relation2.is_view %}
+ {% set from_results = run_query('show create view ' + relation1.render() )
%}
+ {% set to_results = run_query('show create view ' + relation2.render() ) %}
+ {% call statement('exchange_view_relation') %}
+ alter view {{ relation1 }} as {{ to_results[0]['Create
View'].split('AS',1)[1] }}
+ {% endcall %}
+ {% if is_drop_r1 %}
+ {% do doris__drop_relation(relation2) %}
+ {% else %}
+ {% call statement('exchange_view_relation') %}
+ alter view {{ relation2 }} as {{ from_results[0]['Create
View'].split('AS',1)[1] }}
+ {% endcall %}
+ {% endif %}
+ {% else %}
+ {% call statement('exchange_relation') %}
+ ALTER TABLE {{ relation1 }} REPLACE WITH TABLE `{{ relation2.table }}`
PROPERTIES('swap' = '{{not is_drop_r1}}');
+ {% endcall %}
+ {% endif %}
+
+{%- endmacro %}
{% macro doris__timestimp_id() -%}
{{ return( (modules.datetime.datetime.now() ~
"").replace('-','').replace(':','').replace('.','').replace(' ','') ) }}
@@ -166,16 +190,16 @@
WITH LABEL dbt_doris_label_{{doris__timestimp_id()}}
{% else %}
WITH LABEL dbt_doris_label_{{ lable_suffix_id }}
- {% endif %}
+ {% endif %}
{%- endmacro %}
{% macro doris__get_or_create_relation(database, schema, identifier, type) %}
{%- set target_relation = adapter.get_relation(database=database,
schema=schema, identifier=identifier) %}
-
+
{% if target_relation %}
{% do return([true, target_relation]) %}
{% endif %}
-
+
{%- set new_relation = api.Relation.create(
database=none,
schema=schema,
@@ -186,5 +210,5 @@
{% endmacro %}
{% macro catalog_source(catalog,database,table) -%}
- `{{catalog}}`.`{{database}}`.`{{table}}`
+ `{{catalog}}`.`{{database}}`.`{{table}}`
{%- endmacro %}
diff --git
a/extension/dbt-doris/dbt/include/doris/macros/materializations/incremental/incremental.sql
b/extension/dbt-doris/dbt/include/doris/macros/materializations/incremental/incremental.sql
index f15a9d0896..921e4b6bac 100644
---
a/extension/dbt-doris/dbt/include/doris/macros/materializations/incremental/incremental.sql
+++
b/extension/dbt-doris/dbt/include/doris/macros/materializations/incremental/incremental.sql
@@ -28,20 +28,25 @@
{#-- append or no unique key --#}
- {% if unique_key is none or strategy == 'append' %}
+
+ {% if not unique_key or strategy == 'append' %}
{#-- create table first --#}
{% if existing_relation is none %}
{% set build_sql = doris__create_table_as(False, target_relation,
sql) %}
{% elif existing_relation.is_view or full_refresh_mode %}
- {#-- backup data before drop old table #}
+ {#-- backup table is new table ,exchange table backup and old
table #}
{% set backup_identifier = existing_relation.identifier ~
"__dbt_backup" %}
{% set backup_relation =
existing_relation.incorporate(path={"identifier": backup_identifier}) %}
{% do adapter.drop_relation(backup_relation) %} {#-- likes 'drop
table if exists ... ' --#}
- {% do adapter.rename_relation(target_relation, backup_relation) %}
- {% set build_sql = doris__create_table_as(False, target_relation,
sql) %}
- {% do to_drop.append(backup_relation) %}
+ {% set run_sql = doris__create_table_as(False, backup_relation,
sql) %}
+ {% call statement("run_sql") %}
+ {{ run_sql }}
+ {% endcall %}
+ {% do exchange_relation(target_relation, backup_relation, True) %}
+ {% set build_sql = "show frontends" %}
{#-- append data --#}
{% else %}
+ {% do to_drop.append(tmp_relation) %}
{% do run_query(create_table_as(True, tmp_relation, sql)) %}
{% set build_sql = tmp_insert(tmp_relation, target_relation,
unique_key=none) %}
{% endif %}
@@ -52,13 +57,16 @@
{% set build_sql = doris__create_unique_table_as(False,
target_relation, sql) %}
{#-- insert data refresh --#}
{% elif existing_relation.is_view or full_refresh_mode %}
- {#-- backup data before drop old table #}
+ {#-- backup table is new table ,exchange table backup and old
table #}
{% set backup_identifier = existing_relation.identifier ~
"__dbt_backup" %}
{% set backup_relation =
existing_relation.incorporate(path={"identifier": backup_identifier}) %}
{% do adapter.drop_relation(backup_relation) %} {#-- likes 'drop
table if exists ... ' --#}
- {% do adapter.rename_relation(target_relation, backup_relation) %}
- {% set build_sql = doris__create_unique_table_as(False,
target_relation, sql) %}
- {% do to_drop.append(backup_relation) %}
+ {% set run_sql = doris__create_unique_table_as(False,
backup_relation, sql) %}
+ {% call statement("run_sql") %}
+ {{ run_sql }}
+ {% endcall %}
+ {% do exchange_relation(target_relation, backup_relation, True) %}
+ {% set build_sql = "show frontends" %}
{#-- append data --#}
{% else %}
{#-- check doris unique table --#}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
