This is an automated email from the ASF dual-hosted git repository.
zykkk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 020f238fbc [feature](dbt) read table columns from model config file
(#21831)
020f238fbc is described below
commit 020f238fbc7bdd02e1a597a61b5c4dc3c3bdddbc
Author: catpineapple <[email protected]>
AuthorDate: Mon Jul 17 15:45:12 2023 +0800
[feature](dbt) read table columns from model config file (#21831)
1、read table columns (datat_ype) from model config file
2、read table description(comment) from model config file
---
extension/dbt-doris/dbt/adapters/doris/impl.py | 31 ++++++++++++++++++++--
.../dbt/include/doris/macros/adapters/columns.sql | 14 +++++++++-
.../dbt/include/doris/macros/adapters/relation.sql | 5 ++++
.../materializations/table/create_table_as.sql | 16 +++++++++--
extension/dbt-doris/setup.py | 1 +
5 files changed, 62 insertions(+), 5 deletions(-)
diff --git a/extension/dbt-doris/dbt/adapters/doris/impl.py
b/extension/dbt-doris/dbt/adapters/doris/impl.py
index 49ae2abb5f..33431610aa 100644
--- a/extension/dbt-doris/dbt/adapters/doris/impl.py
+++ b/extension/dbt-doris/dbt/adapters/doris/impl.py
@@ -22,7 +22,20 @@ from dbt.adapters.sql import SQLAdapter
from concurrent.futures import Future
from enum import Enum
-from typing import Callable, Dict, List, Optional, Set, Tuple
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Iterable,
+ Iterator,
+ List,
+ Mapping,
+ Optional,
+ Set,
+ Tuple,
+ Type,
+ Union,
+)
import agate
import dbt.exceptions
@@ -186,4 +199,18 @@ class DorisAdapter(SQLAdapter):
# default. A lot of searching has lead me to believe that the
# '+ interval' syntax used in postgres/redshift is relatively common
# and might even be the SQL standard's intention.
- return f"{add_to} + interval {number} {interval}"
+ return f"{add_to} + interval {number} {interval}"
+
+ @classmethod
+ def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str,
Any]]) -> List:
+ rendered_column_constraints = []
+ for v in raw_columns.values():
+ col_name = cls.quote(v["name"]) if v.get("quote") else v["name"]
+ data_type = v.get('data_type')
+ if data_type is not None:
+ rendered_column_constraint = [f"cast(`{col_name}` as
{data_type}) as `{col_name}`"]
+ else:
+ rendered_column_constraint = [f"`{col_name}` as `{col_name}`"]
+ rendered_column_constraints.append("
".join(rendered_column_constraint))
+
+ return rendered_column_constraints
\ No newline at end of file
diff --git a/extension/dbt-doris/dbt/include/doris/macros/adapters/columns.sql
b/extension/dbt-doris/dbt/include/doris/macros/adapters/columns.sql
index 5eb54d9156..401334656e 100644
--- a/extension/dbt-doris/dbt/include/doris/macros/adapters/columns.sql
+++ b/extension/dbt-doris/dbt/include/doris/macros/adapters/columns.sql
@@ -32,4 +32,16 @@ where table_schema = '{{ relation.schema }}'
{% macro doris__alter_column_type(relation,column_name,new_column_type) -%}
'''Changes column name or data type'''
-{% endmacro %}
\ No newline at end of file
+{% endmacro %}
+
+{% macro table_columns_and_constraints() %}
+ {# loop through user_provided_columns to create DDL with data types and
constraints #}
+ {%- set raw_column_constraints =
adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}
+ {% for c in raw_column_constraints -%}
+ {{ c }}{{ "," if not loop.last or raw_model_constraints }}
+ {% endfor %}
+{% endmacro %}
+
+{% macro doris__get_table_columns_and_constraints() -%}
+ {{ return(table_columns_and_constraints()) }}
+{%- endmacro %}
\ No newline at end of file
diff --git a/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql
b/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql
index a75a9ce069..7c9f2ba57b 100644
--- a/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql
+++ b/extension/dbt-doris/dbt/include/doris/macros/adapters/relation.sql
@@ -58,6 +58,11 @@
{% endif %}
{%- endmacro %}
+{% macro doris__table_comment() -%}
+ {% set description = model.get('description', "") %}
+ COMMENT '{{description}}'
+{%- endmacro %}
+
{% macro doris__unique_key() -%}
{% set cols = config.get('unique_key', validator=validation.any[list,
basestring]) %}
diff --git
a/extension/dbt-doris/dbt/include/doris/macros/materializations/table/create_table_as.sql
b/extension/dbt-doris/dbt/include/doris/macros/materializations/table/create_table_as.sql
index a7ad5c325c..3b14c86586 100644
---
a/extension/dbt-doris/dbt/include/doris/macros/materializations/table/create_table_as.sql
+++
b/extension/dbt-doris/dbt/include/doris/macros/materializations/table/create_table_as.sql
@@ -24,9 +24,10 @@
{% endif %}
create table {{ table }}
{{ doris__duplicate_key() }}
+ {{ doris__table_comment()}}
{{ doris__partition_by() }}
{{ doris__distributed_by() }}
- {{ doris__properties() }} as {{ sql }};
+ {{ doris__properties() }} as {{ doris__table_colume_type(sql) }};
{%- endmacro %}
@@ -36,8 +37,19 @@
{{ sql_header if sql_header is not none }}
create table {{ table }}
{{ doris__unique_key() }}
+ {{ doris__table_comment()}}
{{ doris__partition_by() }}
{{ doris__distributed_by() }}
- {{ doris__properties() }} as {{ sql }};
+ {{ doris__properties() }} as {{ doris__table_colume_type(sql) }};
+{%- endmacro %}
+
+
+{% macro doris__table_colume_type(sql) -%}
+ {% set cols = model.get('columns') %}
+ {% if cols %}
+ select {{get_table_columns_and_constraints()}} from ({{sql}})
`_table_colume_type_name`
+ {% else %}
+ {{sql}}
+ {%- endif -%}
{%- endmacro %}
\ No newline at end of file
diff --git a/extension/dbt-doris/setup.py b/extension/dbt-doris/setup.py
index a4f6961005..62916d1021 100644
--- a/extension/dbt-doris/setup.py
+++ b/extension/dbt-doris/setup.py
@@ -39,6 +39,7 @@ setup(
install_requires=[
"dbt-core~={}".format(dbt_core_version),
"mysql-connector-python>=8.0.0,<8.1",
+ "urllib3~=1.0",
],
python_requires=">=3.7.2",
)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
