This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5184b31620 [feature](docker)Add new example MySQL-Flink-Doris Demo
(#20469)
5184b31620 is described below
commit 5184b31620f845da4c09a8ee3332f44cbd0dbb26
Author: FreeOnePlus <[email protected]>
AuthorDate: Tue Jun 6 15:23:11 2023 +0800
[feature](docker)Add new example MySQL-Flink-Doris Demo (#20469)
Add new example MySQL-Flink-Doris Demo
---
.../example/mysql-flink-doris/Readme.md | 23 ++++++
.../example/mysql-flink-doris/docker-compose.yaml | 93 ++++++++++++++++++++++
.../example/mysql-flink-doris/doris-init-sql.sql | 33 ++++++++
.../example/mysql-flink-doris/flink-init-sql.sql | 57 +++++++++++++
.../example/mysql-flink-doris/mysql-init-sql.sql | 32 ++++++++
5 files changed, 238 insertions(+)
diff --git
a/docker/runtime/docker-compose-demo/example/mysql-flink-doris/Readme.md
b/docker/runtime/docker-compose-demo/example/mysql-flink-doris/Readme.md
new file mode 100644
index 0000000000..2858515b57
--- /dev/null
+++ b/docker/runtime/docker-compose-demo/example/mysql-flink-doris/Readme.md
@@ -0,0 +1,23 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+# MySQL-Flink-Doris
+
+1. `docker-compose.yaml` 脚本中需要自行下载
`Flink-Doris-Connector`、`Flink-MySQL-Connector` 的 JAR 包，然后按目录进行映射。
+2. 当前版本里可以采网络桥接模式来配置网络，也可以使用 host 模式来进行网络配置，需要进行参数调整。
+3. 启动以后按照 MySQL-init、Doris-init、Flink-init 的顺序进行执行即可。
diff --git
a/docker/runtime/docker-compose-demo/example/mysql-flink-doris/docker-compose.yaml
b/docker/runtime/docker-compose-demo/example/mysql-flink-doris/docker-compose.yaml
new file mode 100644
index 0000000000..b0d06b8180
--- /dev/null
+++
b/docker/runtime/docker-compose-demo/example/mysql-flink-doris/docker-compose.yaml
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+version: "3"
+services:
+ mysql:
+ image: mysql:8.0.31
+ volumes:
+ - /data/mysql/:/var/lib/mysql
+ environment:
+ MYSQL_ROOT_PASSWORD: 123456
+ MYSQL_DATABASE: doris-test
+ MYSQL_PASSWORD: 123456
+ command:
+ --character-set-server=utf8
+ hostname: mysql
+ ports:
+ - "3306:3306"
+
+ jobmanager:
+ image: apache/flink:1.15-scala_2.12
+ volumes:
+ -
/data/flink/job/flink-doris-connector-1.15-1.2.1.jar:/opt/flink/lib/flink-doris-connector-1.15-1.2.1.jar
+ -
/data/flink/job/flink-sql-connector-mysql-cdc-2.2.1.jar:/opt/flink/lib/flink-sql-connector-mysql-cdc-2.2.1.jar
+ expose:
+ - "6123"
+ ports:
+ - "8081:8081"
+ command: jobmanager
+ environment:
+ - JOB_MANAGER_RPC_ADDRESS=jobmanager
+
+ taskmanager:
+ image: apache/flink:1.15-scala_2.12
+ volumes:
+ -
/data/flink/task/flink-doris-connector-1.15-1.2.1.jar:/opt/flink/lib/flink-doris-connector-1.15-1.2.1.jar
+ -
/data/flink/task/flink-sql-connector-mysql-cdc-2.2.1.jar:/opt/flink/lib/flink-sql-connector-mysql-cdc-2.2.1.jar
+ expose:
+ - "6121"
+ - "6122"
+ depends_on:
+ - jobmanager
+ command: taskmanager
+ links:
+ - "jobmanager:jobmanager"
+ environment:
+ - JOB_MANAGER_RPC_ADDRESS=jobmanager
+
+ fe:
+ image: apache/doris:1.2.2-fe-x86_64
+ volumes:
+ - /data/apache-doris/fe/doris-meta/:/opt/apache-doris/fe/doris-meta
+ - /data/apache-doris/fe/log/:/opt/apache-doris/fe/log
+ environment:
+ - FE_SERVERS=fe1:172.20.80.1:9010
+ - FE_ID=1
+ restart: on-failure
+ networks:
+ doris_net:
+ ipv4_address: 172.20.80.1
+
+ be:
+ image: apache/doris:1.2.2-be-x86_64
+ volumes:
+ - /data/apache-doris/be/storage/:/opt/apache-doris/be/storage
+ - /data/apache-doris/be/log/:/opt/apache-doris/be/log
+ environment:
+ - FE_SERVERS=fe1:172.20.80.1:9010
+ - BE_ADDR=172.20.80.2:9050
+ depends_on:
+ - fe
+ restart: on-failure
+ networks:
+ doris_net:
+ ipv4_address: 172.20.80.2
+networks:
+ doris_net:
+ ipam:
+ config:
+ - subnet: 172.20.80.0/24
diff --git
a/docker/runtime/docker-compose-demo/example/mysql-flink-doris/doris-init-sql.sql
b/docker/runtime/docker-compose-demo/example/mysql-flink-doris/doris-init-sql.sql
new file mode 100644
index 0000000000..296af49cb1
--- /dev/null
+++
b/docker/runtime/docker-compose-demo/example/mysql-flink-doris/doris-init-sql.sql
@@ -0,0 +1,33 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+CREATE DATABASE test;
+ use test;
+DROP TABLE IF EXISTS `mysql_order`;
+CREATE TABLE test.mysql_order
+(
+ order_id INT,
+ order_date DATETIME,
+ customer_name VARCHAR(255),
+ price DECIMAL(10, 5),
+ product_id INT,
+ order_status BOOLEAN
+)
+UNIQUE KEY(order_id)
+DISTRIBUTED BY HASH(order_id) BUCKETS 3
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git
a/docker/runtime/docker-compose-demo/example/mysql-flink-doris/flink-init-sql.sql
b/docker/runtime/docker-compose-demo/example/mysql-flink-doris/flink-init-sql.sql
new file mode 100644
index 0000000000..b502b3357b
--- /dev/null
+++
b/docker/runtime/docker-compose-demo/example/mysql-flink-doris/flink-init-sql.sql
@@ -0,0 +1,57 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+CREATE TABLE mysql_source (
+ order_id INT,
+ order_date TIMESTAMP(0),
+ customer_name STRING,
+ price DECIMAL(10, 5),
+ product_id INT,
+ order_status BOOLEAN,
+ PRIMARY KEY (order_id) NOT ENFORCED
+) WITH (
+ 'connector' = 'mysql-cdc',
+ 'hostname' = '${MySQL_CIP}',
+ 'port' = '3306',
+ 'username' = 'root',
+ 'password' = '123456',
+ 'database-name' = 'test',
+ 'table-name' = 'orders'
+ );
+
+SET execution.checkpointing.interval = 10s;
+
+CREATE TABLE doris_sink (
+ order_id INT,
+ order_date TIMESTAMP(0),
+ customer_name VARCHAR(255),
+ price DECIMAL(10, 5),
+ product_id INT,
+ order_status BOOLEAN
+)
+WITH (
+ 'connector' = 'doris',
+ 'fenodes' = '${HOST_ADDR}:8030',
+ 'table.identifier' = 'test.mysql_order',
+ 'username' = 'root',
+ 'password' = '',
+ 'sink.properties.format' = 'json',
+ 'sink.properties.read_json_by_line' = 'true',
+ 'sink.enable-delete' = 'true',
+ 'sink.label-prefix' = 'doris_label'
+ );
+
+INSERT INTO doris_sink SELECT
order_id,order_date,customer_name,price,product_id,order_status FROM
mysql_source;
diff --git
a/docker/runtime/docker-compose-demo/example/mysql-flink-doris/mysql-init-sql.sql
b/docker/runtime/docker-compose-demo/example/mysql-flink-doris/mysql-init-sql.sql
new file mode 100644
index 0000000000..e326a1e623
--- /dev/null
+++
b/docker/runtime/docker-compose-demo/example/mysql-flink-doris/mysql-init-sql.sql
@@ -0,0 +1,32 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+CREATE DATABASE test;
+ USE test;
+DROP TABLE IF EXISTS `orders`;
+CREATE TABLE orders (
+ order_id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ order_date DATETIME NOT NULL,
+ customer_name VARCHAR(255) NOT NULL,
+ price DECIMAL(10, 5) NOT NULL,
+ product_id INTEGER NOT NULL,
+ order_status BOOLEAN NOT NULL -- Whether order has
been placed
+) AUTO_INCREMENT = 10001;
+
+INSERT INTO orders
+VALUES (default, '2020-07-05 10:08:22', 'Jark', 50.50, 102, false),
+ (default, '2020-07-05 10:11:09', 'Sally', 15.00, 105, false),
+ (default, '2020-07-05 12:00:30', 'Edward', 25.25, 106, false);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
