[
https://issues.apache.org/jira/browse/HUDI-5100?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5100:
---------------------------
Description:
Flink batch is written to hudi. When the number of slots is less than
write.tasks, it cannot be written.
sql:
{code:java}
flink configuration:
configuration.setString( "taskmanager.numberOfTaskSlots", "2");
configuration.setString("execution.runtime-mode", "BATCH");
CREATE CATALOG myhudi WITH(
'type' = 'hudi',
'default-database' = 'default',
'catalog.path' = '/user/hdpu/warehouse',
'mode' = 'hms',
'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
'table-prop.connector' = 'hudi',
'table-prop.table.type' = 'MERGE_ON_READ',
'table-prop.compaction.tasks' = '4',
'table-prop.write.tasks' = '4',
'table-prop.index.bootstrap.enabled' = 'true',
'table-prop.hive_sync.skip_ro_suffix' = 'true',
'table-prop.compaction.delta_commits' = '1',
'table-prop.compaction.async.enabled' = 'true',
'table-prop.changelog.enabled' = 'true',
'table-prop.index.type' = 'BUCKET',
'table-prop.index.global.enabled' = 'true',
'table-prop.read.utc-timezone' = 'false'
);
create table if not exists myhudi.test_hudi3.hudi_datagen_incre_c1
(id bigint not null, name string,ts3 timestamp(3)
,PRIMARY KEY (`id`) NOT ENFORCED
)
with('write.tasks' = 4)
;
show create table myhudi.test_hudi3.hudi_datagen_incre_c1;
insert into myhudi.test_hudi3.hudi_datagen_incre_c1
values
(1, 'cc12' ,12);{code}
!image-2022-10-27-10-24-14-374.png!
!image-2022-10-27-10-29-17-593.png!
was:
Flink batch is written to hudi. When the number of slots is less than
write.tasks, it cannot be written.
sql:
{code:java}
flink configuration:
configuration.setString( "taskmanager.numberOfTaskSlots", "2");
configuration.setString("execution.runtime-mode", "BATCH");
CREATE TABLE datagen (
id INT,
name string,
ts3 timestamp(3)
) WITH (
'connector' = 'datagen',
'rows-per-second'='10',
'fields.id.kind'='sequence',
'fields.id.start'='1',
'fields.id.end'='2000',
'number-of-rows' = '2000'
);
CREATE CATALOG myhudi WITH(
'type' = 'hudi',
'default-database' = 'default',
'catalog.path' = '/user/hdpu/warehouse',
'mode' = 'hms',
'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
'table-prop.connector' = 'hudi',
'table-prop.table.type' = 'MERGE_ON_READ',
'table-prop.compaction.tasks' = '4',
'table-prop.write.tasks' = '4',
'table-prop.index.bootstrap.enabled' = 'true',
'table-prop.hive_sync.skip_ro_suffix' = 'true',
'table-prop.compaction.delta_commits' = '1',
'table-prop.compaction.async.enabled' = 'true',
'table-prop.changelog.enabled' = 'true',
'table-prop.index.type' = 'BUCKET',
'table-prop.index.global.enabled' = 'true',
'table-prop.read.utc-timezone' = 'false'
);
create table if not exists myhudi.test_hudi3.hudi_datagen_incre_c1
(id bigint not null, name string,ts3 timestamp(3)
,PRIMARY KEY (`id`) NOT ENFORCED
)
with('write.tasks' = 4)
;
show create table myhudi.test_hudi3.hudi_datagen_incre_c1;
insert into myhudi.test_hudi3.hudi_datagen_incre_c1
select id,name,ts3
from datagen; {code}
!image-2022-10-27-10-24-14-374.png!
!image-2022-10-27-10-29-17-593.png!
> Flink batch is written to hudi. When the number of slots is less than
> write.tasks, it cannot be written
> -------------------------------------------------------------------------------------------------------
>
> Key: HUDI-5100
> URL: https://issues.apache.org/jira/browse/HUDI-5100
> Project: Apache Hudi
> Issue Type: Improvement
> Components: flink-sql
> Reporter: waywtdcc
> Priority: Major
> Labels: pull-request-available
> Fix For: 0.13.0
>
> Attachments: image-2022-10-27-10-24-14-374.png,
> image-2022-10-27-10-29-17-593.png
>
>
> Flink batch is written to hudi. When the number of slots is less than
> write.tasks, it cannot be written.
>
> sql:
> {code:java}
> flink configuration:
> configuration.setString( "taskmanager.numberOfTaskSlots", "2");
> configuration.setString("execution.runtime-mode", "BATCH");
>
> CREATE CATALOG myhudi WITH(
> 'type' = 'hudi',
> 'default-database' = 'default',
> 'catalog.path' = '/user/hdpu/warehouse',
> 'mode' = 'hms',
> 'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
> 'table-prop.connector' = 'hudi',
> 'table-prop.table.type' = 'MERGE_ON_READ',
> 'table-prop.compaction.tasks' = '4',
> 'table-prop.write.tasks' = '4',
> 'table-prop.index.bootstrap.enabled' = 'true',
> 'table-prop.hive_sync.skip_ro_suffix' = 'true',
> 'table-prop.compaction.delta_commits' = '1',
> 'table-prop.compaction.async.enabled' = 'true',
> 'table-prop.changelog.enabled' = 'true',
> 'table-prop.index.type' = 'BUCKET',
> 'table-prop.index.global.enabled' = 'true',
> 'table-prop.read.utc-timezone' = 'false'
> );
> create table if not exists myhudi.test_hudi3.hudi_datagen_incre_c1
> (id bigint not null, name string,ts3 timestamp(3)
> ,PRIMARY KEY (`id`) NOT ENFORCED
> )
> with('write.tasks' = 4)
> ;
> show create table myhudi.test_hudi3.hudi_datagen_incre_c1;
> insert into myhudi.test_hudi3.hudi_datagen_incre_c1
> values
> (1, 'cc12' ,12);{code}
>
> !image-2022-10-27-10-24-14-374.png!
> !image-2022-10-27-10-29-17-593.png!
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
