chenbodeng719 commented on issue #8279:
URL: https://github.com/apache/hudi/issues/8279#issuecomment-1492207358
- bulk insert conf
```
CREATE TABLE hbase2hudi_sink(
uid STRING PRIMARY KEY NOT ENFORCED,
oridata STRING,
update_time TIMESTAMP_LTZ(3)
) WITH (
'table.type' = 'MERGE_ON_READ',
'connector' = 'hudi',
'path' = '%s',
'write.operation' = 'bulk_insert',
'precombine.field' = 'update_time',
'write.tasks' = '256'
)
```
- build index
```
'index.bootstrap.enabled' = 'true'
```
- restart after build index and catch up with the consumer
```
CREATE TABLE hudi_sink(
uid STRING PRIMARY KEY NOT ENFORCED,
oridata STRING,
update_time TIMESTAMP_LTZ(3)
) WITH (
'table.type' = 'MERGE_ON_READ',
'connector' = 'hudi',
'path' = '%s',
'write.operation' = 'upsert',
'precombine.field' = 'update_time',
'payload.class' =
'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
'write.tasks' = '256',
'hoodie.index.type' = 'BUCKET',
'hoodie.bucket.index.hash.field' = 'uid',
'hoodie.bucket.index.num.buckets' = '256',
'index.bootstrap.enabled' = 'false'
)
```
- reduce cluster conf
```
CREATE TABLE hudi_sink(
uid STRING PRIMARY KEY NOT ENFORCED,
oridata STRING,
update_time TIMESTAMP_LTZ(3)
) WITH (
'table.type' = 'MERGE_ON_READ',
'connector' = 'hudi',
'path' = '%s',
'write.operation' = 'upsert',
'precombine.field' = 'update_time',
'payload.class' =
'org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload',
'write.tasks' = '50',
'hoodie.index.type' = 'BUCKET',
'hoodie.bucket.index.hash.field' = 'uid',
'hoodie.bucket.index.num.buckets' = '256',
'index.bootstrap.enabled' = 'false'
)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
