This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 8ed8a20b6902b95a3e698a519822bdb0210a0a7f Author: steve-xi-awx <[email protected]> AuthorDate: Fri Mar 8 08:29:03 2024 +0800 [HUDI-7488] The BigQuerySyncTool can't work well when the hudi table schema changed (#10830) --- .../gcp/bigquery/HoodieBigQuerySyncClient.java | 25 ++++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java index 5a23a4079ae..32430b53329 100644 --- a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java +++ b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java @@ -44,6 +44,7 @@ import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.ViewDefinition; +import com.google.cloud.bigquery.StandardTableDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -198,16 +199,22 @@ public class HoodieBigQuerySyncClient extends HoodieSyncClient { LOG.info("No table update is needed."); return; // No need to update schema. } - ExternalTableDefinition.Builder builder = definition.toBuilder(); - builder.setSchema(finalSchema); - builder.setAutodetect(false); - if (definition.getHivePartitioningOptions() != null) { - builder.setHivePartitioningOptions(definition.getHivePartitioningOptions().toBuilder().setRequirePartitionFilter(requirePartitionFilter).build()); + if (!StringUtils.isNullOrEmpty(bigLakeConnectionId)) { + Table updatedTable = + existingTable.toBuilder().setDefinition(StandardTableDefinition.of(finalSchema)).build(); + updatedTable.update(); + } else { + ExternalTableDefinition.Builder builder = definition.toBuilder(); + builder.setSchema(finalSchema); + builder.setAutodetect(false); + if (definition.getHivePartitioningOptions() != null) { + builder.setHivePartitioningOptions(definition.getHivePartitioningOptions().toBuilder().setRequirePartitionFilter(requirePartitionFilter).build()); + } + Table updatedTable = existingTable.toBuilder() + .setDefinition(builder.build()) + .build(); + bigquery.update(updatedTable); } - Table updatedTable = existingTable.toBuilder() - .setDefinition(builder.build()) - .build(); - bigquery.update(updatedTable); } public void createVersionsTable(String tableName, String sourceUri, String sourceUriPrefix, List<String> partitionFields) {
