This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a1ba9728310 [HUDI-7414] Remove redundant base path config in BQ sync
(#11395)
a1ba9728310 is described below
commit a1ba97283103746273168905208db9e94d77710d
Author: Shiyan Xu <[email protected]>
AuthorDate: Wed Jun 5 03:54:58 2024 -0500
[HUDI-7414] Remove redundant base path config in BQ sync (#11395)
---
.../java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java | 8 --------
.../java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncConfig.java | 3 ---
.../org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java | 2 --
3 files changed, 13 deletions(-)
diff --git
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java
index ed8895ca217..c2532dca7e9 100644
---
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java
+++
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java
@@ -100,13 +100,6 @@ public class BigQuerySyncConfig extends HoodieSyncConfig
implements Serializable
.markAdvanced()
.withDocumentation("Name of the source uri gcs path prefix of the
table");
- public static final ConfigProperty<String> BIGQUERY_SYNC_SYNC_BASE_PATH =
ConfigProperty
- .key("hoodie.gcp.bigquery.sync.base_path")
- .noDefaultValue()
- .withInferFunction(cfg ->
Option.ofNullable(cfg.getString(META_SYNC_BASE_PATH)))
- .markAdvanced()
- .withDocumentation("Base path of the hoodie table to sync");
-
public static final ConfigProperty<String> BIGQUERY_SYNC_PARTITION_FIELDS =
ConfigProperty
.key("hoodie.gcp.bigquery.sync.partition_fields")
.noDefaultValue()
@@ -179,7 +172,6 @@ public class BigQuerySyncConfig extends HoodieSyncConfig
implements Serializable
props.setPropertyIfNonNull(BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE.key(),
useBqManifestFile);
props.setPropertyIfNonNull(BIGQUERY_SYNC_SOURCE_URI.key(), sourceUri);
props.setPropertyIfNonNull(BIGQUERY_SYNC_SOURCE_URI_PREFIX.key(),
sourceUriPrefix);
- props.setPropertyIfNonNull(BIGQUERY_SYNC_SYNC_BASE_PATH.key(),
hoodieSyncConfigParams.basePath);
props.setPropertyIfNonNull(BIGQUERY_SYNC_PARTITION_FIELDS.key(),
StringUtils.join(",", hoodieSyncConfigParams.partitionFields));
props.setPropertyIfNonNull(BIGQUERY_SYNC_USE_FILE_LISTING_FROM_METADATA.key(),
hoodieSyncConfigParams.useFileListingFromMetadata);
props.setPropertyIfNonNull(BIGQUERY_SYNC_BIG_LAKE_CONNECTION_ID.key(),
bigLakeConnectionId);
diff --git
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncConfig.java
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncConfig.java
index 2c17749158f..31f49d9a342 100644
---
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncConfig.java
+++
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncConfig.java
@@ -35,7 +35,6 @@ import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_PART
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_PROJECT_ID;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI_PREFIX;
-import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SYNC_BASE_PATH;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_TABLE_NAME;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_USE_FILE_LISTING_FROM_METADATA;
@@ -54,7 +53,6 @@ public class TestBigQuerySyncConfig {
props.setProperty(BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE.key(), "true");
props.setProperty(BIGQUERY_SYNC_SOURCE_URI.key(),
"gs://test-bucket/dwh/table_name/dt=*");
props.setProperty(BIGQUERY_SYNC_SOURCE_URI_PREFIX.key(),
"gs://test-bucket/dwh/table_name/");
- props.setProperty(BIGQUERY_SYNC_SYNC_BASE_PATH.key(),
"gs://test-bucket/dwh/table_name");
props.setProperty(BIGQUERY_SYNC_PARTITION_FIELDS.key(), "a,b");
props.setProperty(BIGQUERY_SYNC_USE_FILE_LISTING_FROM_METADATA.key(),
"true");
BigQuerySyncConfig syncConfig = new BigQuerySyncConfig(props);
@@ -65,7 +63,6 @@ public class TestBigQuerySyncConfig {
assertEquals(true,
syncConfig.getBoolean(BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE));
assertEquals("gs://test-bucket/dwh/table_name/dt=*",
syncConfig.getString(BIGQUERY_SYNC_SOURCE_URI));
assertEquals("gs://test-bucket/dwh/table_name/",
syncConfig.getString(BIGQUERY_SYNC_SOURCE_URI_PREFIX));
- assertEquals("gs://test-bucket/dwh/table_name",
syncConfig.getString(BIGQUERY_SYNC_SYNC_BASE_PATH));
assertEquals(Arrays.asList("a", "b"),
syncConfig.getSplitStrings(BIGQUERY_SYNC_PARTITION_FIELDS));
assertEquals(true,
syncConfig.getBoolean(BIGQUERY_SYNC_USE_FILE_LISTING_FROM_METADATA));
}
diff --git
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java
index 3f8ee6b9966..bfda76eb706 100644
---
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java
+++
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java
@@ -32,7 +32,6 @@ import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_REQU
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI_PREFIX;
-import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SYNC_BASE_PATH;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_TABLE_NAME;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_USE_FILE_LISTING_FROM_METADATA;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -66,7 +65,6 @@ public class TestBigQuerySyncToolArgs {
assertEquals("foobartable",
props.getProperty(BIGQUERY_SYNC_TABLE_NAME.key()));
assertEquals("gs://foobartable/year=*",
props.getProperty(BIGQUERY_SYNC_SOURCE_URI.key()));
assertEquals("gs://foobartable/",
props.getProperty(BIGQUERY_SYNC_SOURCE_URI_PREFIX.key()));
- assertEquals("gs://foobartable",
props.getProperty(BIGQUERY_SYNC_SYNC_BASE_PATH.key()));
assertEquals("year,month,day",
props.getProperty(BIGQUERY_SYNC_PARTITION_FIELDS.key()));
assertEquals("true",
props.getProperty(BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE.key()));
assertEquals("true",
props.getProperty(BIGQUERY_SYNC_USE_FILE_LISTING_FROM_METADATA.key()));
