This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 5fd9263e21b [HUDI-6333] Allow using manifest file directly to create a
table (#8898)
5fd9263e21b is described below
commit 5fd9263e21bc15dcf6ed97722975876ad36dd08a
Author: Jinpeng <[email protected]>
AuthorDate: Wed Jun 21 23:56:18 2023 -0400
[HUDI-6333] Allow using manifest file directly to create a table (#8898)
Co-authored-by: jp0317 <[email protected]>
---
.../hudi/gcp/bigquery/BigQuerySyncConfig.java | 16 +++++++-
.../apache/hudi/gcp/bigquery/BigQuerySyncTool.java | 35 +++++++++++++++--
.../gcp/bigquery/HoodieBigQuerySyncClient.java | 44 ++++++++++++++++++++++
.../hudi/gcp/bigquery/TestBigQuerySyncConfig.java | 3 ++
.../gcp/bigquery/TestBigQuerySyncToolArgs.java | 3 ++
.../hudi/sync/common/util/ManifestFileWriter.java | 15 ++++----
.../sync/common/util/TestManifestFileWriter.java | 9 +++--
7 files changed, 110 insertions(+), 15 deletions(-)
diff --git
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java
index fc1a02b22f3..1f99a57b550 100644
---
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java
+++
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncConfig.java
@@ -78,6 +78,14 @@ public class BigQuerySyncConfig extends HoodieSyncConfig
implements Serializable
.markAdvanced()
.withDocumentation("Name of the target table in BigQuery");
+ public static final ConfigProperty<Boolean>
BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE = ConfigProperty
+ .key("hoodie.gcp.bigquery.sync.use_bq_manifest_file")
+ .defaultValue(false)
+ .markAdvanced()
+ .withDocumentation("If true, generate a manifest file with data file
absolute paths and use BigQuery manifest file support to "
+ + "directly create one external table over the Hudi table. If false
(default), generate a manifest file with data file "
+ + "names and create two external tables and one view in BigQuery.
Query the view for the same results as querying the Hudi table");
+
public static final ConfigProperty<String> BIGQUERY_SYNC_SOURCE_URI =
ConfigProperty
.key("hoodie.gcp.bigquery.sync.source_uri")
.noDefaultValue()
@@ -136,9 +144,14 @@ public class BigQuerySyncConfig extends HoodieSyncConfig
implements Serializable
public String datasetName;
@Parameter(names = {"--dataset-location"}, description = "Location of the
target dataset in BigQuery", required = true)
public String datasetLocation;
+ @Parameter(names = {"--use-bq-manifest-file"}, description = "If true,
generate a manifest file with data file absolute paths and use "
+ + " BigQuery manifest file support to directly create one external
table over the Hudi table. If false (default), generate a manifest "
+ + " file with data file names and create two external tables and one
view in BigQuery. Query the view for the same results as querying "
+ + "the Hudi table")
+ public Boolean useBqManifestFile;
@Parameter(names = {"--source-uri"}, description = "Name of the source uri
gcs path of the table", required = true)
public String sourceUri;
- @Parameter(names = {"--source-uri-prefix"}, description = "Name of the
source uri gcs path prefix of the table", required = true)
+ @Parameter(names = {"--source-uri-prefix"}, description = "Name of the
source uri gcs path prefix of the table", required = false)
public String sourceUriPrefix;
public boolean isHelp() {
@@ -151,6 +164,7 @@ public class BigQuerySyncConfig extends HoodieSyncConfig
implements Serializable
props.setPropertyIfNonNull(BIGQUERY_SYNC_DATASET_NAME.key(),
datasetName);
props.setPropertyIfNonNull(BIGQUERY_SYNC_DATASET_LOCATION.key(),
datasetLocation);
props.setPropertyIfNonNull(BIGQUERY_SYNC_TABLE_NAME.key(),
hoodieSyncConfigParams.tableName);
+ props.setPropertyIfNonNull(BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE.key(),
useBqManifestFile);
props.setPropertyIfNonNull(BIGQUERY_SYNC_SOURCE_URI.key(), sourceUri);
props.setPropertyIfNonNull(BIGQUERY_SYNC_SOURCE_URI_PREFIX.key(),
sourceUriPrefix);
props.setPropertyIfNonNull(BIGQUERY_SYNC_SYNC_BASE_PATH.key(),
hoodieSyncConfigParams.basePath);
diff --git
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncTool.java
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncTool.java
index 042fbee210b..e0f5ace6c3a 100644
--- a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncTool.java
+++ b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySyncTool.java
@@ -33,6 +33,7 @@ import java.util.Properties;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_ASSUME_DATE_PARTITIONING;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_DATASET_NAME;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_PARTITION_FIELDS;
+import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI_PREFIX;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SYNC_BASE_PATH;
@@ -82,6 +83,14 @@ public class BigQuerySyncTool extends HoodieSyncTool {
}
}
+ private boolean tableExists(HoodieBigQuerySyncClient bqSyncClient, String
tableName) {
+ if (bqSyncClient.tableExists(tableName)) {
+ LOG.info(tableName + " already exists");
+ return true;
+ }
+ return false;
+ }
+
private void syncCoWTable(HoodieBigQuerySyncClient bqSyncClient) {
ValidationUtils.checkState(bqSyncClient.getTableType() ==
HoodieTableType.COPY_ON_WRITE);
LOG.info("Sync hoodie table " + snapshotViewName + " at base path " +
bqSyncClient.getBasePath());
@@ -96,13 +105,30 @@ public class BigQuerySyncTool extends HoodieSyncTool {
.setUseFileListingFromMetadata(config.getBoolean(BIGQUERY_SYNC_USE_FILE_LISTING_FROM_METADATA))
.setAssumeDatePartitioning(config.getBoolean(BIGQUERY_SYNC_ASSUME_DATE_PARTITIONING))
.build();
+
+ if (config.getBoolean(BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE)) {
+ manifestFileWriter.writeManifestFile(true);
+
+ if (!tableExists(bqSyncClient, tableName)) {
+ bqSyncClient.createTableUsingBqManifestFile(
+ tableName,
+ manifestFileWriter.getManifestSourceUri(true),
+ config.getString(BIGQUERY_SYNC_SOURCE_URI_PREFIX));
+ LOG.info("Completed table " + tableName + " creation using the
manifest file");
+ }
+
+ LOG.info("Sync table complete for " + tableName);
+ return;
+ }
+
manifestFileWriter.writeManifestFile(false);
- if (!bqSyncClient.tableExists(manifestTableName)) {
- bqSyncClient.createManifestTable(manifestTableName,
manifestFileWriter.getManifestSourceUri());
+ if (!tableExists(bqSyncClient, manifestTableName)) {
+ bqSyncClient.createManifestTable(manifestTableName,
manifestFileWriter.getManifestSourceUri(false));
LOG.info("Manifest table creation complete for " + manifestTableName);
}
- if (!bqSyncClient.tableExists(versionsTableName)) {
+
+ if (!tableExists(bqSyncClient, versionsTableName)) {
bqSyncClient.createVersionsTable(
versionsTableName,
config.getString(BIGQUERY_SYNC_SOURCE_URI),
@@ -110,7 +136,8 @@ public class BigQuerySyncTool extends HoodieSyncTool {
config.getSplitStrings(BIGQUERY_SYNC_PARTITION_FIELDS));
LOG.info("Versions table creation complete for " + versionsTableName);
}
- if (!bqSyncClient.tableExists(snapshotViewName)) {
+
+ if (!tableExists(bqSyncClient, snapshotViewName)) {
bqSyncClient.createSnapshotView(snapshotViewName, versionsTableName,
manifestTableName);
LOG.info("Snapshot view creation complete for " + snapshotViewName);
}
diff --git
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java
index 22d270f6a0a..17990e76929 100644
---
a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java
+++
b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java
@@ -19,6 +19,7 @@
package org.apache.hudi.gcp.bigquery;
+import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.sync.common.HoodieSyncClient;
import com.google.cloud.bigquery.BigQuery;
@@ -31,6 +32,10 @@ import com.google.cloud.bigquery.ExternalTableDefinition;
import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.FormatOptions;
import com.google.cloud.bigquery.HivePartitioningOptions;
+import com.google.cloud.bigquery.Job;
+import com.google.cloud.bigquery.JobId;
+import com.google.cloud.bigquery.JobInfo;
+import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.Schema;
import com.google.cloud.bigquery.StandardSQLTypeName;
import com.google.cloud.bigquery.Table;
@@ -43,6 +48,7 @@ import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_DATASET_LOCATION;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_DATASET_NAME;
@@ -78,6 +84,44 @@ public class HoodieBigQuerySyncClient extends
HoodieSyncClient {
}
}
+ public void createTableUsingBqManifestFile(String tableName, String
bqManifestFileUri, String sourceUriPrefix) {
+ try {
+ String withClauses = "";
+ String extraOptions = "";
+ if (!StringUtils.isNullOrEmpty(sourceUriPrefix)) {
+ withClauses = "WITH PARTITION COLUMNS";
+ extraOptions = String.format("hive_partition_uri_prefix=\"%s\",",
sourceUriPrefix);
+ }
+ String query =
+ String.format(
+ "CREATE EXTERNAL TABLE `%s.%s` %s OPTIONS (%s "
+ + "uris=[\"%s\"], format=\"PARQUET\",
file_set_spec_type=\"NEW_LINE_DELIMITED_MANIFEST\")",
+ datasetName,
+ tableName,
+ withClauses,
+ extraOptions,
+ bqManifestFileUri);
+
+ QueryJobConfiguration queryConfig =
QueryJobConfiguration.newBuilder(query)
+ .setUseLegacySql(false)
+ .build();
+ JobId jobId = JobId.of(UUID.randomUUID().toString());
+ Job queryJob =
bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build());
+
+ queryJob = queryJob.waitFor();
+
+ if (queryJob == null) {
+ LOG.error("Job for table creation no longer exists");
+ } else if (queryJob.getStatus().getError() != null) {
+ LOG.error("Job for table creation failed: " +
queryJob.getStatus().getError().toString());
+ } else {
+ LOG.info("External table created using manifest file.");
+ }
+ } catch (InterruptedException | BigQueryException e) {
+ throw new HoodieBigQuerySyncException("Failed to create external table
using manifest file. ", e);
+ }
+ }
+
public void createManifestTable(String tableName, String sourceUri) {
try {
TableId tableId = TableId.of(projectId, datasetName, tableName);
diff --git
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncConfig.java
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncConfig.java
index 6d96e9b9123..bffd9a6485c 100644
---
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncConfig.java
+++
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncConfig.java
@@ -34,6 +34,7 @@ import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_DATA
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_DATASET_NAME;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_PARTITION_FIELDS;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_PROJECT_ID;
+import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI_PREFIX;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SYNC_BASE_PATH;
@@ -51,6 +52,7 @@ public class TestBigQuerySyncConfig {
props.setProperty(BIGQUERY_SYNC_DATASET_NAME.key(), "foodataset");
props.setProperty(BIGQUERY_SYNC_DATASET_LOCATION.key(), "US");
props.setProperty(BIGQUERY_SYNC_TABLE_NAME.key(), "footable");
+ props.setProperty(BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE.key(), "true");
props.setProperty(BIGQUERY_SYNC_SOURCE_URI.key(),
"gs://test-bucket/dwh/table_name/dt=*");
props.setProperty(BIGQUERY_SYNC_SOURCE_URI_PREFIX.key(),
"gs://test-bucket/dwh/table_name/");
props.setProperty(BIGQUERY_SYNC_SYNC_BASE_PATH.key(),
"gs://test-bucket/dwh/table_name");
@@ -62,6 +64,7 @@ public class TestBigQuerySyncConfig {
assertEquals("foodataset",
syncConfig.getString(BIGQUERY_SYNC_DATASET_NAME));
assertEquals("US", syncConfig.getString(BIGQUERY_SYNC_DATASET_LOCATION));
assertEquals("footable", syncConfig.getString(BIGQUERY_SYNC_TABLE_NAME));
+ assertEquals(true,
syncConfig.getBoolean(BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE));
assertEquals("gs://test-bucket/dwh/table_name/dt=*",
syncConfig.getString(BIGQUERY_SYNC_SOURCE_URI));
assertEquals("gs://test-bucket/dwh/table_name/",
syncConfig.getString(BIGQUERY_SYNC_SOURCE_URI_PREFIX));
assertEquals("gs://test-bucket/dwh/table_name",
syncConfig.getString(BIGQUERY_SYNC_SYNC_BASE_PATH));
diff --git
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java
index 898358484d9..24981c4c64b 100644
---
a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java
+++
b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySyncToolArgs.java
@@ -28,6 +28,7 @@ import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_DATA
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_DATASET_NAME;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_PARTITION_FIELDS;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_PROJECT_ID;
+import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SOURCE_URI_PREFIX;
import static
org.apache.hudi.gcp.bigquery.BigQuerySyncConfig.BIGQUERY_SYNC_SYNC_BASE_PATH;
@@ -51,6 +52,7 @@ public class TestBigQuerySyncToolArgs {
"--source-uri-prefix", "gs://foobartable/",
"--base-path", "gs://foobartable",
"--partitioned-by", "year,month,day",
+ "--use-bq-manifest-file",
"--use-file-listing-from-metadata"
};
cmd.parse(args);
@@ -64,6 +66,7 @@ public class TestBigQuerySyncToolArgs {
assertEquals("gs://foobartable/",
props.getProperty(BIGQUERY_SYNC_SOURCE_URI_PREFIX.key()));
assertEquals("gs://foobartable",
props.getProperty(BIGQUERY_SYNC_SYNC_BASE_PATH.key()));
assertEquals("year,month,day",
props.getProperty(BIGQUERY_SYNC_PARTITION_FIELDS.key()));
+ assertEquals("true",
props.getProperty(BIGQUERY_SYNC_USE_BQ_MANIFEST_FILE.key()));
assertEquals("true",
props.getProperty(BIGQUERY_SYNC_USE_FILE_LISTING_FROM_METADATA.key()));
assertFalse(props.containsKey(BIGQUERY_SYNC_ASSUME_DATE_PARTITIONING.key()));
}
diff --git
a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
index 5203654d035..c078884efc8 100644
---
a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
+++
b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/ManifestFileWriter.java
@@ -43,6 +43,7 @@ import java.util.stream.Stream;
public class ManifestFileWriter {
public static final String MANIFEST_FOLDER_NAME = "manifest";
+ public static final String ABSOLUTE_PATH_MANIFEST_FOLDER_NAME =
"absolute-path-manifest";
public static final String MANIFEST_FILE_NAME = "latest-snapshot.csv";
private static final Logger LOG =
LoggerFactory.getLogger(ManifestFileWriter.class);
@@ -69,7 +70,7 @@ public class ManifestFileWriter {
} else {
LOG.info("Writing base file names to manifest file: " +
baseFiles.size());
}
- final Path manifestFilePath = getManifestFilePath();
+ final Path manifestFilePath = getManifestFilePath(useAbsolutePath);
try (FSDataOutputStream outputStream =
metaClient.getFs().create(manifestFilePath, true);
BufferedWriter writer = new BufferedWriter(new
OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) {
for (String f : baseFiles) {
@@ -101,16 +102,16 @@ public class ManifestFileWriter {
}
}
- public Path getManifestFolder() {
- return new Path(metaClient.getMetaPath(), MANIFEST_FOLDER_NAME);
+ public Path getManifestFolder(boolean useAbsolutePath) {
+ return new Path(metaClient.getMetaPath(), useAbsolutePath ?
ABSOLUTE_PATH_MANIFEST_FOLDER_NAME : MANIFEST_FOLDER_NAME);
}
- public Path getManifestFilePath() {
- return new Path(getManifestFolder(), MANIFEST_FILE_NAME);
+ public Path getManifestFilePath(boolean useAbsolutePath) {
+ return new Path(getManifestFolder(useAbsolutePath), MANIFEST_FILE_NAME);
}
- public String getManifestSourceUri() {
- return new Path(getManifestFolder(), "*").toUri().toString();
+ public String getManifestSourceUri(boolean useAbsolutePath) {
+ return new Path(getManifestFolder(useAbsolutePath),
"*").toUri().toString();
}
public static Builder builder() {
diff --git
a/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java
b/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java
index c1dffc8e4de..b01125853cb 100644
---
a/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java
+++
b/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestManifestFileWriter.java
@@ -59,7 +59,7 @@ public class TestManifestFileWriter extends
HoodieCommonTestHarness {
createTestDataForPartitionedTable(metaClient, 3);
ManifestFileWriter manifestFileWriter =
ManifestFileWriter.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).build();
manifestFileWriter.writeManifestFile(false);
- Path manifestFilePath = manifestFileWriter.getManifestFilePath();
+ Path manifestFilePath = manifestFileWriter.getManifestFilePath(false);
try (InputStream is = metaClient.getFs().open(manifestFilePath)) {
List<String> expectedLines = FileIOUtils.readAsUTFStringLines(is);
assertEquals(9, expectedLines.size(), "there should be 9 base files in
total; 3 per partition.");
@@ -73,7 +73,7 @@ public class TestManifestFileWriter extends
HoodieCommonTestHarness {
createTestDataForPartitionedTable(metaClient, 3);
ManifestFileWriter manifestFileWriter =
ManifestFileWriter.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).build();
manifestFileWriter.writeManifestFile(true);
- Path manifestFilePath = manifestFileWriter.getManifestFilePath();
+ Path manifestFilePath = manifestFileWriter.getManifestFilePath(true);
try (InputStream is = metaClient.getFs().open(manifestFilePath)) {
List<String> expectedLines = FileIOUtils.readAsUTFStringLines(is);
assertEquals(9, expectedLines.size(), "there should be 9 base files in
total; 3 per partition.");
@@ -93,7 +93,10 @@ public class TestManifestFileWriter extends
HoodieCommonTestHarness {
@Test
public void getManifestSourceUri() {
ManifestFileWriter manifestFileWriter =
ManifestFileWriter.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).build();
- String sourceUri = manifestFileWriter.getManifestSourceUri();
+ String sourceUri = manifestFileWriter.getManifestSourceUri(false);
assertEquals(new Path(basePath, ".hoodie/manifest/*").toUri().toString(),
sourceUri);
+
+ sourceUri = manifestFileWriter.getManifestSourceUri(true);
+ assertEquals(new Path(basePath,
".hoodie/absolute-path-manifest/*").toUri().toString(), sourceUri);
}
}
