This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new b34a204 Fixing small file handling, inline compaction defaults
b34a204 is described below
commit b34a204a527a156406908686e54484a0c3d8a3d7
Author: Vinoth Chandar <[email protected]>
AuthorDate: Tue Mar 12 15:59:41 2019 -0700
Fixing small file handling, inline compaction defaults
- Small file limit is now 100MB by default
- Turned on inline compaction by default for MOR
- Changes take effect on DataSource and DeltaStreamer
---
.../src/main/java/com/uber/hoodie/HoodieWriteClient.java | 8 +++-----
.../java/com/uber/hoodie/config/HoodieCompactionConfig.java | 4 ++--
.../src/main/java/com/uber/hoodie/DataSourceUtils.java | 7 +++++++
hoodie-spark/src/test/scala/DataSourceTest.scala | 3 ++-
.../hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java | 10 ++++++++--
5 files changed, 22 insertions(+), 10 deletions(-)
diff --git a/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
b/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
index 1f584de..632c13d 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
@@ -528,8 +528,7 @@ public class HoodieWriteClient<T extends
HoodieRecordPayload> implements Seriali
try {
activeTimeline.saveAsComplete(new HoodieInstant(true, actionType,
commitTime),
Optional.of(metadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
- // Save was a success
- // Do a inline compaction if enabled
+ // Save was a success & Do a inline compaction if enabled
if (config.isInlineCompaction()) {
metadata.addMetadata(HoodieCompactionConfig.INLINE_COMPACT_PROP,
"true");
forceCompact(extraMetadata);
@@ -1103,7 +1102,7 @@ public class HoodieWriteClient<T extends
HoodieRecordPayload> implements Seriali
HoodieTimeline.compareTimestamps(instant.getTimestamp(),
instantTime,
HoodieTimeline.GREATER_OR_EQUAL)).collect(Collectors.toList());
Preconditions.checkArgument(conflictingInstants.isEmpty(),
- "Following instants have timestamps >= compactionInstant. Instants :"
+ "Following instants have timestamps >= compactionInstant (" +
instantTime + ") Instants :"
+ conflictingInstants);
HoodieTable<T> table = HoodieTable.getHoodieTable(metaClient, config, jsc);
HoodieCompactionPlan workload = table.scheduleCompaction(jsc, instantTime);
@@ -1343,8 +1342,7 @@ public class HoodieWriteClient<T extends
HoodieRecordPayload> implements Seriali
}
/**
- * Performs a compaction operation on a dataset. WARNING: Compaction
operation cannot be executed
- * asynchronously. Please always use this serially before or after an
insert/upsert action.
+ * Performs a compaction operation on a dataset, serially before or after an
insert/upsert action.
*/
private Optional<String> forceCompact(Optional<Map<String, String>>
extraMetadata) throws IOException {
Optional<String> compactionInstantTimeOpt =
scheduleCompaction(extraMetadata);
diff --git
a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java
b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java
index 2cde401..95e0c9b 100644
---
a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java
+++
b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java
@@ -47,8 +47,8 @@ public class HoodieCompactionConfig extends
DefaultHoodieConfig {
public static final String MIN_COMMITS_TO_KEEP_PROP =
"hoodie.keep.min.commits";
// Upsert uses this file size to compact new data onto existing files..
public static final String PARQUET_SMALL_FILE_LIMIT_BYTES =
"hoodie.parquet.small.file.limit";
- // Turned off by default
- public static final String DEFAULT_PARQUET_SMALL_FILE_LIMIT_BYTES =
String.valueOf(0);
+ // By default, treat any file <= 100MB as a small file.
+ public static final String DEFAULT_PARQUET_SMALL_FILE_LIMIT_BYTES =
String.valueOf(104857600);
/**
* Configs related to specific table types
**/
diff --git a/hoodie-spark/src/main/java/com/uber/hoodie/DataSourceUtils.java
b/hoodie-spark/src/main/java/com/uber/hoodie/DataSourceUtils.java
index 429c439..28811bb 100644
--- a/hoodie-spark/src/main/java/com/uber/hoodie/DataSourceUtils.java
+++ b/hoodie-spark/src/main/java/com/uber/hoodie/DataSourceUtils.java
@@ -126,6 +126,12 @@ public class DataSourceUtils {
public static HoodieWriteClient createHoodieClient(JavaSparkContext jssc,
String schemaStr,
String basePath, String tblName, Map<String, String> parameters) throws
Exception {
+
+ // inline compaction is on by default for MOR
+ boolean inlineCompact =
parameters.containsKey(DataSourceWriteOptions.STORAGE_TYPE_OPT_KEY())
+ &&
parameters.get(DataSourceWriteOptions.STORAGE_TYPE_OPT_KEY()).equals(DataSourceWriteOptions
+ .MOR_STORAGE_TYPE_OPT_VAL());
+
HoodieWriteConfig writeConfig =
HoodieWriteConfig.newBuilder().combineInput(true, true)
.withPath(basePath).withAutoCommit(false)
.withSchema(schemaStr).forTable(tblName).withIndexConfig(
@@ -134,6 +140,7 @@ public class DataSourceUtils {
.withPayloadClass(parameters.get(
DataSourceWriteOptions
.PAYLOAD_CLASS_OPT_KEY()))
+ .withInlineCompaction(inlineCompact)
.build())
// override above with Hoodie configs specified as options.
.withProps(parameters).build();
diff --git a/hoodie-spark/src/test/scala/DataSourceTest.scala
b/hoodie-spark/src/test/scala/DataSourceTest.scala
index 42def5b..1520f2f 100644
--- a/hoodie-spark/src/test/scala/DataSourceTest.scala
+++ b/hoodie-spark/src/test/scala/DataSourceTest.scala
@@ -29,9 +29,9 @@ import org.junit.{Before, Test}
import org.scalatest.junit.AssertionsForJUnit
import scala.collection.JavaConversions._
+import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, Future}
-import scala.concurrent.ExecutionContext.Implicits.global
/**
* Basic tests on the spark datasource
@@ -131,6 +131,7 @@ class DataSourceTest extends AssertionsForJUnit {
val inputDF1: Dataset[Row] =
spark.read.json(spark.sparkContext.parallelize(records1, 2))
inputDF1.write.format("com.uber.hoodie")
.options(commonOpts)
+ .option("hoodie.compact.inline", "false") // else fails due to
compaction & deltacommit instant times being same
.option(DataSourceWriteOptions.OPERATION_OPT_KEY,
DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL)
.option(DataSourceWriteOptions.STORAGE_TYPE_OPT_KEY,
DataSourceWriteOptions.MOR_STORAGE_TYPE_OPT_VAL)
.mode(SaveMode.Overwrite)
diff --git
a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java
b/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java
index 665fcff..0ad10de 100644
---
a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java
+++
b/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java
@@ -36,6 +36,7 @@ import com.uber.hoodie.WriteStatus;
import com.uber.hoodie.common.model.HoodieCommitMetadata;
import com.uber.hoodie.common.model.HoodieRecord;
import com.uber.hoodie.common.model.HoodieRecordPayload;
+import com.uber.hoodie.common.model.HoodieTableType;
import com.uber.hoodie.common.table.HoodieTableMetaClient;
import com.uber.hoodie.common.table.HoodieTimeline;
import com.uber.hoodie.common.table.timeline.HoodieInstant;
@@ -323,17 +324,22 @@ public class HoodieDeltaStreamer implements Serializable {
}
}
- private HoodieWriteConfig getHoodieClientConfig(SchemaProvider
schemaProvider) throws Exception {
+ private HoodieWriteConfig getHoodieClientConfig(SchemaProvider
schemaProvider) {
HoodieWriteConfig.Builder builder =
HoodieWriteConfig.newBuilder().combineInput(true,
true).withPath(cfg.targetBasePath)
.withAutoCommit(false)
-
.withCompactionConfig(HoodieCompactionConfig.newBuilder().withPayloadClass(cfg.payloadClassName).build())
+ .withCompactionConfig(HoodieCompactionConfig.newBuilder()
+ .withPayloadClass(cfg.payloadClassName)
+ // turn on inline compaction by default, for MOR tables
+ .withInlineCompaction(HoodieTableType.valueOf(cfg.storageType)
== HoodieTableType.MERGE_ON_READ)
+ .build())
.forTable(cfg.targetTableName)
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build())
.withProps(props);
if (null != schemaProvider) {
builder =
builder.withSchema(schemaProvider.getTargetSchema().toString());
}
+
return builder.build();
}