This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a9b64da allow segment name postfix for SegmentProcessorFramework
(#8230)
a9b64da is described below
commit a9b64da7e7a97dfa2d2099b70dcdfdb6b869ece4
Author: Xiaobing <[email protected]>
AuthorDate: Fri Feb 18 21:54:04 2022 -0800
allow segment name postfix for SegmentProcessorFramework (#8230)
Allow to set segment name postfix when using SegmentProcessorFramework,
right now that's hard coded as null.
Meanwhile, fix a bug in NormalizedDate name generator, right now it doesn't
join postfix into segment name.
---
.../apache/pinot/core/common/MinionConstants.java | 1 +
.../processing/framework/SegmentConfig.java | 20 +++++-
.../framework/SegmentProcessorFramework.java | 6 +-
.../framework/SegmentProcessorFrameworkTest.java | 17 ++---
.../pinot/plugin/minion/tasks/MergeTaskUtils.java | 1 +
.../plugin/minion/tasks/MergeTaskUtilsTest.java | 6 ++
.../name/NormalizedDateSegmentNameGenerator.java | 19 +++---
.../NormalizedDateSegmentNameGeneratorTest.java | 78 +++++++++++++++++++++-
8 files changed, 126 insertions(+), 22 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
b/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
index ccf1b44..d4a17c7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java
@@ -99,6 +99,7 @@ public class MinionConstants {
public static final String MAX_NUM_RECORDS_PER_SEGMENT_KEY =
"maxNumRecordsPerSegment";
public static final String MAX_NUM_PARALLEL_BUCKETS =
"maxNumParallelBuckets";
public static final String SEGMENT_NAME_PREFIX_KEY = "segmentNamePrefix";
+ public static final String SEGMENT_NAME_POSTFIX_KEY = "segmentNamePostfix";
}
public static class MergeRollupTask extends MergeTask {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentConfig.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentConfig.java
index 33aa896..7ac49f8 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentConfig.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentConfig.java
@@ -34,13 +34,16 @@ public class SegmentConfig {
private final int _maxNumRecordsPerSegment;
private final String _segmentNamePrefix;
+ private final String _segmentNamePostfix;
@JsonCreator
private SegmentConfig(@JsonProperty(value = "maxNumRecordsPerSegment",
required = true) int maxNumRecordsPerSegment,
- @JsonProperty("segmentNamePrefix") @Nullable String segmentNamePrefix) {
+ @JsonProperty("segmentNamePrefix") @Nullable String segmentNamePrefix,
+ @JsonProperty("segmentNamePostfix") @Nullable String segmentNamePostfix)
{
Preconditions.checkState(maxNumRecordsPerSegment > 0, "Max num records per
segment must be > 0");
_maxNumRecordsPerSegment = maxNumRecordsPerSegment;
_segmentNamePrefix = segmentNamePrefix;
+ _segmentNamePostfix = segmentNamePostfix;
}
/**
@@ -55,12 +58,18 @@ public class SegmentConfig {
return _segmentNamePrefix;
}
+ @Nullable
+ public String getSegmentNamePostfix() {
+ return _segmentNamePostfix;
+ }
+
/**
* Builder for SegmentConfig
*/
public static class Builder {
private int _maxNumRecordsPerSegment = DEFAULT_MAX_NUM_RECORDS_PER_SEGMENT;
private String _segmentNamePrefix;
+ private String _segmentNamePostfix;
public Builder setMaxNumRecordsPerSegment(int maxNumRecordsPerSegment) {
_maxNumRecordsPerSegment = maxNumRecordsPerSegment;
@@ -72,15 +81,20 @@ public class SegmentConfig {
return this;
}
+ public Builder setSegmentNamePostfix(String segmentNamePostfix) {
+ _segmentNamePostfix = segmentNamePostfix;
+ return this;
+ }
+
public SegmentConfig build() {
Preconditions.checkState(_maxNumRecordsPerSegment > 0, "Max num records
per segment must be > 0");
- return new SegmentConfig(_maxNumRecordsPerSegment, _segmentNamePrefix);
+ return new SegmentConfig(_maxNumRecordsPerSegment, _segmentNamePrefix,
_segmentNamePostfix);
}
}
@Override
public String toString() {
return "SegmentConfig{" + "_maxNumRecordsPerSegment=" +
_maxNumRecordsPerSegment + ", _segmentNamePrefix='"
- + _segmentNamePrefix + '\'' + '}';
+ + _segmentNamePrefix + '\'' + ", _segmentNamePostfix='" +
_segmentNamePostfix + '\'' + '}';
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessorFramework.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessorFramework.java
index 2bf0714..9b8dd6c 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessorFramework.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessorFramework.java
@@ -116,14 +116,18 @@ public class SegmentProcessorFramework {
TableConfig tableConfig = _segmentProcessorConfig.getTableConfig();
Schema schema = _segmentProcessorConfig.getSchema();
String segmentNamePrefix =
_segmentProcessorConfig.getSegmentConfig().getSegmentNamePrefix();
+ String segmentNamePostfix =
_segmentProcessorConfig.getSegmentConfig().getSegmentNamePostfix();
SegmentGeneratorConfig generatorConfig = new
SegmentGeneratorConfig(tableConfig, schema);
generatorConfig.setOutDir(_segmentsOutputDir.getPath());
if (tableConfig.getIndexingConfig().getSegmentNameGeneratorType() != null)
{
generatorConfig.setSegmentNameGenerator(
- SegmentNameGeneratorFactory.createSegmentNameGenerator(tableConfig,
schema, segmentNamePrefix, null, false));
+ SegmentNameGeneratorFactory
+ .createSegmentNameGenerator(tableConfig, schema,
segmentNamePrefix, segmentNamePostfix, false));
} else {
+ // SimpleSegmentNameGenerator is used by default.
generatorConfig.setSegmentNamePrefix(segmentNamePrefix);
+ generatorConfig.setSegmentNamePostfix(segmentNamePostfix);
}
int maxNumRecordsPerSegment =
_segmentProcessorConfig.getSegmentConfig().getMaxNumRecordsPerSegment();
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessorFrameworkTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessorFrameworkTest.java
index 5aacc24..7dccaa8 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessorFrameworkTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessorFrameworkTest.java
@@ -413,39 +413,40 @@ public class SegmentProcessorFrameworkTest {
// Segment config
config = new
SegmentProcessorConfig.Builder().setTableConfig(_tableConfig).setSchema(_schema).setSegmentConfig(
- new
SegmentConfig.Builder().setMaxNumRecordsPerSegment(4).setSegmentNamePrefix("myPrefix").build()).build();
+ new
SegmentConfig.Builder().setMaxNumRecordsPerSegment(4).setSegmentNamePrefix("myPrefix")
+ .setSegmentNamePostfix("myPostfix").build()).build();
framework = new SegmentProcessorFramework(_singleSegment, config,
workingDir);
outputSegments = framework.process();
assertEquals(outputSegments.size(), 3);
outputSegments.sort(null);
segmentMetadata = new SegmentMetadataImpl(outputSegments.get(0));
assertEquals(segmentMetadata.getTotalDocs(), 4);
- assertEquals(segmentMetadata.getName(),
"myPrefix_1597719600000_1597795200000_0");
+ assertEquals(segmentMetadata.getName(),
"myPrefix_1597719600000_1597795200000_myPostfix_0");
segmentMetadata = new SegmentMetadataImpl(outputSegments.get(1));
assertEquals(segmentMetadata.getTotalDocs(), 4);
- assertEquals(segmentMetadata.getName(),
"myPrefix_1597802400000_1597878000000_1");
+ assertEquals(segmentMetadata.getName(),
"myPrefix_1597802400000_1597878000000_myPostfix_1");
segmentMetadata = new SegmentMetadataImpl(outputSegments.get(2));
assertEquals(segmentMetadata.getTotalDocs(), 2);
- assertEquals(segmentMetadata.getName(),
"myPrefix_1597881600000_1597892400000_2");
+ assertEquals(segmentMetadata.getName(),
"myPrefix_1597881600000_1597892400000_myPostfix_2");
FileUtils.cleanDirectory(workingDir);
rewindRecordReaders(_singleSegment);
config = new
SegmentProcessorConfig.Builder().setTableConfig(_tableConfigSegmentNameGeneratorEnabled)
.setSchema(_schema).setSegmentConfig(new
SegmentConfig.Builder().setMaxNumRecordsPerSegment(4)
- .setSegmentNamePrefix("myPrefix").build()).build();
+
.setSegmentNamePrefix("myPrefix").setSegmentNamePostfix("myPostfix").build()).build();
framework = new SegmentProcessorFramework(_singleSegment, config,
workingDir);
outputSegments = framework.process();
assertEquals(outputSegments.size(), 3);
outputSegments.sort(null);
segmentMetadata = new SegmentMetadataImpl(outputSegments.get(0));
assertEquals(segmentMetadata.getTotalDocs(), 4);
- assertEquals(segmentMetadata.getName(),
"myPrefix_2020-08-18_2020-08-19_0");
+ assertEquals(segmentMetadata.getName(),
"myPrefix_2020-08-18_2020-08-19_myPostfix_0");
segmentMetadata = new SegmentMetadataImpl(outputSegments.get(1));
assertEquals(segmentMetadata.getTotalDocs(), 4);
- assertEquals(segmentMetadata.getName(),
"myPrefix_2020-08-19_2020-08-19_1");
+ assertEquals(segmentMetadata.getName(),
"myPrefix_2020-08-19_2020-08-19_myPostfix_1");
segmentMetadata = new SegmentMetadataImpl(outputSegments.get(2));
assertEquals(segmentMetadata.getTotalDocs(), 2);
- assertEquals(segmentMetadata.getName(),
"myPrefix_2020-08-20_2020-08-20_2");
+ assertEquals(segmentMetadata.getName(),
"myPrefix_2020-08-20_2020-08-20_myPostfix_2");
FileUtils.cleanDirectory(workingDir);
rewindRecordReaders(_singleSegment);
}
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/MergeTaskUtils.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/MergeTaskUtils.java
index f6b3aec..2e5e07b 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/MergeTaskUtils.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/MergeTaskUtils.java
@@ -142,6 +142,7 @@ public class MergeTaskUtils {
segmentConfigBuilder.setMaxNumRecordsPerSegment(Integer.parseInt(maxNumRecordsPerSegment));
}
segmentConfigBuilder.setSegmentNamePrefix(taskConfig.get(MergeTask.SEGMENT_NAME_PREFIX_KEY));
+
segmentConfigBuilder.setSegmentNamePostfix(taskConfig.get(MergeTask.SEGMENT_NAME_POSTFIX_KEY));
return segmentConfigBuilder.build();
}
}
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/MergeTaskUtilsTest.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/MergeTaskUtilsTest.java
index 0139392..af32914 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/MergeTaskUtilsTest.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/MergeTaskUtilsTest.java
@@ -161,12 +161,18 @@ public class MergeTaskUtilsTest {
Map<String, String> taskConfig = new HashMap<>();
taskConfig.put(MergeTask.MAX_NUM_RECORDS_PER_SEGMENT_KEY, "10000");
taskConfig.put(MergeTask.SEGMENT_NAME_PREFIX_KEY, "myPrefix");
+ taskConfig.put(MergeTask.SEGMENT_NAME_POSTFIX_KEY, "myPostfix");
SegmentConfig segmentConfig = MergeTaskUtils.getSegmentConfig(taskConfig);
assertEquals(segmentConfig.getMaxNumRecordsPerSegment(), 10000);
assertEquals(segmentConfig.getSegmentNamePrefix(), "myPrefix");
+ assertEquals(segmentConfig.getSegmentNamePostfix(), "myPostfix");
+ assertEquals(segmentConfig.toString(),
+ "SegmentConfig{_maxNumRecordsPerSegment=10000,
_segmentNamePrefix='myPrefix', "
+ + "_segmentNamePostfix='myPostfix'}");
segmentConfig = MergeTaskUtils.getSegmentConfig(Collections.emptyMap());
assertEquals(segmentConfig.getMaxNumRecordsPerSegment(),
SegmentConfig.DEFAULT_MAX_NUM_RECORDS_PER_SEGMENT);
assertNull(segmentConfig.getSegmentNamePrefix());
+ assertNull(segmentConfig.getSegmentNamePostfix());
}
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
index 629c611..a65ef57 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
@@ -58,7 +58,9 @@ public class NormalizedDateSegmentNameGenerator implements
SegmentNameGenerator
_segmentNamePrefix != null && isValidSegmentName(_segmentNamePrefix));
_excludeSequenceId = excludeSequenceId;
_appendPushType = "APPEND".equalsIgnoreCase(pushType);
- _segmentNamePostfix = segmentNamePostfix;
+ _segmentNamePostfix = segmentNamePostfix != null ?
segmentNamePostfix.trim() : null;
+ Preconditions.checkArgument(
+ _segmentNamePostfix == null ||
isValidSegmentName(_segmentNamePostfix));
// Include time info for APPEND push type
if (_appendPushType) {
@@ -96,13 +98,9 @@ public class NormalizedDateSegmentNameGenerator implements
SegmentNameGenerator
// Include time value for APPEND push type
if (_appendPushType) {
return JOINER.join(_segmentNamePrefix,
getNormalizedDate(Preconditions.checkNotNull(minTimeValue)),
- getNormalizedDate(Preconditions.checkNotNull(maxTimeValue)),
sequenceIdInSegmentName);
+ getNormalizedDate(Preconditions.checkNotNull(maxTimeValue)),
_segmentNamePostfix, sequenceIdInSegmentName);
} else {
- if (_segmentNamePostfix != null) {
- return JOINER.join(_segmentNamePrefix, _segmentNamePostfix,
sequenceIdInSegmentName);
- } else {
- return JOINER.join(_segmentNamePrefix, sequenceIdInSegmentName);
- }
+ return JOINER.join(_segmentNamePrefix, _segmentNamePostfix,
sequenceIdInSegmentName);
}
}
@@ -129,8 +127,11 @@ public class NormalizedDateSegmentNameGenerator implements
SegmentNameGenerator
@Override
public String toString() {
StringBuilder stringBuilder =
- new StringBuilder("NormalizedDateSegmentNameGenerator:
segmentNamePrefix=").append(_segmentNamePrefix)
- .append(", appendPushType=").append(_appendPushType);
+ new StringBuilder("NormalizedDateSegmentNameGenerator:
segmentNamePrefix=").append(_segmentNamePrefix);
+ if (_segmentNamePostfix != null) {
+ stringBuilder.append(",
segmentNamePostfix=").append(_segmentNamePostfix);
+ }
+ stringBuilder.append(", appendPushType=").append(_appendPushType);
if (_excludeSequenceId) {
stringBuilder.append(", excludeSequenceId=true");
}
diff --git
a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGeneratorTest.java
b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGeneratorTest.java
index 02ab1d2..535a9fd 100644
---
a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGeneratorTest.java
+++
b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGeneratorTest.java
@@ -30,7 +30,9 @@ public class NormalizedDateSegmentNameGeneratorTest {
private static final String TABLE_NAME = "myTable";
private static final String MALFORMED_TABLE_NAME = "my/Table";
private static final String SEGMENT_NAME_PREFIX = "myTable_daily";
+ private static final String SEGMENT_NAME_POSTFIX = "myPostfix";
private static final String MALFORMED_SEGMENT_NAME_PREFIX = "myTable\\daily";
+ private static final String MALFORMED_SEGMENT_NAME_POSTFIX = "my\\postfix";
private static final String APPEND_PUSH_TYPE = "APPEND";
private static final String REFRESH_PUSH_TYPE = "REFRESH";
private static final String EPOCH_TIME_FORMAT = "EPOCH";
@@ -65,6 +67,18 @@ public class NormalizedDateSegmentNameGeneratorTest {
}
@Test
+ public void testWithSegmentNamePrefixPostfix() {
+ SegmentNameGenerator segmentNameGenerator =
+ new NormalizedDateSegmentNameGenerator(TABLE_NAME,
SEGMENT_NAME_PREFIX, false, REFRESH_PUSH_TYPE, null, null,
+ SEGMENT_NAME_POSTFIX);
+ assertEquals(segmentNameGenerator.toString(),
+ "NormalizedDateSegmentNameGenerator: segmentNamePrefix=myTable_daily,
segmentNamePostfix=myPostfix, "
+ + "appendPushType=false");
+ assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID,
null, null), "myTable_daily_myPostfix");
+ assertEquals(segmentNameGenerator.generateSegmentName(VALID_SEQUENCE_ID,
null, null), "myTable_daily_myPostfix_1");
+ }
+
+ @Test
public void testWithUntrimmedSegmentNamePrefix() {
SegmentNameGenerator segmentNameGenerator =
new NormalizedDateSegmentNameGenerator(TABLE_NAME, SEGMENT_NAME_PREFIX
+ " ", false, REFRESH_PUSH_TYPE, null,
@@ -76,6 +90,18 @@ public class NormalizedDateSegmentNameGeneratorTest {
}
@Test
+ public void testWithUntrimmedSegmentNamePrefixPostfix() {
+ SegmentNameGenerator segmentNameGenerator =
+ new NormalizedDateSegmentNameGenerator(TABLE_NAME, SEGMENT_NAME_PREFIX
+ " ", false, REFRESH_PUSH_TYPE, null,
+ null, SEGMENT_NAME_POSTFIX + " ");
+ assertEquals(segmentNameGenerator.toString(),
+ "NormalizedDateSegmentNameGenerator: segmentNamePrefix=myTable_daily,
segmentNamePostfix=myPostfix, "
+ + "appendPushType=false");
+ assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID,
null, null), "myTable_daily_myPostfix");
+ assertEquals(segmentNameGenerator.generateSegmentName(VALID_SEQUENCE_ID,
null, null), "myTable_daily_myPostfix_1");
+ }
+
+ @Test
public void testExcludeSequenceId() {
SegmentNameGenerator segmentNameGenerator =
new NormalizedDateSegmentNameGenerator(TABLE_NAME, null, true,
REFRESH_PUSH_TYPE, null, null, null);
@@ -98,6 +124,18 @@ public class NormalizedDateSegmentNameGeneratorTest {
}
@Test
+ public void testWithPrefixPostfixExcludeSequenceId() {
+ SegmentNameGenerator segmentNameGenerator =
+ new NormalizedDateSegmentNameGenerator(TABLE_NAME,
SEGMENT_NAME_PREFIX, true, REFRESH_PUSH_TYPE, null, null,
+ SEGMENT_NAME_POSTFIX);
+ assertEquals(segmentNameGenerator.toString(),
+ "NormalizedDateSegmentNameGenerator: segmentNamePrefix=myTable_daily,
segmentNamePostfix=myPostfix, "
+ + "appendPushType=false, excludeSequenceId=true");
+ assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID,
null, null), "myTable_daily_myPostfix");
+ assertEquals(segmentNameGenerator.generateSegmentName(VALID_SEQUENCE_ID,
null, null), "myTable_daily_myPostfix");
+ }
+
+ @Test
public void testAppend() {
SegmentNameGenerator segmentNameGenerator =
new NormalizedDateSegmentNameGenerator(TABLE_NAME, null, false,
APPEND_PUSH_TYPE, DAILY_PUSH_FREQUENCY,
@@ -112,6 +150,35 @@ public class NormalizedDateSegmentNameGeneratorTest {
}
@Test
+ public void testAppendWithSegmentNamePrefix() {
+ SegmentNameGenerator segmentNameGenerator =
+ new NormalizedDateSegmentNameGenerator(TABLE_NAME,
SEGMENT_NAME_PREFIX, false, APPEND_PUSH_TYPE,
+ DAILY_PUSH_FREQUENCY, new DateTimeFormatSpec(1,
TimeUnit.DAYS.toString(), EPOCH_TIME_FORMAT), null);
+ assertEquals(segmentNameGenerator.toString(),
+ "NormalizedDateSegmentNameGenerator: segmentNamePrefix=myTable_daily,
appendPushType=true, "
+ + "outputSDF=yyyy-MM-dd, inputTimeUnit=DAYS");
+ assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID,
1L, 3L),
+ "myTable_daily_1970-01-02_1970-01-04");
+ assertEquals(segmentNameGenerator.generateSegmentName(VALID_SEQUENCE_ID,
1L, 3L),
+ "myTable_daily_1970-01-02_1970-01-04_1");
+ }
+
+ @Test
+ public void testAppendWithSegmentNamePrefixPostfix() {
+ SegmentNameGenerator segmentNameGenerator =
+ new NormalizedDateSegmentNameGenerator(TABLE_NAME,
SEGMENT_NAME_PREFIX, false, APPEND_PUSH_TYPE,
+ DAILY_PUSH_FREQUENCY, new DateTimeFormatSpec(1,
TimeUnit.DAYS.toString(), EPOCH_TIME_FORMAT),
+ SEGMENT_NAME_POSTFIX);
+ assertEquals(segmentNameGenerator.toString(),
+ "NormalizedDateSegmentNameGenerator: segmentNamePrefix=myTable_daily,
segmentNamePostfix=myPostfix, "
+ + "appendPushType=true, outputSDF=yyyy-MM-dd, inputTimeUnit=DAYS");
+ assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID,
1L, 3L),
+ "myTable_daily_1970-01-02_1970-01-04_myPostfix");
+ assertEquals(segmentNameGenerator.generateSegmentName(VALID_SEQUENCE_ID,
1L, 3L),
+ "myTable_daily_1970-01-02_1970-01-04_myPostfix_1");
+ }
+
+ @Test
public void testHoursTimeType() {
SegmentNameGenerator segmentNameGenerator =
new NormalizedDateSegmentNameGenerator(TABLE_NAME, null, false,
APPEND_PUSH_TYPE, DAILY_PUSH_FREQUENCY,
@@ -156,7 +223,7 @@ public class NormalizedDateSegmentNameGeneratorTest {
}
@Test
- public void testMalFormedTableNameAndSegmentNamePrefix() {
+ public void testMalFormedTableNameAndSegmentNamePrefixPostfix() {
try {
new NormalizedDateSegmentNameGenerator(MALFORMED_TABLE_NAME, null,
false, APPEND_PUSH_TYPE, DAILY_PUSH_FREQUENCY,
new DateTimeFormatSpec(1, TimeUnit.DAYS.toString(),
SIMPLE_DATE_TIME_FORMAT, STRING_SLASH_DATE_FORMAT), null);
@@ -172,6 +239,15 @@ public class NormalizedDateSegmentNameGeneratorTest {
} catch (IllegalArgumentException e) {
// Expected
}
+ try {
+ new NormalizedDateSegmentNameGenerator(
+ TABLE_NAME, SEGMENT_NAME_PREFIX, false, APPEND_PUSH_TYPE,
DAILY_PUSH_FREQUENCY,
+ new DateTimeFormatSpec(1, TimeUnit.DAYS.toString(),
SIMPLE_DATE_TIME_FORMAT, STRING_SLASH_DATE_FORMAT),
+ MALFORMED_SEGMENT_NAME_POSTFIX);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ // Expected
+ }
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
