This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 47e91a2 Add deprecated annotation for segment zk metadata (#4320)
47e91a2 is described below
commit 47e91a2551305ec751c29487d156d46404dab06b
Author: Seunghyun Lee <[email protected]>
AuthorDate: Mon Jun 17 21:53:01 2019 -0700
Add deprecated annotation for segment zk metadata (#4320)
* Add deprecated annotation for segment zk metadata
This is a follow up work for #4288 and #4289
1. Added deprecated annotation for segment zk metadata.
2. Remove all getTableName & setTableName usages from testing code
- There is no usage of getTableName() for segment zk metadata except
the case where we fill segment metadata from segment zk metadata
for realtime segment creation.
- For segment metadata, the only usage is during the segment upload
- We still write segment name to both segment zk metadata and segment
metadata for rollback support.
* Addressing comments
---
...ixExternalViewBasedTimeBoundaryServiceTest.java | 5 +++--
.../pinot/common/metadata/ZKMetadataProvider.java | 25 +++++++++++-----------
.../common/metadata/segment/SegmentZKMetadata.java | 15 ++++++++++---
.../apache/pinot/common/utils/CommonConstants.java | 4 +++-
.../common/metadata/SegmentZKMetadataTest.java | 6 ------
.../pinot/controller/api/upload/ZKOperator.java | 7 +++---
.../helix/core/PinotHelixResourceManager.java | 14 ++++++------
.../core/realtime/PinotRealtimeSegmentManager.java | 2 +-
.../controller/helix/PinotResourceManagerTest.java | 9 ++++----
.../helix/core/PinotHelixResourceManagerTest.java | 8 ++-----
.../segment/FlushThresholdUpdaterTest.java | 1 -
.../helix/core/retention/RetentionManagerTest.java | 1 -
.../realtime/HLRealtimeSegmentDataManager.java | 2 +-
.../manager/realtime/RealtimeTableDataManager.java | 5 +++--
.../core/segment/index/SegmentMetadataImpl.java | 1 +
.../realtime/LLRealtimeSegmentDataManagerTest.java | 1 -
.../ControllerPeriodicTasksIntegrationTests.java | 8 +++----
.../converter/ColumnarToStarTreeConverter.java | 5 ++++-
18 files changed, 62 insertions(+), 57 deletions(-)
diff --git
a/pinot-broker/src/test/java/org/apache/pinot/broker/routing/HelixExternalViewBasedTimeBoundaryServiceTest.java
b/pinot-broker/src/test/java/org/apache/pinot/broker/routing/HelixExternalViewBasedTimeBoundaryServiceTest.java
index 20ccc9c..74add38 100644
---
a/pinot-broker/src/test/java/org/apache/pinot/broker/routing/HelixExternalViewBasedTimeBoundaryServiceTest.java
+++
b/pinot-broker/src/test/java/org/apache/pinot/broker/routing/HelixExternalViewBasedTimeBoundaryServiceTest.java
@@ -142,11 +142,12 @@ public class
HelixExternalViewBasedTimeBoundaryServiceTest {
private void addSegmentZKMetadata(String rawTableName, int endTimeInDays,
TimeUnit timeUnit) {
for (int i = 1; i <= endTimeInDays; i++) {
OfflineSegmentZKMetadata offlineSegmentZKMetadata = new
OfflineSegmentZKMetadata();
- offlineSegmentZKMetadata.setTableName(rawTableName);
offlineSegmentZKMetadata.setSegmentName(rawTableName + i);
offlineSegmentZKMetadata.setEndTime(i * timeUnit.convert(1L,
TimeUnit.DAYS));
offlineSegmentZKMetadata.setTimeUnit(timeUnit);
- ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore,
offlineSegmentZKMetadata);
+ ZKMetadataProvider
+ .setOfflineSegmentZKMetadata(_propertyStore,
TableNameBuilder.OFFLINE.tableNameWithType(rawTableName),
+ offlineSegmentZKMetadata);
}
}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
b/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
index 7cf7448..2a3ad60 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metadata/ZKMetadataProvider.java
@@ -145,30 +145,29 @@ public class ZKMetadataProvider {
}
public static boolean
setOfflineSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore,
- OfflineSegmentZKMetadata offlineSegmentZKMetadata, int expectedVersion) {
+ String offlineTableName, OfflineSegmentZKMetadata
offlineSegmentZKMetadata, int expectedVersion) {
// NOTE: Helix will throw ZkBadVersionException if version does not match
try {
- return propertyStore.set(constructPropertyStorePathForSegment(
-
TableNameBuilder.OFFLINE.tableNameWithType(offlineSegmentZKMetadata.getTableName()),
- offlineSegmentZKMetadata.getSegmentName()),
offlineSegmentZKMetadata.toZNRecord(), expectedVersion,
- AccessOption.PERSISTENT);
+ return propertyStore
+ .set(constructPropertyStorePathForSegment(offlineTableName,
offlineSegmentZKMetadata.getSegmentName()),
+ offlineSegmentZKMetadata.toZNRecord(), expectedVersion,
AccessOption.PERSISTENT);
} catch (ZkBadVersionException e) {
return false;
}
}
public static boolean
setOfflineSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore,
- OfflineSegmentZKMetadata offlineSegmentZKMetadata) {
- return propertyStore.set(constructPropertyStorePathForSegment(
-
TableNameBuilder.OFFLINE.tableNameWithType(offlineSegmentZKMetadata.getTableName()),
- offlineSegmentZKMetadata.getSegmentName()),
offlineSegmentZKMetadata.toZNRecord(), AccessOption.PERSISTENT);
+ String offlineTableName, OfflineSegmentZKMetadata
offlineSegmentZKMetadata) {
+ return propertyStore
+ .set(constructPropertyStorePathForSegment(offlineTableName,
offlineSegmentZKMetadata.getSegmentName()),
+ offlineSegmentZKMetadata.toZNRecord(), AccessOption.PERSISTENT);
}
public static boolean
setRealtimeSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore,
- RealtimeSegmentZKMetadata realtimeSegmentZKMetadata) {
- return propertyStore.set(constructPropertyStorePathForSegment(
-
TableNameBuilder.REALTIME.tableNameWithType(realtimeSegmentZKMetadata.getTableName()),
- realtimeSegmentZKMetadata.getSegmentName()),
realtimeSegmentZKMetadata.toZNRecord(), AccessOption.PERSISTENT);
+ String realtimeTableName, RealtimeSegmentZKMetadata
realtimeSegmentZKMetadata) {
+ return propertyStore
+ .set(constructPropertyStorePathForSegment(realtimeTableName,
realtimeSegmentZKMetadata.getSegmentName()),
+ realtimeSegmentZKMetadata.toZNRecord(), AccessOption.PERSISTENT);
}
@Nullable
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
index 67366a5..ec13e5a 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metadata/segment/SegmentZKMetadata.java
@@ -46,7 +46,6 @@ public abstract class SegmentZKMetadata implements ZKMetadata
{
protected static final String NULL = "null";
private String _segmentName;
- private String _tableName;
private SegmentType _segmentType;
private long _startTime = -1;
private long _endTime = -1;
@@ -62,6 +61,9 @@ public abstract class SegmentZKMetadata implements ZKMetadata
{
private Map<String, String> _customMap;
private String _crypterName;
+ @Deprecated
+ private String _tableName;
+
public SegmentZKMetadata() {
}
@@ -103,10 +105,12 @@ public abstract class SegmentZKMetadata implements
ZKMetadata {
_segmentName = segmentName;
}
+ @Deprecated
public String getTableName() {
return _tableName;
}
+ @Deprecated
public void setTableName(String tableName) {
_tableName = tableName;
}
@@ -266,7 +270,10 @@ public abstract class SegmentZKMetadata implements
ZKMetadata {
public ZNRecord toZNRecord() {
ZNRecord znRecord = new ZNRecord(_segmentName);
znRecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME,
_segmentName);
- znRecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, _tableName);
+
+ if (_tableName != null) {
+ znRecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, _tableName);
+ }
if (_crypterName != null) {
znRecord.setSimpleField(CommonConstants.Segment.CRYPTER_NAME,
_crypterName);
@@ -308,7 +315,9 @@ public abstract class SegmentZKMetadata implements
ZKMetadata {
public Map<String, String> toMap() {
Map<String, String> configMap = new HashMap<>();
configMap.put(CommonConstants.Segment.SEGMENT_NAME, _segmentName);
- configMap.put(CommonConstants.Segment.TABLE_NAME, _tableName);
+ if (_tableName != null) {
+ configMap.put(CommonConstants.Segment.TABLE_NAME, _tableName);
+ }
configMap.put(CommonConstants.Segment.SEGMENT_TYPE,
_segmentType.toString());
if (_timeUnit == null) {
configMap.put(CommonConstants.Segment.TIME_UNIT, null);
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/CommonConstants.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/CommonConstants.java
index 8132cdb..ff68450 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/CommonConstants.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/CommonConstants.java
@@ -330,7 +330,6 @@ public class CommonConstants {
}
public static final String SEGMENT_NAME = "segment.name";
- public static final String TABLE_NAME = "segment.table.name";
public static final String SEGMENT_TYPE = "segment.type";
public static final String CRYPTER_NAME = "segment.crypter";
public static final String INDEX_VERSION = "segment.index.version";
@@ -360,6 +359,9 @@ public class CommonConstants {
public enum SegmentType {
OFFLINE, REALTIME
}
+
+ @Deprecated
+ public static final String TABLE_NAME = "segment.table.name";
}
public static class SegmentOperations {
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
index 551f940..db9ba32 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/metadata/SegmentZKMetadataTest.java
@@ -119,7 +119,6 @@ public class SegmentZKMetadataTest {
String segmentName = "testTable_R_1000_2000_groupId0_part0";
ZNRecord record = new ZNRecord(segmentName);
record.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, segmentName);
- record.setSimpleField(CommonConstants.Segment.TABLE_NAME, "testTable");
record.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
record.setEnumField(CommonConstants.Segment.SEGMENT_TYPE,
CommonConstants.Segment.SegmentType.REALTIME);
record.setEnumField(CommonConstants.Segment.Realtime.STATUS,
CommonConstants.Segment.Realtime.Status.DONE);
@@ -137,7 +136,6 @@ public class SegmentZKMetadataTest {
private RealtimeSegmentZKMetadata getTestDoneRealtimeSegmentMetadata() {
RealtimeSegmentZKMetadata realtimeSegmentMetadata = new
RealtimeSegmentZKMetadata();
realtimeSegmentMetadata.setSegmentName("testTable_R_1000_2000_groupId0_part0");
- realtimeSegmentMetadata.setTableName("testTable");
realtimeSegmentMetadata.setSegmentType(SegmentType.REALTIME);
realtimeSegmentMetadata.setIndexVersion("v1");
realtimeSegmentMetadata.setStartTime(1000);
@@ -156,7 +154,6 @@ public class SegmentZKMetadataTest {
String segmentName = "testTable_R_1000_groupId0_part0";
ZNRecord record = new ZNRecord(segmentName);
record.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, segmentName);
- record.setSimpleField(CommonConstants.Segment.TABLE_NAME, "testTable");
record.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
record.setEnumField(CommonConstants.Segment.SEGMENT_TYPE,
CommonConstants.Segment.SegmentType.REALTIME);
record.setEnumField(CommonConstants.Segment.Realtime.STATUS,
CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
@@ -174,7 +171,6 @@ public class SegmentZKMetadataTest {
private RealtimeSegmentZKMetadata getTestInProgressRealtimeSegmentMetadata()
{
RealtimeSegmentZKMetadata realtimeSegmentMetadata = new
RealtimeSegmentZKMetadata();
realtimeSegmentMetadata.setSegmentName("testTable_R_1000_groupId0_part0");
- realtimeSegmentMetadata.setTableName("testTable");
realtimeSegmentMetadata.setSegmentType(SegmentType.REALTIME);
realtimeSegmentMetadata.setIndexVersion("v1");
realtimeSegmentMetadata.setStartTime(1000);
@@ -193,7 +189,6 @@ public class SegmentZKMetadataTest {
String segmentName = "testTable_O_3000_4000";
ZNRecord record = new ZNRecord(segmentName);
record.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, segmentName);
- record.setSimpleField(CommonConstants.Segment.TABLE_NAME, "testTable");
record.setSimpleField(CommonConstants.Segment.CRYPTER_NAME, "testCrypter");
record.setSimpleField(CommonConstants.Segment.INDEX_VERSION, "v1");
record.setEnumField(CommonConstants.Segment.SEGMENT_TYPE,
CommonConstants.Segment.SegmentType.OFFLINE);
@@ -212,7 +207,6 @@ public class SegmentZKMetadataTest {
private OfflineSegmentZKMetadata getTestOfflineSegmentMetadata() {
OfflineSegmentZKMetadata offlineSegmentMetadata = new
OfflineSegmentZKMetadata();
offlineSegmentMetadata.setSegmentName("testTable_O_3000_4000");
- offlineSegmentMetadata.setTableName("testTable");
offlineSegmentMetadata.setCrypterName("testCrypter");
offlineSegmentMetadata.setSegmentType(SegmentType.OFFLINE);
offlineSegmentMetadata.setIndexVersion("v1");
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
index 52d2caa..ab1a6ef 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/ZKOperator.java
@@ -120,7 +120,8 @@ public class ZKOperator {
// Lock the segment by setting the upload start time in ZK
existingSegmentZKMetadata.setSegmentUploadStartTime(System.currentTimeMillis());
- if
(!_pinotHelixResourceManager.updateZkMetadata(existingSegmentZKMetadata,
znRecord.getVersion())) {
+ if (!_pinotHelixResourceManager
+ .updateZkMetadata(offlineTableName, existingSegmentZKMetadata,
znRecord.getVersion())) {
throw new ControllerApplicationException(LOGGER,
"Failed to lock the segment: " + segmentName + " of table: " +
offlineTableName + ", retry later",
Response.Status.CONFLICT);
@@ -158,7 +159,7 @@ public class ZKOperator {
// (creation time is not included in the crc)
existingSegmentZKMetadata.setCreationTime(segmentMetadata.getIndexCreationTime());
existingSegmentZKMetadata.setRefreshTime(System.currentTimeMillis());
- if
(!_pinotHelixResourceManager.updateZkMetadata(existingSegmentZKMetadata)) {
+ if (!_pinotHelixResourceManager.updateZkMetadata(offlineTableName,
existingSegmentZKMetadata)) {
throw new RuntimeException(
"Failed to update ZK metadata for segment: " + segmentName + "
of table: " + offlineTableName);
}
@@ -179,7 +180,7 @@ public class ZKOperator {
_pinotHelixResourceManager.refreshSegment(offlineTableName,
segmentMetadata, existingSegmentZKMetadata);
}
} catch (Exception e) {
- if
(!_pinotHelixResourceManager.updateZkMetadata(existingSegmentZKMetadata)) {
+ if (!_pinotHelixResourceManager.updateZkMetadata(offlineTableName,
existingSegmentZKMetadata)) {
LOGGER.error("Failed to update ZK metadata for segment: {} of table:
{}", segmentName, offlineTableName);
}
throw e;
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index a4d4d0b..5eeeddc 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -1595,7 +1595,7 @@ public class PinotHelixResourceManager {
offlineSegmentZKMetadata.setDownloadUrl(downloadUrl);
offlineSegmentZKMetadata.setCrypterName(crypter);
offlineSegmentZKMetadata.setPushTime(System.currentTimeMillis());
- if (!ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore,
offlineSegmentZKMetadata)) {
+ if (!ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore,
offlineTableName, offlineSegmentZKMetadata)) {
throw new RuntimeException(
"Failed to set segment ZK metadata for table: " + offlineTableName +
", segment: " + segmentName);
}
@@ -1610,12 +1610,14 @@ public class PinotHelixResourceManager {
ZKMetadataProvider.constructPropertyStorePathForSegment(tableNameWithType,
segmentName));
}
- public boolean updateZkMetadata(@Nonnull OfflineSegmentZKMetadata
segmentMetadata, int expectedVersion) {
- return ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore,
segmentMetadata, expectedVersion);
+ public boolean updateZkMetadata(@Nonnull String offlineTableName, @Nonnull
OfflineSegmentZKMetadata segmentMetadata,
+ int expectedVersion) {
+ return ZKMetadataProvider
+ .setOfflineSegmentZKMetadata(_propertyStore, offlineTableName,
segmentMetadata, expectedVersion);
}
- public boolean updateZkMetadata(@Nonnull OfflineSegmentZKMetadata
segmentMetadata) {
- return ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore,
segmentMetadata);
+ public boolean updateZkMetadata(@Nonnull String offlineTableName, @Nonnull
OfflineSegmentZKMetadata segmentMetadata) {
+ return ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore,
offlineTableName, segmentMetadata);
}
public void refreshSegment(@Nonnull String offlineTableName, @Nonnull
SegmentMetadata segmentMetadata,
@@ -1627,7 +1629,7 @@ public class PinotHelixResourceManager {
// segment or load from local
offlineSegmentZKMetadata =
ZKMetadataUtils.updateSegmentMetadata(offlineSegmentZKMetadata,
segmentMetadata);
offlineSegmentZKMetadata.setRefreshTime(System.currentTimeMillis());
- if (!ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore,
offlineSegmentZKMetadata)) {
+ if (!ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore,
offlineTableName, offlineSegmentZKMetadata)) {
throw new RuntimeException(
"Failed to update ZK metadata for segment: " + segmentName + " of
table: " + offlineTableName);
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotRealtimeSegmentManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotRealtimeSegmentManager.java
index 331a4a0..32642dd 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotRealtimeSegmentManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotRealtimeSegmentManager.java
@@ -251,7 +251,7 @@ public class PinotRealtimeSegmentManager implements
HelixPropertyListener, IZkCh
realtimeSegmentMetadataToAdd.setSegmentName(segmentId);
// Add the new metadata to the property store
-
ZKMetadataProvider.setRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(),
+
ZKMetadataProvider.setRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(),
resourceName,
realtimeSegmentMetadataToAdd);
// Update the ideal state to add the new realtime segment
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/PinotResourceManagerTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/PinotResourceManagerTest.java
index eb57c7f..d65e818 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/PinotResourceManagerTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/PinotResourceManagerTest.java
@@ -69,22 +69,21 @@ public class PinotResourceManagerTest extends
ControllerTest {
@Test
public void testUpdateSegmentZKMetadata() {
OfflineSegmentZKMetadata segmentZKMetadata = new
OfflineSegmentZKMetadata();
- segmentZKMetadata.setTableName("testTable");
segmentZKMetadata.setSegmentName("testSegment");
// Segment ZK metadata does not exist
-
Assert.assertFalse(_helixResourceManager.updateZkMetadata(segmentZKMetadata,
0));
+
Assert.assertFalse(_helixResourceManager.updateZkMetadata("testTable_OFFLINE",
segmentZKMetadata, 0));
// Set segment ZK metadata
-
Assert.assertTrue(_helixResourceManager.updateZkMetadata(segmentZKMetadata));
+
Assert.assertTrue(_helixResourceManager.updateZkMetadata("testTable_OFFLINE",
segmentZKMetadata));
// Update ZK metadata
Assert.assertEquals(
_helixResourceManager.getSegmentMetadataZnRecord("testTable_OFFLINE",
"testSegment").getVersion(), 0);
-
Assert.assertTrue(_helixResourceManager.updateZkMetadata(segmentZKMetadata, 0));
+
Assert.assertTrue(_helixResourceManager.updateZkMetadata("testTable_OFFLINE",
segmentZKMetadata, 0));
Assert.assertEquals(
_helixResourceManager.getSegmentMetadataZnRecord("testTable_OFFLINE",
"testSegment").getVersion(), 1);
-
Assert.assertFalse(_helixResourceManager.updateZkMetadata(segmentZKMetadata,
0));
+
Assert.assertFalse(_helixResourceManager.updateZkMetadata("testTable_OFFLINE",
segmentZKMetadata, 0));
}
/**
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
index 7d9f8d9..a850ac4 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerTest.java
@@ -244,29 +244,25 @@ public class PinotHelixResourceManagerTest extends
ControllerTest {
// Test retrieving OFFLINE segment ZK metadata
{
OfflineSegmentZKMetadata offlineSegmentZKMetadata = new
OfflineSegmentZKMetadata();
- offlineSegmentZKMetadata.setTableName(OFFLINE_TABLE_NAME);
offlineSegmentZKMetadata.setSegmentName(segmentName);
- ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore,
offlineSegmentZKMetadata);
+ ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore,
OFFLINE_TABLE_NAME, offlineSegmentZKMetadata);
List<OfflineSegmentZKMetadata> retrievedMetadataList =
_helixResourceManager.getOfflineSegmentMetadata(OFFLINE_TABLE_NAME);
Assert.assertEquals(retrievedMetadataList.size(), 1);
OfflineSegmentZKMetadata retrievedMetadata =
retrievedMetadataList.get(0);
- Assert.assertEquals(retrievedMetadata.getTableName(),
OFFLINE_TABLE_NAME);
Assert.assertEquals(retrievedMetadata.getSegmentName(), segmentName);
}
// Test retrieving REALTIME segment ZK metadata
{
RealtimeSegmentZKMetadata realtimeMetadata = new
RealtimeSegmentZKMetadata();
- realtimeMetadata.setTableName(REALTIME_TABLE_NAME);
realtimeMetadata.setSegmentName(segmentName);
realtimeMetadata.setStatus(CommonConstants.Segment.Realtime.Status.DONE);
- ZKMetadataProvider.setRealtimeSegmentZKMetadata(_propertyStore,
realtimeMetadata);
+ ZKMetadataProvider.setRealtimeSegmentZKMetadata(_propertyStore,
REALTIME_TABLE_NAME, realtimeMetadata);
List<RealtimeSegmentZKMetadata> retrievedMetadataList =
_helixResourceManager.getRealtimeSegmentMetadata(REALTIME_TABLE_NAME);
Assert.assertEquals(retrievedMetadataList.size(), 1);
RealtimeSegmentZKMetadata retrievedMetadata =
retrievedMetadataList.get(0);
- Assert.assertEquals(retrievedMetadata.getTableName(),
REALTIME_TABLE_NAME);
Assert.assertEquals(retrievedMetadata.getSegmentName(), segmentName);
Assert.assertEquals(realtimeMetadata.getStatus(),
CommonConstants.Segment.Realtime.Status.DONE);
}
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java
index aefaf3e..425ebff 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/realtime/segment/FlushThresholdUpdaterTest.java
@@ -244,7 +244,6 @@ public class FlushThresholdUpdaterTest {
newSegMetadata.setStartOffset(startOffset);
newSegMetadata.setEndOffset(Long.MAX_VALUE);
newSegMetadata.setNumReplicas(3);
- newSegMetadata.setTableName(realtimeTableName);
newSegMetadata.setSegmentName(newSegmentName.getSegmentName());
newSegMetadata.setStatus(CommonConstants.Segment.Realtime.Status.IN_PROGRESS);
return newSegMetadata;
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/retention/RetentionManagerTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/retention/RetentionManagerTest.java
index d4adfe1..bb20ca3 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/retention/RetentionManagerTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/retention/RetentionManagerTest.java
@@ -299,7 +299,6 @@ public class RetentionManagerTest {
segmentMetadata.setEndOffset(-1L);
segmentMetadata.setNumReplicas(replicaCount);
- segmentMetadata.setTableName(TEST_TABLE_NAME);
return segmentMetadata;
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
index 9b5c542..b8b7f17 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/HLRealtimeSegmentDataManager.java
@@ -355,7 +355,7 @@ public class HLRealtimeSegmentDataManager extends
RealtimeSegmentDataManager {
metadataToOverwrite.setEndTime(segEndTime);
metadataToOverwrite.setTimeUnit(timeUnit);
metadataToOverwrite.setTotalRawDocs(realtimeSegment.getNumDocsIndexed());
- notifier.notifySegmentCommitted(metadataToOverwrite, segment);
+ notifier.notifySegmentCommitted(tableNameWithType,
metadataToOverwrite, segment);
segmentLogger
.info("Completed write of segment completion to Helix, waiting
for controller to assign a new segment");
} catch (Exception e) {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
index cde73a4..8c8d4a4 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/RealtimeTableDataManager.java
@@ -180,8 +180,9 @@ public class RealtimeTableDataManager extends
BaseTableDataManager {
return consumerDir.getAbsolutePath();
}
- public void notifySegmentCommitted(RealtimeSegmentZKMetadata metadata,
ImmutableSegment segment) {
- ZKMetadataProvider.setRealtimeSegmentZKMetadata(_propertyStore, metadata);
+ public void notifySegmentCommitted(String tableNameWithType,
RealtimeSegmentZKMetadata metadata,
+ ImmutableSegment segment) {
+ ZKMetadataProvider.setRealtimeSegmentZKMetadata(_propertyStore,
tableNameWithType, metadata);
addSegment(segment);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
index bcdca10..352a851 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/SegmentMetadataImpl.java
@@ -240,6 +240,7 @@ public class SegmentMetadataImpl implements SegmentMetadata
{
//set the table name
_tableName = segmentMetadataPropertiesConfiguration.getString(TABLE_NAME);
+
// Set segment name.
_segmentName =
segmentMetadataPropertiesConfiguration.getString(SEGMENT_NAME);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
index 3d7bfce..fd5505d 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/LLRealtimeSegmentDataManagerTest.java
@@ -124,7 +124,6 @@ public class LLRealtimeSegmentDataManagerTest {
private LLCRealtimeSegmentZKMetadata createZkMetadata() {
LLCRealtimeSegmentZKMetadata segmentZKMetadata = new
LLCRealtimeSegmentZKMetadata();
- segmentZKMetadata.setTableName(_tableName);
segmentZKMetadata.setSegmentName(_segmentNameStr);
segmentZKMetadata.setStartOffset(_startOffset);
return segmentZKMetadata;
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java
index ccd76b4..8c9ffe5 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java
@@ -168,9 +168,9 @@ public class ControllerPeriodicTasksIntegrationTests
extends BaseClusterIntegrat
/**
* Setup offline table, with segments from avro
*/
- private void setupOfflineTableAndSegments(String table, List<File>
avroFiles) throws Exception {
+ private void setupOfflineTableAndSegments(String tableName, List<File>
avroFiles) throws Exception {
TestUtils.ensureDirectoriesExistAndEmpty(_segmentDir, _tarDir);
- setTableName(table);
+ setTableName(tableName);
_realtimeTableConfig = null;
File schemaFile = getSchemaFile();
@@ -184,11 +184,11 @@ public class ControllerPeriodicTasksIntegrationTests
extends BaseClusterIntegrat
Assert.assertNotNull(outgoingTimeUnit);
String timeType = outgoingTimeUnit.toString();
- addOfflineTable(table, timeColumnName, timeType, TENANT_NAME, TENANT_NAME,
null, SegmentVersion.v1, null, null, null);
+ addOfflineTable(tableName, timeColumnName, timeType, TENANT_NAME,
TENANT_NAME, null, SegmentVersion.v1, null, null, null);
completeTableConfiguration();
ExecutorService executor = Executors.newCachedThreadPool();
- ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, 0,
_segmentDir, _tarDir, table, false,
+ ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, 0,
_segmentDir, _tarDir, tableName, false,
null, null, null, executor);
executor.shutdown();
executor.awaitTermination(10, TimeUnit.MINUTES);
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/ColumnarToStarTreeConverter.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/ColumnarToStarTreeConverter.java
index 34094b9..e1f039b 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/ColumnarToStarTreeConverter.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/segment/converter/ColumnarToStarTreeConverter.java
@@ -45,6 +45,9 @@ public class ColumnarToStarTreeConverter {
@Option(name = "-outputDir", required = true, usage = "Path to output
directory")
private String _outputDirName = null;
+ @Option(name = "-tableName", required = true, usage = "Table name without
type")
+ private String _rawTableName = null;
+
@Option(name = "-starTreeConfigFile", required = false, usage = "Path to
Star Tree configuration file")
private String _starTreeConfigFileName = null;
@@ -115,7 +118,7 @@ public class ColumnarToStarTreeConverter {
config.setFormat(FileFormat.PINOT);
config.setOutDir(_outputDirName);
config.setOverwrite(_overwrite);
- config.setTableName(segmentMetadata.getTableName());
+ config.setTableName(_rawTableName);
config.setSegmentName(segmentMetadata.getName());
StarTreeIndexSpec starTreeIndexSpec = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
