yuanbenson commented on code in PR #9295:
URL: https://github.com/apache/pinot/pull/9295#discussion_r958990701
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentUploadIntegrationTest.java:
##########
@@ -188,6 +202,133 @@ public void testUploadAndQuery()
testCountStar(numDocs);
}
+ /**
+ * Runs both SegmentMetadataPushJobRunner and SegmentTarPushJobRunner while
enabling consistent data push.
+ * Checks that segments are properly loaded and segment lineage entry were
also in expected states.
+ */
+ @Test
+ public void testUploadAndQueryWithConsistentPush()
+ throws Exception {
+ // Create and upload the schema and table config
+ Schema schema = createSchema();
+ addSchema(schema);
+ TableConfig offlineTableConfig =
createOfflineTableConfigWithConsistentPush();
+ addTableConfig(offlineTableConfig);
+
+ List<File> avroFiles = getAllAvroFiles();
+
+ ClusterIntegrationTestUtils.buildSegmentFromAvro(avroFiles.get(0),
offlineTableConfig, schema, "_with_move",
+ _segmentDir, _tarDir);
+
+ // First test standalone metadata push job runner
+ BaseSegmentPushJobRunner runner = new SegmentMetadataPushJobRunner();
+ SegmentGenerationJobSpec jobSpec = new SegmentGenerationJobSpec();
+ PushJobSpec pushJobSpec = new PushJobSpec();
+ pushJobSpec.setCopyToDeepStoreForMetadataPush(true);
+ jobSpec.setPushJobSpec(pushJobSpec);
+ PinotFSSpec fsSpec = new PinotFSSpec();
+ fsSpec.setScheme("file");
+ fsSpec.setClassName("org.apache.pinot.spi.filesystem.LocalPinotFS");
+ jobSpec.setPinotFSSpecs(Lists.newArrayList(fsSpec));
+ jobSpec.setOutputDirURI(_tarDir.getAbsolutePath());
+ TableSpec tableSpec = new TableSpec();
+ tableSpec.setTableName(DEFAULT_TABLE_NAME);
+
tableSpec.setTableConfigURI(_controllerRequestURLBuilder.forUpdateTableConfig(DEFAULT_TABLE_NAME));
+ jobSpec.setTableSpec(tableSpec);
+ PinotClusterSpec clusterSpec = new PinotClusterSpec();
+ clusterSpec.setControllerURI(_controllerBaseApiUrl);
+ jobSpec.setPinotClusterSpecs(new PinotClusterSpec[]{clusterSpec});
+
+ File dataDir = new File(_controllerConfig.getDataDir());
+ File dataDirSegments = new File(dataDir, DEFAULT_TABLE_NAME);
+
+ Assert.assertEquals(_tarDir.listFiles().length, 1);
+
+ runner.init(jobSpec);
+ runner.run();
+
+ // Segment should be seen in dataDir
+ Assert.assertTrue(dataDirSegments.exists());
+ Assert.assertEquals(dataDirSegments.listFiles().length, 1);
+ Assert.assertEquals(_tarDir.listFiles().length, 1);
+
+ // test segment loaded
+ JsonNode segmentsList = getSegmentsList();
+ Assert.assertEquals(segmentsList.size(), 1);
+ String segmentNameWithMove = segmentsList.get(0).asText();
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
