chenboat commented on a change in pull request #5221: Add a new server api for
download of segments.
URL: https://github.com/apache/incubator-pinot/pull/5221#discussion_r405892246
##########
File path:
pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
##########
@@ -149,4 +160,62 @@ public void testSegmentCrcMetadata()
Assert.assertEquals(segmentsCrc.get(segmentName).asText(), crc);
}
}
+
+ @Test
+ public void testDownloadSegments()
+ throws Exception {
+ // Verify the content of the downloaded segment from a realtime table.
+ Assert.assertTrue(downLoadAndVerifySegmentContent(REALTIME_TABLE_NAME,
_realtimeIndexSegments.get(0)));
+ // Verify the content of the downloaded segment from an offline table.
+ Assert.assertTrue(downLoadAndVerifySegmentContent(OFFLINE_TABLE_NAME,
_offlineIndexSegments.get(0)));
+
+ // Verify non-existent table and segment download return NOT_FOUND status.
+ Response response =
_webTarget.path("/tables/UNKNOWN_REALTIME/segments/segmentname").request()
+ .get(Response.class);
+ Assert.assertEquals(response.getStatus(),
Response.Status.NOT_FOUND.getStatusCode());
+
+ response = _webTarget.path("/tables/" + REALTIME_TABLE_NAME +
"/segments/UNKNOWN_SEGMENT").request().get(Response.class);
+ Assert.assertEquals(response.getStatus(),
Response.Status.NOT_FOUND.getStatusCode());
+ }
+
+ // Verify metadata file from segments.
+ private boolean downLoadAndVerifySegmentContent(String tableNameWithType,
IndexSegment segment) {
+ String segmentPath = "/tables/" + tableNameWithType + "/segments/" +
segment.getSegmentName();
+
+ // Download the segment and save to a temp local file.
+ Response response =
_webTarget.path(segmentPath).request().get(Response.class);
+ Assert.assertEquals(response.getStatus(),
Response.Status.OK.getStatusCode());
+ File segmentFile = response.readEntity(File.class);
+
+ File tempMetadataDir = new File(FileUtils.getTempDirectory(),
"segment_metadata");
+ try (// Extract metadata.properties
+ InputStream metadataPropertiesInputStream = TarGzCompressionUtils
+ .unTarOneFile(new FileInputStream(segmentFile),
V1Constants.MetadataKeys.METADATA_FILE_NAME);
+ // Extract creation.meta
+ InputStream creationMetaInputStream = TarGzCompressionUtils
+ .unTarOneFile(new FileInputStream(segmentFile),
V1Constants.SEGMENT_CREATION_META)) {
+ Preconditions
+ .checkState(tempMetadataDir.mkdirs(), "Failed to create directory:
%s", tempMetadataDir.getAbsolutePath());
+
+ Preconditions.checkNotNull(metadataPropertiesInputStream, "%s does not
exist",
+ V1Constants.MetadataKeys.METADATA_FILE_NAME);
+ java.nio.file.Path metadataPropertiesPath = FileSystems.getDefault()
+ .getPath(tempMetadataDir.getAbsolutePath(),
V1Constants.MetadataKeys.METADATA_FILE_NAME);
+ Files.copy(metadataPropertiesInputStream, metadataPropertiesPath);
+
+ Preconditions.checkNotNull(creationMetaInputStream, "%s does not exist",
V1Constants.SEGMENT_CREATION_META);
+ java.nio.file.Path creationMetaPath =
+ FileSystems.getDefault().getPath(tempMetadataDir.getAbsolutePath(),
V1Constants.SEGMENT_CREATION_META);
+ Files.copy(creationMetaInputStream, creationMetaPath);
+ // Load segment metadata
+ SegmentMetadataImpl metadata = new SegmentMetadataImpl(tempMetadataDir);
+
+ Assert.assertEquals(tableNameWithType, metadata.getTableName());
+ return true;
+ } catch (Exception e) {
+ return false;
Review comment:
Done.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
