KKcorps commented on a change in pull request #5249:
URL: https://github.com/apache/incubator-pinot/pull/5249#discussion_r412459771
##########
File path:
pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
##########
@@ -0,0 +1,279 @@
+package org.apache.pinot.plugin.filesystem;
+
+import com.adobe.testing.s3mock.testng.S3Mock;
+import java.io.File;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.io.IOUtils;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Listeners;
+import org.testng.annotations.Test;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+
+@Test
+@Listeners(com.adobe.testing.s3mock.testng.S3MockListener.class)
+public class S3PinotFSTest {
+ final String DELIMITER = "/";
+ S3PinotFS _s3PinotFS;
+ S3Client _s3Client;
+ final String BUCKET = "test-bucket";
+ final String SCHEME = "s3";
+ final String FILE_FORMAT = "%s://%s/%s";
+ final String DIR_FORMAT = "%s://%s";
+
+ @BeforeMethod
+ public void setUp() {
+ S3Mock s3Mock = S3Mock.getInstance();
+ _s3Client = s3Mock.createS3ClientV2();
+ _s3PinotFS = new S3PinotFS();
+ _s3PinotFS.init(_s3Client);
+
_s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build());
+ }
+
+ private void createEmptyFile(String folderName, String fileName) {
+ String fileNameWithFolder = folderName + DELIMITER + fileName;
+ _s3Client
+ .putObject(S3TestUtils.getPutObjectRequest(BUCKET,
fileNameWithFolder), RequestBody.fromBytes(new byte[0]));
+ }
+
+ @Test
+ public void testTouchFileInBucket()
+ throws Exception {
+
+ String[] originalFiles = new String[]{"a-touch.txt", "b-touch.txt",
"c-touch.txt"};
+
+ for (String fileName : originalFiles) {
+ _s3PinotFS.touch(URI.create(String.format(FILE_FORMAT, SCHEME, BUCKET,
fileName)));
+ }
+ ListObjectsV2Response listObjectsV2Response =
+ _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, "",
true));
+
+ String[] response =
listObjectsV2Response.contents().stream().map(S3Object::key).filter(x ->
x.contains("touch"))
+ .toArray(String[]::new);
+
+ Assert.assertEquals(response.length, originalFiles.length);
+ Assert.assertTrue(Arrays.equals(response, originalFiles));
+ }
+
+ @Test
+ public void testTouchFilesInFolder()
+ throws Exception {
+
+ String folder = "my-files";
+ String[] originalFiles = new String[]{"a-touch.txt", "b-touch.txt",
"c-touch.txt"};
+
+ for (String fileName : originalFiles) {
+ String fileNameWithFolder = folder + DELIMITER + fileName;
+ _s3PinotFS.touch(URI.create(String.format(FILE_FORMAT, SCHEME, BUCKET,
fileNameWithFolder)));
+ }
+ ListObjectsV2Response listObjectsV2Response =
+ _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET,
folder, false));
+
+ String[] response =
listObjectsV2Response.contents().stream().map(S3Object::key).filter(x ->
x.contains("touch"))
+ .toArray(String[]::new);
+ Assert.assertEquals(response.length, originalFiles.length);
+
+ Assert.assertTrue(Arrays.equals(response,
Arrays.stream(originalFiles).map(x -> folder + DELIMITER + x).toArray()));
+ }
+
+ public void testListFilesInBucketNonRecursive()
Review comment:
done
##########
File path:
pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
##########
@@ -0,0 +1,279 @@
+package org.apache.pinot.plugin.filesystem;
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
