This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new afd4c953cf Adds check if controller tmp dir exists (#14503)
afd4c953cf is described below
commit afd4c953cfce34d451e204b27a1915541c80e8ed
Author: NOOB <[email protected]>
AuthorDate: Sat Nov 30 06:09:05 2024 +0530
Adds check if controller tmp dir exists (#14503)
* Adds check if tmp dir exists
* use java.nio
* Creates parent dirs as well
* Adds tests
* Adds test for upload segment
* Addresses PR comments
* removes unused imports
* deletes dir in teardown
* removes formating
* pinot-common/src/main/java/org/apache/pinot/common/utils/FileUtils.java
* nit
* nit
* nit
* fixes lint
* Addresses PR comment
---
.../org/apache/pinot/common/utils/FileUtils.java | 13 +++++
.../api/resources/ControllerFilePathProvider.java | 3 +
.../api/ControllerFilePathProviderTest.java | 43 +++++++++++++++
...otSegmentUploadDownloadRestletResourceTest.java | 64 ++++++++++++++++++++++
4 files changed, 123 insertions(+)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUtils.java
index 2c1cb10c43..62df434128 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/FileUtils.java
@@ -22,6 +22,8 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.Arrays;
@@ -134,4 +136,15 @@ public class FileUtils {
return filePath;
}
+
+ public static void ensureDirectoryExists(Path path)
+ throws IllegalArgumentException {
+ if (!Files.exists(path)) {
+ try {
+ Files.createDirectories(path);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ControllerFilePathProvider.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ControllerFilePathProvider.java
index 67312ffdf9..b9d38341b7 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ControllerFilePathProvider.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ControllerFilePathProvider.java
@@ -126,14 +126,17 @@ public class ControllerFilePathProvider {
}
public File getFileUploadTempDir() {
+
org.apache.pinot.common.utils.FileUtils.ensureDirectoryExists(_fileUploadTempDir.toPath());
return _fileUploadTempDir;
}
public File getUntarredFileTempDir() {
+
org.apache.pinot.common.utils.FileUtils.ensureDirectoryExists(_untarredFileTempDir.toPath());
return _untarredFileTempDir;
}
public File getFileDownloadTempDir() {
+
org.apache.pinot.common.utils.FileUtils.ensureDirectoryExists(_fileDownloadTempDir.toPath());
return _fileDownloadTempDir;
}
}
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/ControllerFilePathProviderTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/ControllerFilePathProviderTest.java
index 8151e0ad59..3ec6c85ef8 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/ControllerFilePathProviderTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/ControllerFilePathProviderTest.java
@@ -107,6 +107,49 @@ public class ControllerFilePathProviderTest {
FileUtils.forceDelete(DATA_DIR);
}
+ @Test
+ public void testLocalTempMissing()
+ throws Exception {
+ FileUtils.deleteQuietly(DATA_DIR);
+ PinotFSFactory.init(new PinotConfiguration());
+
+ ControllerConf controllerConf = new ControllerConf();
+ controllerConf.setControllerHost(HOST);
+ controllerConf.setControllerPort(PORT);
+ controllerConf.setDataDir(DATA_DIR.getPath());
+ controllerConf.setLocalTempDir(LOCAL_TEMP_DIR.getPath());
+ ControllerFilePathProvider.init(controllerConf);
+ ControllerFilePathProvider provider =
ControllerFilePathProvider.getInstance();
+
+ File fileUploadTempDir = provider.getFileUploadTempDir();
+ assertEquals(fileUploadTempDir, new File(LOCAL_TEMP_DIR,
"fileUploadTemp"));
+ checkDirExistAndEmpty(fileUploadTempDir);
+
+ File untarredFileTempDir = provider.getUntarredFileTempDir();
+ assertEquals(untarredFileTempDir, new File(LOCAL_TEMP_DIR,
"untarredFileTemp"));
+ checkDirExistAndEmpty(untarredFileTempDir);
+
+ File fileDownloadTempDir = provider.getFileDownloadTempDir();
+ assertEquals(fileDownloadTempDir, new File(LOCAL_TEMP_DIR,
"fileDownloadTemp"));
+ checkDirExistAndEmpty(fileDownloadTempDir);
+
+ FileUtils.deleteQuietly(fileUploadTempDir);
+ FileUtils.deleteQuietly(untarredFileTempDir);
+ FileUtils.deleteQuietly(fileDownloadTempDir);
+
+ fileUploadTempDir = provider.getFileUploadTempDir();
+ assertEquals(fileUploadTempDir, new File(LOCAL_TEMP_DIR,
"fileUploadTemp"));
+ checkDirExistAndEmpty(fileUploadTempDir);
+
+ untarredFileTempDir = provider.getUntarredFileTempDir();
+ assertEquals(untarredFileTempDir, new File(LOCAL_TEMP_DIR,
"untarredFileTemp"));
+ checkDirExistAndEmpty(untarredFileTempDir);
+
+ fileDownloadTempDir = provider.getFileDownloadTempDir();
+ assertEquals(fileDownloadTempDir, new File(LOCAL_TEMP_DIR,
"fileDownloadTemp"));
+ checkDirExistAndEmpty(fileDownloadTempDir);
+ }
+
private void checkDirExistAndEmpty(File dir) {
assertTrue(dir.isDirectory());
String[] children = dir.list();
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResourceTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResourceTest.java
index 9a074fa4a7..60aaa346d4 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResourceTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResourceTest.java
@@ -24,6 +24,8 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -31,6 +33,7 @@ import java.util.UUID;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.utils.TarCompressionUtils;
+import org.apache.pinot.controller.ControllerConf;
import
org.apache.pinot.controller.api.exception.ControllerApplicationException;
import org.apache.pinot.controller.api.upload.SegmentMetadataInfo;
import org.apache.pinot.spi.crypt.NoOpPinotCrypter;
@@ -38,6 +41,7 @@ import org.apache.pinot.spi.crypt.PinotCrypterFactory;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.glassfish.jersey.media.multipart.BodyPart;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.file.FileDataBodyPart;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
@@ -56,6 +60,11 @@ public class PinotSegmentUploadDownloadRestletResourceTest {
private static final String TABLE_NAME = "table_abc";
private static final String SEGMENT_NAME = "segment_xyz";
+ private static final String HOST = "localhost";
+ private static final String PORT = "12345";
+ private static final File DATA_DIR =
+ new File(FileUtils.getTempDirectory(),
"PinotSegmentUploadDownloadRestletResourceTest");
+ private static final File LOCAL_TEMP_DIR = new File(DATA_DIR, "localTemp");
private PinotSegmentUploadDownloadRestletResource _resource = new
PinotSegmentUploadDownloadRestletResource();
private File _encryptedFile;
@@ -71,6 +80,7 @@ public class PinotSegmentUploadDownloadRestletResourceTest {
@AfterMethod
public void tearDown() throws IOException {
FileUtils.deleteDirectory(_tempDir);
+ FileUtils.deleteDirectory(DATA_DIR);
}
@BeforeClass
@@ -249,4 +259,58 @@ public class PinotSegmentUploadDownloadRestletResourceTest
{
// validate – should not throw exception
PinotSegmentUploadDownloadRestletResource.validateMultiPartForBatchSegmentUpload(bodyParts);
}
+
+ @Test
+ public void testCreateSegmentFileFromMultipart()
+ throws NoSuchMethodException, InvalidControllerConfigException,
IOException {
+ PinotSegmentUploadDownloadRestletResource resource = new
PinotSegmentUploadDownloadRestletResource();
+ Class<?> clazz = resource.getClass();
+
+ FormDataMultiPart mockFormDataMultiPart = mock(FormDataMultiPart.class);
+ // Mock input stream to return the test content
+ InputStream mockInputStream = new ByteArrayInputStream("This is a test
content".getBytes());
+ FormDataBodyPart mockBodyPart = mock(FormDataBodyPart.class);
+
when(mockBodyPart.getValueAs(InputStream.class)).thenReturn(mockInputStream);
+
+ Map<String, List<FormDataBodyPart>> map = Map.of(
+ "test", new ArrayList<>(List.of(mockBodyPart))
+ );
+ when(mockFormDataMultiPart.getFields()).thenReturn(map);
+
+ ControllerConf controllerConf = new ControllerConf();
+ controllerConf.setControllerHost(HOST);
+ controllerConf.setControllerPort(PORT);
+ controllerConf.setDataDir(DATA_DIR.getPath());
+ controllerConf.setLocalTempDir(LOCAL_TEMP_DIR.getPath());
+ ControllerFilePathProvider.init(controllerConf);
+
+ ControllerFilePathProvider provider =
ControllerFilePathProvider.getInstance();
+
+ FileUtils.deleteDirectory(provider.getFileUploadTempDir());
+ String tempFileName = "tmp-" + UUID.randomUUID();
+ File tempDecryptedFile = new File(provider.getFileUploadTempDir(),
tempFileName);
+
+ Method createSegmentFileFromMultipartMethod =
+ clazz.getDeclaredMethod("createSegmentFileFromMultipart",
FormDataMultiPart.class, File.class);
+ createSegmentFileFromMultipartMethod.setAccessible(true);
+
+ try {
+ createSegmentFileFromMultipartMethod.invoke(resource,
mockFormDataMultiPart, tempDecryptedFile);
+ } catch (Exception e) {
+ throw new AssertionError("Method threw an exception: " + e.getMessage(),
e);
+ }
+
+ File tempDir = provider.getFileUploadTempDir();
+ File parentOfTempDir = tempDir.getParentFile();
+ assert parentOfTempDir != null;
+ FileUtils.deleteDirectory(parentOfTempDir);
+
+ tempFileName = "tmp-" + UUID.randomUUID();
+ tempDecryptedFile = new File(provider.getFileUploadTempDir(),
tempFileName);
+ try {
+ createSegmentFileFromMultipartMethod.invoke(resource,
mockFormDataMultiPart, tempDecryptedFile);
+ } catch (Exception e) {
+ throw new AssertionError("Method threw an exception: " + e.getMessage(),
e);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
