>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17763 )
Change subject: [NO ISSUE][EXT]: Correctly point to JSON files to convert to Parquet ...................................................................... [NO ISSUE][EXT]: Correctly point to JSON files to convert to Parquet Change-Id: Ia6d4531ffe202c4eeaff10652018c05952a564c3 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17763 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> --- M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/BinaryFileConverterUtil.java 4 files changed, 35 insertions(+), 7 deletions(-) Approvals: Wail Alkowaileet: Looks good to me, approved Hussain Towaileb: Looks good to me, but someone else must approve Jenkins: Verified; Verified diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java index c936fd2..f61ccbe 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java @@ -96,12 +96,19 @@ BinaryFileConverterUtil.cleanBinaryDirectory(basePath, BINARY_GEN_BASEDIR); //Convert files in DEFAULT_PARQUET_SRC_PATH to parquet BinaryFileConverterUtil.convertToParquet(basePath, parquetRawJsonDir, BINARY_GEN_BASEDIR); + } + + /** + * Generate binary files (e.g., parquet files) + */ + public static void createBinaryFilesRecursively(String dataToConvertDirPath) throws IOException { + //base path + File basePath = new File("."); // convert certain files related to dynamic prefixes - String dataPath = "data/json/external-filter"; - int startIndex = "data/json/".length(); // length to substring out of the final file name - BinaryFileConverterUtil.convertToParquetRecursively(basePath, dataPath, BINARY_GEN_BASEDIR, JSON_FILTER, - startIndex); + int startIndex = dataToConvertDirPath.indexOf("/external-filter"); + BinaryFileConverterUtil.convertToParquetRecursively(basePath, dataToConvertDirPath, BINARY_GEN_BASEDIR, + JSON_FILTER, startIndex); } public static void setDataPaths(String jsonDataPath, String csvDataPath, String tsvDataPath) { diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java index 0121b58..246ea13 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java @@ -19,6 +19,7 @@ package org.apache.asterix.test.external_dataset.aws; import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.createBinaryFiles; +import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.createBinaryFilesRecursively; import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.setDataPaths; import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.setUploaders; import static org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil.DEFAULT_PARQUET_SRC_PATH; @@ -106,6 +107,9 @@ private static final Logger LOGGER = LogManager.getLogger(); + private static final String PATH_BASE = joinPath("data"); + private static final String EXTERNAL_FILTER_DATA_PATH = joinPath(PATH_BASE, "json", "external-filter"); + // subclasses of this class MUST instantiate these variables before using them to avoid unexpected behavior static String SUITE_TESTS; static String ONLY_TESTS; @@ -186,6 +190,7 @@ final TestExecutor testExecutor = new AwsTestExecutor(); LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor); createBinaryFiles(DEFAULT_PARQUET_SRC_PATH); + createBinaryFilesRecursively(EXTERNAL_FILTER_DATA_PATH); setNcEndpoints(testExecutor); startAwsS3MockServer(); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java index 49f40ac..08f3816 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java @@ -94,6 +94,7 @@ private static final String CSV_DATA_PATH = joinPath("data", "csv"); private static final String TSV_DATA_PATH = joinPath("data", "tsv"); private static final String PARQUET_RAW_DATA_PATH = joinPath("data", "hdfs", "parquet"); + public static final String EXTERNAL_FILTER_DATA_PATH = joinPath("data", "json", "external-filter"); // Region, container and definitions private static final String PLAYGROUND_CONTAINER = "playground"; @@ -122,6 +123,7 @@ public static void setUp() throws Exception { final TestExecutor testExecutor = new AzureTestExecutor(); ExternalDatasetTestUtils.createBinaryFiles(PARQUET_RAW_DATA_PATH); + createBinaryFilesRecursively(EXTERNAL_FILTER_DATA_PATH); LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor); setNcEndpoints(testExecutor); createBlobServiceClient(); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/BinaryFileConverterUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/BinaryFileConverterUtil.java index 5cf4976..93ac24e 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/BinaryFileConverterUtil.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/BinaryFileConverterUtil.java @@ -71,11 +71,11 @@ ParquetFileExampleGeneratorUtil.writeExample(); } - public static void convertToParquetRecursively(File localDataRoot, String dirPath, String dest, - FilenameFilter filter, int startIndex) throws IOException { + public static void convertToParquetRecursively(File localDataRoot, String src, String dest, FilenameFilter filter, + int startIndex) throws IOException { File destPath = new File(localDataRoot, dest); - File dir = new File(dirPath); + File dir = new File(src); if (!dir.exists() || !dir.isDirectory()) { return; } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17763 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: Ia6d4531ffe202c4eeaff10652018c05952a564c3 Gerrit-Change-Number: 17763 Gerrit-PatchSet: 3 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
