>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17763 )
Change subject: [NO ISSUE][EXT]: Correctly point to JSON files to convert to
Parquet
......................................................................
[NO ISSUE][EXT]: Correctly point to JSON files to convert to Parquet
Change-Id: Ia6d4531ffe202c4eeaff10652018c05952a564c3
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/BinaryFileConverterUtil.java
3 files changed, 28 insertions(+), 7 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/63/17763/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index c936fd2..f61ccbe 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -96,12 +96,19 @@
BinaryFileConverterUtil.cleanBinaryDirectory(basePath,
BINARY_GEN_BASEDIR);
//Convert files in DEFAULT_PARQUET_SRC_PATH to parquet
BinaryFileConverterUtil.convertToParquet(basePath, parquetRawJsonDir,
BINARY_GEN_BASEDIR);
+ }
+
+ /**
+ * Generate binary files (e.g., parquet files)
+ */
+ public static void createBinaryFilesRecursively(String
dataToConvertDirPath) throws IOException {
+ //base path
+ File basePath = new File(".");
// convert certain files related to dynamic prefixes
- String dataPath = "data/json/external-filter";
- int startIndex = "data/json/".length(); // length to substring out of
the final file name
- BinaryFileConverterUtil.convertToParquetRecursively(basePath,
dataPath, BINARY_GEN_BASEDIR, JSON_FILTER,
- startIndex);
+ int startIndex = dataToConvertDirPath.indexOf("/external-filter");
+ BinaryFileConverterUtil.convertToParquetRecursively(basePath,
dataToConvertDirPath, BINARY_GEN_BASEDIR,
+ JSON_FILTER, startIndex);
}
public static void setDataPaths(String jsonDataPath, String csvDataPath,
String tsvDataPath) {
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 0121b58..246ea13 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -19,6 +19,7 @@
package org.apache.asterix.test.external_dataset.aws;
import static
org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.createBinaryFiles;
+import static
org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.createBinaryFilesRecursively;
import static
org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.setDataPaths;
import static
org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.setUploaders;
import static
org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil.DEFAULT_PARQUET_SRC_PATH;
@@ -106,6 +107,9 @@
private static final Logger LOGGER = LogManager.getLogger();
+ private static final String PATH_BASE = joinPath("data");
+ private static final String EXTERNAL_FILTER_DATA_PATH =
joinPath(PATH_BASE, "json", "external-filter");
+
// subclasses of this class MUST instantiate these variables before using
them to avoid unexpected behavior
static String SUITE_TESTS;
static String ONLY_TESTS;
@@ -186,6 +190,7 @@
final TestExecutor testExecutor = new AwsTestExecutor();
LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor);
createBinaryFiles(DEFAULT_PARQUET_SRC_PATH);
+ createBinaryFilesRecursively(EXTERNAL_FILTER_DATA_PATH);
setNcEndpoints(testExecutor);
startAwsS3MockServer();
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/BinaryFileConverterUtil.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/BinaryFileConverterUtil.java
index 5cf4976..93ac24e 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/BinaryFileConverterUtil.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/parquet/BinaryFileConverterUtil.java
@@ -71,11 +71,11 @@
ParquetFileExampleGeneratorUtil.writeExample();
}
- public static void convertToParquetRecursively(File localDataRoot, String
dirPath, String dest,
- FilenameFilter filter, int startIndex) throws IOException {
+ public static void convertToParquetRecursively(File localDataRoot, String
src, String dest, FilenameFilter filter,
+ int startIndex) throws IOException {
File destPath = new File(localDataRoot, dest);
- File dir = new File(dirPath);
+ File dir = new File(src);
if (!dir.exists() || !dir.isDirectory()) {
return;
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17763
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ia6d4531ffe202c4eeaff10652018c05952a564c3
Gerrit-Change-Number: 17763
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
