>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17806 )
Change subject: [ASTERIXDB-3269][EXT]: Handle root properly when computed field
is at first segment
......................................................................
[ASTERIXDB-3269][EXT]: Handle root properly when computed field is at first
segment
Change-Id: Idc97e6eef1d13953f16a37b340f4ba13983ecd74
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
M
asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
A
asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
A
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm
A
asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm
A
asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json
17 files changed, 265 insertions(+), 7 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/06/17806/1
diff --git
a/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json
b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json
new file mode 100644
index 0000000..4b16428
--- /dev/null
+++
b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json
@@ -0,0 +1 @@
+{ "id": 2 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json
b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json
new file mode 100644
index 0000000..7052c42
--- /dev/null
+++
b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json
@@ -0,0 +1 @@
+{ "id": 1 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index 0db9a2a..90f46ad 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -19,6 +19,7 @@
package org.apache.asterix.test.external_dataset;
import static
org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.BOM_FILE_CONTAINER;
+import static
org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.DYNAMIC_PREFIX_AT_START_CONTAINER;
import static
org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.FIXED_DATA_CONTAINER;
import static
org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil.BINARY_GEN_BASEDIR;
@@ -69,6 +70,7 @@
public static final int OVER_1000_OBJECTS_COUNT = 2999;
private static Uploader playgroundDataLoader;
+ private static Uploader dynamicPrefixAtStartDataLoader;
private static Uploader fixedDataLoader;
private static Uploader mixedDataLoader;
private static Uploader bomFileLoader;
@@ -118,9 +120,10 @@
TSV_DATA_PATH = tsvDataPath;
}
- public static void setUploaders(Uploader playgroundDataLoader, Uploader
fixedDataLoader, Uploader mixedDataLoader,
- Uploader bomFileLoader) {
+ public static void setUploaders(Uploader playgroundDataLoader, Uploader
dynamicPrefixAtStartDataLoader,
+ Uploader fixedDataLoader, Uploader mixedDataLoader, Uploader
bomFileLoader) {
ExternalDatasetTestUtils.playgroundDataLoader = playgroundDataLoader;
+ ExternalDatasetTestUtils.dynamicPrefixAtStartDataLoader =
dynamicPrefixAtStartDataLoader;
ExternalDatasetTestUtils.fixedDataLoader = fixedDataLoader;
ExternalDatasetTestUtils.mixedDataLoader = mixedDataLoader;
ExternalDatasetTestUtils.bomFileLoader = bomFileLoader;
@@ -158,6 +161,23 @@
}
/**
+ * Special container where dynamic prefix is the first segment
+ */
+ public static void prepareDynamicPrefixAtStartContainer() {
+ LOGGER.info("Loading dynamic prefix data to " +
DYNAMIC_PREFIX_AT_START_CONTAINER);
+
+ // Files data
+ String path =
+ Paths.get(JSON_DATA_PATH, "external-filter",
"computed-field-at-start", "foo-2023-01-01", "data.json")
+ .toString();
+ dynamicPrefixAtStartDataLoader.upload("foo-2023-01-01/data.json",
path, true, false);
+
+ path = Paths.get(JSON_DATA_PATH, "external-filter",
"computed-field-at-start", "bar-2023-01-01", "data.json")
+ .toString();
+ dynamicPrefixAtStartDataLoader.upload("bar-2023-01-01/data.json",
path, true, false);
+ }
+
+ /**
* This bucket is being filled by fixed data, a test is counting all
records in this bucket. If this bucket is
* changed, the test case will fail and its result will need to be updated
each time
*/
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 246ea13..532da56 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -115,6 +115,7 @@
static String ONLY_TESTS;
static String TEST_CONFIG_FILE_NAME;
static Runnable PREPARE_BUCKET;
+ static Runnable PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET;
static Runnable PREPARE_FIXED_DATA_BUCKET;
static Runnable PREPARE_MIXED_DATA_BUCKET;
static Runnable PREPARE_BOM_FILE_BUCKET;
@@ -144,6 +145,7 @@
protected TestCaseContext tcCtx;
public static final String PLAYGROUND_CONTAINER = "playground";
+ public static final String DYNAMIC_PREFIX_AT_START_CONTAINER =
"dynamic-prefix-at-start-container";
public static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not
use, has fixed data
public static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude";
public static final String BOM_FILE_CONTAINER = "bom-file-container";
@@ -151,6 +153,8 @@
public static final PutObjectRequest.Builder playgroundBuilder =
PutObjectRequest.builder().bucket(PLAYGROUND_CONTAINER);
+ public static final PutObjectRequest.Builder dynamicPrefixAtStartBuilder =
+
PutObjectRequest.builder().bucket(DYNAMIC_PREFIX_AT_START_CONTAINER);
public static final PutObjectRequest.Builder fixedDataBuilder =
PutObjectRequest.builder().bucket(FIXED_DATA_CONTAINER);
public static final PutObjectRequest.Builder includeExcludeBuilder =
@@ -166,7 +170,6 @@
}
// iceberg
-
private static final Schema SCHEMA =
new Schema(required(1, "id", Types.IntegerType.get()), required(2,
"data", Types.StringType.get()));
private static final Configuration CONF = new Configuration();
@@ -348,6 +351,7 @@
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
PREPARE_BUCKET = ExternalDatasetTestUtils::preparePlaygroundContainer;
+ PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET =
ExternalDatasetTestUtils::prepareDynamicPrefixAtStartContainer;
PREPARE_FIXED_DATA_BUCKET =
ExternalDatasetTestUtils::prepareFixedDataContainer;
PREPARE_MIXED_DATA_BUCKET =
ExternalDatasetTestUtils::prepareMixedDataContainer;
PREPARE_BOM_FILE_BUCKET =
ExternalDatasetTestUtils::prepareBomFileContainer;
@@ -397,6 +401,7 @@
.endpointOverride(endpoint);
client = builder.build();
client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
+
client.createBucket(CreateBucketRequest.builder().bucket(DYNAMIC_PREFIX_AT_START_CONTAINER).build());
client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_CONTAINER).build());
client.createBucket(CreateBucketRequest.builder().bucket(INCLUDE_EXCLUDE_CONTAINER).build());
client.createBucket(CreateBucketRequest.builder().bucket(BOM_FILE_CONTAINER).build());
@@ -405,9 +410,11 @@
// Create the bucket and upload some json files
setDataPaths(JSON_DATA_PATH, CSV_DATA_PATH, TSV_DATA_PATH);
- setUploaders(AwsS3ExternalDatasetTest::loadPlaygroundData,
AwsS3ExternalDatasetTest::loadFixedData,
+ setUploaders(AwsS3ExternalDatasetTest::loadPlaygroundData,
+ AwsS3ExternalDatasetTest::loadDynamicPrefixAtStartData,
AwsS3ExternalDatasetTest::loadFixedData,
AwsS3ExternalDatasetTest::loadMixedData,
AwsS3ExternalDatasetTest::loadBomData);
PREPARE_BUCKET.run();
+ PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET.run();
PREPARE_FIXED_DATA_BUCKET.run();
PREPARE_MIXED_DATA_BUCKET.run();
PREPARE_BOM_FILE_BUCKET.run();
@@ -418,6 +425,10 @@
client.putObject(playgroundBuilder.key(key).build(),
getRequestBody(content, fromFile, gzipped));
}
+ private static void loadDynamicPrefixAtStartData(String key, String
content, boolean fromFile, boolean gzipped) {
+ client.putObject(dynamicPrefixAtStartBuilder.key(key).build(),
getRequestBody(content, fromFile, gzipped));
+ }
+
private static void loadFixedData(String key, String content, boolean
fromFile, boolean gzipped) {
client.putObject(fixedDataBuilder.key(key).build(),
getRequestBody(content, fromFile, gzipped));
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
index 08f3816..29af77c 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
@@ -85,6 +85,7 @@
static String ONLY_TESTS;
static String TEST_CONFIG_FILE_NAME;
static Runnable PREPARE_PLAYGROUND_CONTAINER;
+ static Runnable PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER;
static Runnable PREPARE_FIXED_DATA_CONTAINER;
static Runnable PREPARE_INCLUDE_EXCLUDE_CONTAINER;
static Runnable PREPARE_BOM_FILE_BUCKET;
@@ -98,6 +99,7 @@
// Region, container and definitions
private static final String PLAYGROUND_CONTAINER = "playground";
+ private static final String DYNAMIC_PREFIX_AT_START_CONTAINER =
"dynamic-prefix-at-start-container";
private static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not
use, has fixed data
private static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude";
private static final String BOM_FILE_CONTAINER = "bom-file-container";
@@ -108,6 +110,7 @@
// Create a BlobServiceClient object which will be used to create a
container client
private static BlobServiceClient blobServiceClient;
private static BlobContainerClient playgroundContainer;
+ private static BlobContainerClient dynamicPrefixAtStartContainer;
private static BlobContainerClient publicAccessContainer;
private static BlobContainerClient fixedDataContainer;
private static BlobContainerClient mixedDataContainer;
@@ -140,6 +143,7 @@
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
PREPARE_PLAYGROUND_CONTAINER =
ExternalDatasetTestUtils::preparePlaygroundContainer;
+ PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER =
ExternalDatasetTestUtils::prepareDynamicPrefixAtStartContainer;
PREPARE_FIXED_DATA_CONTAINER =
ExternalDatasetTestUtils::prepareFixedDataContainer;
PREPARE_INCLUDE_EXCLUDE_CONTAINER =
ExternalDatasetTestUtils::prepareMixedDataContainer;
PREPARE_BOM_FILE_BUCKET =
ExternalDatasetTestUtils::prepareBomFileContainer;
@@ -177,6 +181,7 @@
LOGGER.info("Creating containers");
playgroundContainer =
blobServiceClient.createBlobContainer(PLAYGROUND_CONTAINER);
+ dynamicPrefixAtStartContainer =
blobServiceClient.createBlobContainer(DYNAMIC_PREFIX_AT_START_CONTAINER);
fixedDataContainer =
blobServiceClient.createBlobContainer(FIXED_DATA_CONTAINER);
mixedDataContainer =
blobServiceClient.createBlobContainer(INCLUDE_EXCLUDE_CONTAINER);
bomContainer =
blobServiceClient.createBlobContainer(BOM_FILE_CONTAINER);
@@ -195,9 +200,11 @@
// Create the bucket and upload some json files
setDataPaths(JSON_DATA_PATH, CSV_DATA_PATH, TSV_DATA_PATH);
setUploaders(AzureBlobStorageExternalDatasetTest::loadPlaygroundData,
+
AzureBlobStorageExternalDatasetTest::loadDynamicPrefixAtStartData,
AzureBlobStorageExternalDatasetTest::loadFixedData,
AzureBlobStorageExternalDatasetTest::loadMixedData,
AzureBlobStorageExternalDatasetTest::loadBomData);
PREPARE_PLAYGROUND_CONTAINER.run();
+ PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER.run();
PREPARE_FIXED_DATA_CONTAINER.run();
PREPARE_INCLUDE_EXCLUDE_CONTAINER.run();
PREPARE_BOM_FILE_BUCKET.run();
@@ -240,6 +247,35 @@
}
}
+ private static void loadDynamicPrefixAtStartData(String key, String
content, boolean fromFile, boolean gzipped) {
+ if (!fromFile) {
+ try (ByteArrayInputStream inputStream = new
ByteArrayInputStream(content.getBytes())) {
+
dynamicPrefixAtStartContainer.getBlobClient(key).upload(inputStream,
inputStream.available());
+ } catch (IOException ex) {
+ throw new IllegalArgumentException(ex.toString());
+ }
+ } else {
+ if (!gzipped) {
+
dynamicPrefixAtStartContainer.getBlobClient(key).uploadFromFile(content);
+ } else {
+ try (ByteArrayOutputStream byteArrayOutputStream = new
ByteArrayOutputStream();
+ GZIPOutputStream gzipOutputStream = new
GZIPOutputStream(byteArrayOutputStream)) {
+
gzipOutputStream.write(Files.readAllBytes(Paths.get(content)));
+ gzipOutputStream.close(); // Need to close or data will be
invalid
+ byte[] gzipBytes = byteArrayOutputStream.toByteArray();
+
+ try (ByteArrayInputStream inputStream = new
ByteArrayInputStream(gzipBytes)) {
+
dynamicPrefixAtStartContainer.getBlobClient(key).upload(inputStream,
inputStream.available());
+ } catch (IOException ex) {
+ throw new IllegalArgumentException(ex.toString());
+ }
+ } catch (IOException ex) {
+ throw new IllegalArgumentException(ex.toString());
+ }
+ }
+ }
+ }
+
private static void loadFixedData(String key, String content, boolean
fromFile, boolean gzipped) {
if (!fromFile) {
try (ByteArrayInputStream inputStream = new
ByteArrayInputStream(content.getBytes())) {
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp
new file mode 100644
index 0000000..cb78e58
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+};
+
+CREATE EXTERNAL DATASET test1(test) USING %adapter% (
+ %template%,
+ ("container"="dynamic-prefix-at-start-container"),
+ ("definition"="foo-{year:int}-{month:int}-{day:int}/"),
+ ("embed-filter-values" = "true"),
+ ("format"="json")
+);
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp
new file mode 100644
index 0000000..7d47884
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// param max-warnings:json=10
+
+USE test;
+
+SELECT value t
+FROM test1 t
+order by t.id;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp
new file mode 100644
index 0000000..36b2bab
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp
new file mode 100644
index 0000000..3f09568
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+};
+
+CREATE EXTERNAL DATASET test1(test) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="parquet-data/foo-{year:int}-{month:int}-{day:int}/"),
+ ("embed-filter-values" = "true"),
+ ("format"="parquet")
+);
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp
new file mode 100644
index 0000000..7d47884
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// param max-warnings:json=10
+
+USE test;
+
+SELECT value t
+FROM test1 t
+order by t.id;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp
new file mode 100644
index 0000000..36b2bab
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm
new file mode 100644
index 0000000..841ede8
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm
@@ -0,0 +1 @@
+{ "id": 1, "month": 1, "year": 2023, "day": 1 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm
new file mode 100644
index 0000000..8c86668
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm
@@ -0,0 +1 @@
+{ "id": 2, "month": 1, "year": 2023, "day": 1 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index 55764ed..5809912 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -302,6 +302,12 @@
<output-dir
compare="Text">computed-field-segment-pattern-mismatch</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="computed-field-at-start">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <output-dir compare="Text">computed-field-at-start</output-dir>
+ </compilation-unit>
+ </test-case>
<!--
<test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
<compilation-unit name="computed-field-segment-pattern-mismatch">
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 623e7bc..eb93ece 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -274,6 +274,12 @@
<output-dir
compare="Text">computed-field-segment-pattern-mismatch</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="computed-field-at-start">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">computed-field-at-start</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
<compilation-unit name="one-field">
<placeholder name="adapter" value="S3" />
@@ -332,6 +338,12 @@
<output-dir
compare="Text">computed-field-segment-pattern-mismatch</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
+ <compilation-unit name="computed-field-at-start">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">computed-field-at-start</output-dir>
+ </compilation-unit>
+ </test-case>
<!-- Dynamic prefixes tests end -->
<test-case FilePath="external-dataset">
<compilation-unit name="common/empty-string-definition">
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 299d0e4..2edf326 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -210,7 +210,7 @@
// remove last "/" and append it only if needed
root = builder.toString();
- root = root.substring(0, root.length() - 1);
+ root = root.isEmpty() ? root : root.substring(0, root.length() - 1);
root = ExternalDataUtils.appendSlash(root, endsWithSlash);
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 6902175..d282ab4 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -762,7 +762,7 @@
String definition =
configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
String subPath = configuration.get(ExternalDataConstants.SUBPATH);
- boolean hasRoot = root != null && !root.isEmpty();
+ boolean hasRoot = root != null;
boolean hasDefinition = definition != null && !definition.isEmpty();
boolean hasSubPath = subPath != null && !subPath.isEmpty();
@@ -794,7 +794,7 @@
}
public static String appendSlash(String string, boolean appendSlash) {
- return appendSlash ? string + (!string.endsWith("/") ? "/" : "") :
string;
+ return appendSlash && !string.isEmpty() ? string +
(!string.endsWith("/") ? "/" : "") : string;
}
/**
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17806
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Idc97e6eef1d13953f16a37b340f4ba13983ecd74
Gerrit-Change-Number: 17806
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
