This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new d755aef8c8 [ASTERIXDB-3269][EXT]: Handle root properly when computed
field is at first segment
d755aef8c8 is described below
commit d755aef8c8a6efc1a56705c2a6cc1ec05fec444e
Author: Hussain Towaileb <[email protected]>
AuthorDate: Mon Sep 25 18:16:16 2023 +0300
[ASTERIXDB-3269][EXT]: Handle root properly when computed field is at first
segment
Change-Id: Idc97e6eef1d13953f16a37b340f4ba13983ecd74
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17806
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Hussain Towaileb <[email protected]>
Reviewed-by: Ian Maxon <[email protected]>
---
.../bar-2023-01-01/data.json | 1 +
.../foo-2023-01-01/data.json | 1 +
.../external_dataset/ExternalDatasetTestUtils.java | 24 ++++++++++++--
.../aws/AwsS3ExternalDatasetOnePartitionTest.java | 5 +++
.../aws/AwsS3ExternalDatasetTest.java | 15 +++++++--
...BlobStorageExternalDatasetOnePartitionTest.java | 5 +++
.../AzureBlobStorageExternalDatasetTest.java | 37 ++++++++++++++++++++++
.../computed-field-at-start/test.000.ddl.sqlpp | 34 ++++++++++++++++++++
.../computed-field-at-start/test.010.query.sqlpp | 26 +++++++++++++++
.../computed-field-at-start/test.999.ddl.sqlpp | 20 ++++++++++++
.../computed-field-at-start/test.000.ddl.sqlpp | 34 ++++++++++++++++++++
.../computed-field-at-start/test.010.query.sqlpp | 26 +++++++++++++++
.../computed-field-at-start/test.999.ddl.sqlpp | 20 ++++++++++++
.../computed-field-at-start/result.010.adm | 1 +
.../parquet/computed-field-at-start/result.010.adm | 1 +
...stsuite_external_dataset_azure_blob_storage.xml | 6 ++++
.../runtimets/testsuite_external_dataset_s3.xml | 6 ++++
.../asterix/external/util/ExternalDataPrefix.java | 2 +-
.../asterix/external/util/ExternalDataUtils.java | 4 +--
19 files changed, 261 insertions(+), 7 deletions(-)
diff --git
a/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json
b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json
new file mode 100644
index 0000000000..4b16428c8a
--- /dev/null
+++
b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json
@@ -0,0 +1 @@
+{ "id": 2 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json
b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json
new file mode 100644
index 0000000000..7052c42c0d
--- /dev/null
+++
b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json
@@ -0,0 +1 @@
+{ "id": 1 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index 0db9a2ac8a..90f46ad51c 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -19,6 +19,7 @@
package org.apache.asterix.test.external_dataset;
import static
org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.BOM_FILE_CONTAINER;
+import static
org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.DYNAMIC_PREFIX_AT_START_CONTAINER;
import static
org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.FIXED_DATA_CONTAINER;
import static
org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil.BINARY_GEN_BASEDIR;
@@ -69,6 +70,7 @@ public class ExternalDatasetTestUtils {
public static final int OVER_1000_OBJECTS_COUNT = 2999;
private static Uploader playgroundDataLoader;
+ private static Uploader dynamicPrefixAtStartDataLoader;
private static Uploader fixedDataLoader;
private static Uploader mixedDataLoader;
private static Uploader bomFileLoader;
@@ -118,9 +120,10 @@ public class ExternalDatasetTestUtils {
TSV_DATA_PATH = tsvDataPath;
}
- public static void setUploaders(Uploader playgroundDataLoader, Uploader
fixedDataLoader, Uploader mixedDataLoader,
- Uploader bomFileLoader) {
+ public static void setUploaders(Uploader playgroundDataLoader, Uploader
dynamicPrefixAtStartDataLoader,
+ Uploader fixedDataLoader, Uploader mixedDataLoader, Uploader
bomFileLoader) {
ExternalDatasetTestUtils.playgroundDataLoader = playgroundDataLoader;
+ ExternalDatasetTestUtils.dynamicPrefixAtStartDataLoader =
dynamicPrefixAtStartDataLoader;
ExternalDatasetTestUtils.fixedDataLoader = fixedDataLoader;
ExternalDatasetTestUtils.mixedDataLoader = mixedDataLoader;
ExternalDatasetTestUtils.bomFileLoader = bomFileLoader;
@@ -157,6 +160,23 @@ public class ExternalDatasetTestUtils {
LOGGER.info("Files added successfully");
}
+ /**
+ * Special container where dynamic prefix is the first segment
+ */
+ public static void prepareDynamicPrefixAtStartContainer() {
+ LOGGER.info("Loading dynamic prefix data to " +
DYNAMIC_PREFIX_AT_START_CONTAINER);
+
+ // Files data
+ String path =
+ Paths.get(JSON_DATA_PATH, "external-filter",
"computed-field-at-start", "foo-2023-01-01", "data.json")
+ .toString();
+ dynamicPrefixAtStartDataLoader.upload("foo-2023-01-01/data.json",
path, true, false);
+
+ path = Paths.get(JSON_DATA_PATH, "external-filter",
"computed-field-at-start", "bar-2023-01-01", "data.json")
+ .toString();
+ dynamicPrefixAtStartDataLoader.upload("bar-2023-01-01/data.json",
path, true, false);
+ }
+
/**
* This bucket is being filled by fixed data, a test is counting all
records in this bucket. If this bucket is
* changed, the test case will fail and its result will need to be updated
each time
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
index c3f22a49e0..86d03a120e 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
@@ -44,6 +44,8 @@ public class AwsS3ExternalDatasetOnePartitionTest extends
AwsS3ExternalDatasetTe
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf";
PREPARE_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareS3Bucket;
+ PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET =
+
AwsS3ExternalDatasetOnePartitionTest::prepareDynamicPrefixAtStartContainer;
PREPARE_FIXED_DATA_BUCKET =
AwsS3ExternalDatasetOnePartitionTest::prepareFixedDataBucket;
PREPARE_MIXED_DATA_BUCKET =
AwsS3ExternalDatasetOnePartitionTest::prepareMixedDataBucket;
PREPARE_BOM_FILE_BUCKET =
AwsS3ExternalDatasetOnePartitionTest::prepareBomDataBucket;
@@ -54,6 +56,9 @@ public class AwsS3ExternalDatasetOnePartitionTest extends
AwsS3ExternalDatasetTe
private static void prepareS3Bucket() {
}
+ private static void prepareDynamicPrefixAtStartContainer() {
+ }
+
private static void prepareFixedDataBucket() {
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 246ea135d4..532da56cf4 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -115,6 +115,7 @@ public class AwsS3ExternalDatasetTest {
static String ONLY_TESTS;
static String TEST_CONFIG_FILE_NAME;
static Runnable PREPARE_BUCKET;
+ static Runnable PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET;
static Runnable PREPARE_FIXED_DATA_BUCKET;
static Runnable PREPARE_MIXED_DATA_BUCKET;
static Runnable PREPARE_BOM_FILE_BUCKET;
@@ -144,6 +145,7 @@ public class AwsS3ExternalDatasetTest {
protected TestCaseContext tcCtx;
public static final String PLAYGROUND_CONTAINER = "playground";
+ public static final String DYNAMIC_PREFIX_AT_START_CONTAINER =
"dynamic-prefix-at-start-container";
public static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not
use, has fixed data
public static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude";
public static final String BOM_FILE_CONTAINER = "bom-file-container";
@@ -151,6 +153,8 @@ public class AwsS3ExternalDatasetTest {
public static final PutObjectRequest.Builder playgroundBuilder =
PutObjectRequest.builder().bucket(PLAYGROUND_CONTAINER);
+ public static final PutObjectRequest.Builder dynamicPrefixAtStartBuilder =
+
PutObjectRequest.builder().bucket(DYNAMIC_PREFIX_AT_START_CONTAINER);
public static final PutObjectRequest.Builder fixedDataBuilder =
PutObjectRequest.builder().bucket(FIXED_DATA_CONTAINER);
public static final PutObjectRequest.Builder includeExcludeBuilder =
@@ -166,7 +170,6 @@ public class AwsS3ExternalDatasetTest {
}
// iceberg
-
private static final Schema SCHEMA =
new Schema(required(1, "id", Types.IntegerType.get()), required(2,
"data", Types.StringType.get()));
private static final Configuration CONF = new Configuration();
@@ -348,6 +351,7 @@ public class AwsS3ExternalDatasetTest {
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
PREPARE_BUCKET = ExternalDatasetTestUtils::preparePlaygroundContainer;
+ PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET =
ExternalDatasetTestUtils::prepareDynamicPrefixAtStartContainer;
PREPARE_FIXED_DATA_BUCKET =
ExternalDatasetTestUtils::prepareFixedDataContainer;
PREPARE_MIXED_DATA_BUCKET =
ExternalDatasetTestUtils::prepareMixedDataContainer;
PREPARE_BOM_FILE_BUCKET =
ExternalDatasetTestUtils::prepareBomFileContainer;
@@ -397,6 +401,7 @@ public class AwsS3ExternalDatasetTest {
.endpointOverride(endpoint);
client = builder.build();
client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
+
client.createBucket(CreateBucketRequest.builder().bucket(DYNAMIC_PREFIX_AT_START_CONTAINER).build());
client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_CONTAINER).build());
client.createBucket(CreateBucketRequest.builder().bucket(INCLUDE_EXCLUDE_CONTAINER).build());
client.createBucket(CreateBucketRequest.builder().bucket(BOM_FILE_CONTAINER).build());
@@ -405,9 +410,11 @@ public class AwsS3ExternalDatasetTest {
// Create the bucket and upload some json files
setDataPaths(JSON_DATA_PATH, CSV_DATA_PATH, TSV_DATA_PATH);
- setUploaders(AwsS3ExternalDatasetTest::loadPlaygroundData,
AwsS3ExternalDatasetTest::loadFixedData,
+ setUploaders(AwsS3ExternalDatasetTest::loadPlaygroundData,
+ AwsS3ExternalDatasetTest::loadDynamicPrefixAtStartData,
AwsS3ExternalDatasetTest::loadFixedData,
AwsS3ExternalDatasetTest::loadMixedData,
AwsS3ExternalDatasetTest::loadBomData);
PREPARE_BUCKET.run();
+ PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET.run();
PREPARE_FIXED_DATA_BUCKET.run();
PREPARE_MIXED_DATA_BUCKET.run();
PREPARE_BOM_FILE_BUCKET.run();
@@ -418,6 +425,10 @@ public class AwsS3ExternalDatasetTest {
client.putObject(playgroundBuilder.key(key).build(),
getRequestBody(content, fromFile, gzipped));
}
+ private static void loadDynamicPrefixAtStartData(String key, String
content, boolean fromFile, boolean gzipped) {
+ client.putObject(dynamicPrefixAtStartBuilder.key(key).build(),
getRequestBody(content, fromFile, gzipped));
+ }
+
private static void loadFixedData(String key, String content, boolean
fromFile, boolean gzipped) {
client.putObject(fixedDataBuilder.key(key).build(),
getRequestBody(content, fromFile, gzipped));
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java
index 59c375a1da..9f9e783586 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java
@@ -41,6 +41,8 @@ public class AzureBlobStorageExternalDatasetOnePartitionTest
extends AzureBlobSt
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf";
PREPARE_PLAYGROUND_CONTAINER =
AzureBlobStorageExternalDatasetOnePartitionTest::preparePlaygroundContainer;
+ PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER =
+
AzureBlobStorageExternalDatasetOnePartitionTest::prepareDynamicPrefixAtStartContainer;
PREPARE_FIXED_DATA_CONTAINER =
AzureBlobStorageExternalDatasetOnePartitionTest::prepareFixedDataContainer;
PREPARE_INCLUDE_EXCLUDE_CONTAINER =
AzureBlobStorageExternalDatasetOnePartitionTest::prepareMixedDataContainer;
PREPARE_BOM_FILE_BUCKET =
AzureBlobStorageExternalDatasetOnePartitionTest::prepareBomDataContainer;
@@ -50,6 +52,9 @@ public class AzureBlobStorageExternalDatasetOnePartitionTest
extends AzureBlobSt
private static void preparePlaygroundContainer() {
}
+ private static void prepareDynamicPrefixAtStartContainer() {
+ }
+
private static void prepareFixedDataContainer() {
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
index 08f3816a49..9858e562d3 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
@@ -85,6 +85,7 @@ public class AzureBlobStorageExternalDatasetTest {
static String ONLY_TESTS;
static String TEST_CONFIG_FILE_NAME;
static Runnable PREPARE_PLAYGROUND_CONTAINER;
+ static Runnable PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER;
static Runnable PREPARE_FIXED_DATA_CONTAINER;
static Runnable PREPARE_INCLUDE_EXCLUDE_CONTAINER;
static Runnable PREPARE_BOM_FILE_BUCKET;
@@ -98,6 +99,7 @@ public class AzureBlobStorageExternalDatasetTest {
// Region, container and definitions
private static final String PLAYGROUND_CONTAINER = "playground";
+ private static final String DYNAMIC_PREFIX_AT_START_CONTAINER =
"dynamic-prefix-at-start-container";
private static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not
use, has fixed data
private static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude";
private static final String BOM_FILE_CONTAINER = "bom-file-container";
@@ -108,6 +110,7 @@ public class AzureBlobStorageExternalDatasetTest {
// Create a BlobServiceClient object which will be used to create a
container client
private static BlobServiceClient blobServiceClient;
private static BlobContainerClient playgroundContainer;
+ private static BlobContainerClient dynamicPrefixAtStartContainer;
private static BlobContainerClient publicAccessContainer;
private static BlobContainerClient fixedDataContainer;
private static BlobContainerClient mixedDataContainer;
@@ -140,6 +143,7 @@ public class AzureBlobStorageExternalDatasetTest {
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
PREPARE_PLAYGROUND_CONTAINER =
ExternalDatasetTestUtils::preparePlaygroundContainer;
+ PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER =
ExternalDatasetTestUtils::prepareDynamicPrefixAtStartContainer;
PREPARE_FIXED_DATA_CONTAINER =
ExternalDatasetTestUtils::prepareFixedDataContainer;
PREPARE_INCLUDE_EXCLUDE_CONTAINER =
ExternalDatasetTestUtils::prepareMixedDataContainer;
PREPARE_BOM_FILE_BUCKET =
ExternalDatasetTestUtils::prepareBomFileContainer;
@@ -177,6 +181,7 @@ public class AzureBlobStorageExternalDatasetTest {
LOGGER.info("Creating containers");
playgroundContainer =
blobServiceClient.createBlobContainer(PLAYGROUND_CONTAINER);
+ dynamicPrefixAtStartContainer =
blobServiceClient.createBlobContainer(DYNAMIC_PREFIX_AT_START_CONTAINER);
fixedDataContainer =
blobServiceClient.createBlobContainer(FIXED_DATA_CONTAINER);
mixedDataContainer =
blobServiceClient.createBlobContainer(INCLUDE_EXCLUDE_CONTAINER);
bomContainer =
blobServiceClient.createBlobContainer(BOM_FILE_CONTAINER);
@@ -195,9 +200,11 @@ public class AzureBlobStorageExternalDatasetTest {
// Create the bucket and upload some json files
setDataPaths(JSON_DATA_PATH, CSV_DATA_PATH, TSV_DATA_PATH);
setUploaders(AzureBlobStorageExternalDatasetTest::loadPlaygroundData,
+
AzureBlobStorageExternalDatasetTest::loadDynamicPrefixAtStartData,
AzureBlobStorageExternalDatasetTest::loadFixedData,
AzureBlobStorageExternalDatasetTest::loadMixedData,
AzureBlobStorageExternalDatasetTest::loadBomData);
PREPARE_PLAYGROUND_CONTAINER.run();
+ PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER.run();
PREPARE_FIXED_DATA_CONTAINER.run();
PREPARE_INCLUDE_EXCLUDE_CONTAINER.run();
PREPARE_BOM_FILE_BUCKET.run();
@@ -240,6 +247,35 @@ public class AzureBlobStorageExternalDatasetTest {
}
}
+ private static void loadDynamicPrefixAtStartData(String key, String
content, boolean fromFile, boolean gzipped) {
+ if (!fromFile) {
+ try (ByteArrayInputStream inputStream = new
ByteArrayInputStream(content.getBytes())) {
+
dynamicPrefixAtStartContainer.getBlobClient(key).upload(inputStream,
inputStream.available());
+ } catch (IOException ex) {
+ throw new IllegalArgumentException(ex.toString());
+ }
+ } else {
+ if (!gzipped) {
+
dynamicPrefixAtStartContainer.getBlobClient(key).uploadFromFile(content);
+ } else {
+ try (ByteArrayOutputStream byteArrayOutputStream = new
ByteArrayOutputStream();
+ GZIPOutputStream gzipOutputStream = new
GZIPOutputStream(byteArrayOutputStream)) {
+
gzipOutputStream.write(Files.readAllBytes(Paths.get(content)));
+ gzipOutputStream.close(); // Need to close or data will be
invalid
+ byte[] gzipBytes = byteArrayOutputStream.toByteArray();
+
+ try (ByteArrayInputStream inputStream = new
ByteArrayInputStream(gzipBytes)) {
+
dynamicPrefixAtStartContainer.getBlobClient(key).upload(inputStream,
inputStream.available());
+ } catch (IOException ex) {
+ throw new IllegalArgumentException(ex.toString());
+ }
+ } catch (IOException ex) {
+ throw new IllegalArgumentException(ex.toString());
+ }
+ }
+ }
+ }
+
private static void loadFixedData(String key, String content, boolean
fromFile, boolean gzipped) {
if (!fromFile) {
try (ByteArrayInputStream inputStream = new
ByteArrayInputStream(content.getBytes())) {
@@ -417,6 +453,7 @@ public class AzureBlobStorageExternalDatasetTest {
private static void deleteContainersSilently() {
deleteContainerSilently(PLAYGROUND_CONTAINER);
+ deleteContainerSilently(DYNAMIC_PREFIX_AT_START_CONTAINER);
deleteContainerSilently(FIXED_DATA_CONTAINER);
deleteContainerSilently(PUBLIC_ACCESS_CONTAINER);
deleteContainerSilently(INCLUDE_EXCLUDE_CONTAINER);
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp
new file mode 100644
index 0000000000..cb78e58898
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+};
+
+CREATE EXTERNAL DATASET test1(test) USING %adapter% (
+ %template%,
+ ("container"="dynamic-prefix-at-start-container"),
+ ("definition"="foo-{year:int}-{month:int}-{day:int}/"),
+ ("embed-filter-values" = "true"),
+ ("format"="json")
+);
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp
new file mode 100644
index 0000000000..7d478849d0
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// param max-warnings:json=10
+
+USE test;
+
+SELECT value t
+FROM test1 t
+order by t.id;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp
new file mode 100644
index 0000000000..36b2bab543
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp
new file mode 100644
index 0000000000..3f09568175
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+};
+
+CREATE EXTERNAL DATASET test1(test) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="parquet-data/foo-{year:int}-{month:int}-{day:int}/"),
+ ("embed-filter-values" = "true"),
+ ("format"="parquet")
+);
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp
new file mode 100644
index 0000000000..7d478849d0
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// param max-warnings:json=10
+
+USE test;
+
+SELECT value t
+FROM test1 t
+order by t.id;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp
new file mode 100644
index 0000000000..36b2bab543
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm
new file mode 100644
index 0000000000..841ede8c27
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm
@@ -0,0 +1 @@
+{ "id": 1, "month": 1, "year": 2023, "day": 1 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm
new file mode 100644
index 0000000000..8c86668a98
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm
@@ -0,0 +1 @@
+{ "id": 2, "month": 1, "year": 2023, "day": 1 }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index 55764eded2..5809912a46 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -302,6 +302,12 @@
<output-dir
compare="Text">computed-field-segment-pattern-mismatch</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="computed-field-at-start">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <output-dir compare="Text">computed-field-at-start</output-dir>
+ </compilation-unit>
+ </test-case>
<!--
<test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
<compilation-unit name="computed-field-segment-pattern-mismatch">
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 623e7bc112..54758e3e6b 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -274,6 +274,12 @@
<output-dir
compare="Text">computed-field-segment-pattern-mismatch</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="computed-field-at-start">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">computed-field-at-start</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
<compilation-unit name="one-field">
<placeholder name="adapter" value="S3" />
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 299d0e4e55..2edf326ff0 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -210,7 +210,7 @@ public final class ExternalDataPrefix implements
Serializable {
// remove last "/" and append it only if needed
root = builder.toString();
- root = root.substring(0, root.length() - 1);
+ root = root.isEmpty() ? root : root.substring(0, root.length() - 1);
root = ExternalDataUtils.appendSlash(root, endsWithSlash);
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 69021759cc..d282ab41a8 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -762,7 +762,7 @@ public class ExternalDataUtils {
String definition =
configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
String subPath = configuration.get(ExternalDataConstants.SUBPATH);
- boolean hasRoot = root != null && !root.isEmpty();
+ boolean hasRoot = root != null;
boolean hasDefinition = definition != null && !definition.isEmpty();
boolean hasSubPath = subPath != null && !subPath.isEmpty();
@@ -794,7 +794,7 @@ public class ExternalDataUtils {
}
public static String appendSlash(String string, boolean appendSlash) {
- return appendSlash ? string + (!string.endsWith("/") ? "/" : "") :
string;
+ return appendSlash && !string.isEmpty() ? string +
(!string.endsWith("/") ? "/" : "") : string;
}
/**
