This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 899ce953ca [ASTERIXDB-3372][COMP]: Validate minimum value for
'max-objects-per-file'
899ce953ca is described below
commit 899ce953ca0753cd7f796c1a3329c3865aad8a09
Author: Hussain Towaileb <hussain.towai...@couchbase.com>
AuthorDate: Mon Apr 1 20:31:24 2024 +0300
[ASTERIXDB-3372][COMP]: Validate minimum value for 'max-objects-per-file'
Change-Id: I13c4a516fa8cf76166c2b04aca4ce5c06fd69aa1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18216
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussai...@gmail.com>
Reviewed-by: Wail Alkowaileet <wael....@gmail.com>
---
.../empty-over.01.ddl.sqlpp | 39 ++++++++++++++++++++++
.../empty-over.02.update.sqlpp | 36 ++++++++++++++++++++
.../empty-over.03.update.sqlpp | 36 ++++++++++++++++++++
.../runtimets/testsuite_external_dataset_s3.xml | 7 ++++
.../asterix/common/exceptions/ErrorCode.java | 1 +
.../src/main/resources/asx_errormsg/en.properties | 1 +
.../external/util/ExternalDataConstants.java | 1 +
.../external/util/WriterValidationUtil.java | 12 +++++--
8 files changed, 130 insertions(+), 3 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.01.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.01.ddl.sqlpp
new file mode 100644
index 0000000000..b68c38b195
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.01.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001";),
+ ("container"="playground"),
+
("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.02.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.02.update.sqlpp
new file mode 100644
index 0000000000..02f6c96f1d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.02.update.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("copy-to-result-with-over")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001";,
+ "container":"playground",
+ "format":"json",
+ "max-objects-per-file": 2
+}
+
+
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.03.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.03.update.sqlpp
new file mode 100644
index 0000000000..c0add711e4
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.03.update.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("copy-to-result-with-over")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001";,
+ "container":"playground",
+ "format":"json",
+ "max-objects-per-file": "hello"
+}
+
+
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 1e90f98bca..ef1e6a0bc8 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -90,6 +90,13 @@
<expected-error>ASX1001: Syntax error: OVER-clause cannot be
empty</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="bad-max-objects-per-file">
+ <output-dir compare="Text">bad-max-objects-per-file</output-dir>
+ <expected-error>Minimum value allowed for 'max-objects-per-file' is
1000. Found 2</expected-error>
+ <expected-error>Expected integer value, got hello</expected-error>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="aws-s3-external-dataset">
<test-case FilePath="external-dataset">
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 8994fc31db..4d3b57bf1c 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -299,6 +299,7 @@ public enum ErrorCode implements IError {
UNSUPPORTED_INDEX_IN_COLUMNAR_FORMAT(1194),
COMPOSITE_KEY_NOT_SUPPORTED(1195),
EXTERNAL_SINK_ERROR(1196),
+ MINIMUM_VALUE_ALLOWED_FOR_PARAM(1197),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 8d624d4a6f..72bb7888e6 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -301,6 +301,7 @@
1194 = Index type '%1$s' is not supported with columnar storage format
1195 = Composite key is not allowed, only single key must be specified.
1196 = External sink error. %1$s
+1197 = Minimum value allowed for '%1$s' is %2$s. Found %3$s
# Feed Errors
3001 = Illegal state.
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 79252ada33..d56924faf6 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -312,6 +312,7 @@ public class ExternalDataConstants {
public static final String KEY_WRITER_MAX_RESULT = "max-objects-per-file";
public static final String KEY_WRITER_COMPRESSION = "compression";
public static final int WRITER_MAX_RESULT_DEFAULT = 10000;
+ public static final int WRITER_MAX_RESULT_MINIMUM = 1000;
public static final Set<String> WRITER_SUPPORTED_FORMATS;
public static final Set<String> WRITER_SUPPORTED_ADAPTERS;
public static final Set<String> WRITER_SUPPORTED_COMPRESSION;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
index b348c1e21c..843600e6e6 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
@@ -19,7 +19,9 @@
package org.apache.asterix.external.util;
import static
org.apache.asterix.common.exceptions.ErrorCode.INVALID_REQ_PARAM_VAL;
-import static
org.apache.asterix.external.util.ExternalDataConstants.KEY_FORMAT;
+import static
org.apache.asterix.common.exceptions.ErrorCode.MINIMUM_VALUE_ALLOWED_FOR_PARAM;
+import static
org.apache.asterix.external.util.ExternalDataConstants.KEY_WRITER_MAX_RESULT;
+import static
org.apache.asterix.external.util.ExternalDataConstants.WRITER_MAX_RESULT_MINIMUM;
import java.util.List;
import java.util.Map;
@@ -69,13 +71,17 @@ public class WriterValidationUtil {
private static void validateMaxResult(Map<String, String> configuration,
SourceLocation sourceLocation)
throws CompilationException {
- String maxResult =
configuration.get(ExternalDataConstants.KEY_WRITER_MAX_RESULT);
+ String maxResult = configuration.get(KEY_WRITER_MAX_RESULT);
if (maxResult == null) {
return;
}
try {
- Integer.parseInt(maxResult);
+ int value = Integer.parseInt(maxResult);
+ if (value < WRITER_MAX_RESULT_MINIMUM) {
+ throw new
CompilationException(MINIMUM_VALUE_ALLOWED_FOR_PARAM, KEY_WRITER_MAX_RESULT,
+ WRITER_MAX_RESULT_MINIMUM, value);
+ }
} catch (NumberFormatException e) {
throw
CompilationException.create(ErrorCode.INTEGER_VALUE_EXPECTED, sourceLocation,
maxResult);
}
