[asterixdb] 07/25: [NO ISSUE][COMP] Additional plan sanity check
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit c65fa869af8af4f1964d0b88fd6376c220477785
Author: Dmitry Lychagin
AuthorDate: Fri Feb 5 19:26:50 2021 -0800
[NO ISSUE][COMP] Additional plan sanity check
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add plan sanity check to ensure that operator's
produced variables are disjoint from its used variables
Change-Id: Id4800f2fff66c9ace89889ab89a6b7f442cd7a0e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9847
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Dmitry Lychagin
Reviewed-by: Ali Alsuliman
---
.../inverted-index-join-noeqjoin/ngram-edit-distance-inline.sqlpp | 2 ++
.../inverted-index-join-noeqjoin/ngram-fuzzyeq-edit-distance.sqlpp | 2 +-
.../inverted-index-join-noeqjoin/olist-edit-distance-inline.sqlpp | 2 ++
...leftouterjoin-probe-pidx-with-join-edit-distance-check-idx_01.sqlpp | 2 ++
...touterjoin-probe-pidx-with-join-edit-distance-check-idx_01_ps.sqlpp | 2 +-
.../queries/inverted-index-join/ngram-edit-distance-check_02.sqlpp | 2 ++
.../queries/inverted-index-join/ngram-edit-distance-check_03.sqlpp | 2 ++
.../queries/inverted-index-join/ngram-edit-distance-check_04.sqlpp | 2 ++
.../queries/inverted-index-join/ngram-edit-distance_02.sqlpp | 2 ++
.../queries/inverted-index-join/ngram-edit-distance_03.sqlpp | 2 ++
.../queries/inverted-index-join/ngram-edit-distance_04.sqlpp | 2 ++
.../queries/inverted-index-join/ngram-fuzzyeq-edit-distance_01.sqlpp | 2 +-
.../queries/inverted-index-join/ngram-fuzzyeq-edit-distance_03.sqlpp | 2 +-
.../queries/inverted-index-join/olist-edit-distance-check_02.sqlpp | 2 ++
.../queries/inverted-index-join/olist-edit-distance-check_03.sqlpp | 2 ++
.../queries/inverted-index-join/olist-edit-distance-check_04.sqlpp | 2 ++
.../queries/inverted-index-join/olist-edit-distance_02.sqlpp | 2 ++
.../queries/inverted-index-join/olist-edit-distance_03.sqlpp | 2 ++
.../queries/inverted-index-join/olist-edit-distance_04.sqlpp | 2 ++
.../queries/inverted-index-join/olist-fuzzyeq-edit-distance_02.sqlpp | 2 +-
.../queries/inverted-index-join/olist-fuzzyeq-edit-distance_03.sqlpp | 2 +-
...touterjoin-probe-pidx-with-join-edit-distance-check-idx_01_ps.sqlpp | 1 +
...touterjoin-probe-pidx-with-join-edit-distance-check-idx_01_ps.sqlpp | 1 +
...touterjoin-probe-pidx-with-join-edit-distance-check-idx_01_ps.sqlpp | 1 +
.../probe-pidx-with-join-invidx-sidx2.3.query.sqlpp| 1 +
.../probe-pidx-with-join-invidx-sidx2.3.query.sqlpp| 1 +
.../probe-pidx-with-join-invidx-sidx2.3.query.sqlpp| 1 +
.../probe-pidx-with-join-invidx-sidx2.3.query.sqlpp| 1 +
.../probe-pidx-with-join-invidx-sidx2.3.ast| 1 +
.../probe-pidx-with-join-invidx-sidx2.3.ast| 1 +
.../probe-pidx-with-join-invidx-sidx2.3.ast| 1 +
.../probe-pidx-with-join-invidx-sidx2.3.ast| 1 +
.../hyracks/algebricks/core/algebra/plan/PlanStructureVerifier.java| 3 +--
33 files changed, 48 insertions(+), 8 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join-noeqjoin/ngram-edit-distance-inline.sqlpp
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join-noeqjoin/ngram-edit-distance-inline.sqlpp
index c29de43..7c71c3e 100644
---
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join-noeqjoin/ngram-edit-distance-inline.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join-noeqjoin/ngram-edit-distance-inline.sqlpp
@@ -44,6 +44,8 @@ create dataset DBLP(DBLPType) primary key id;
create index ngram_index on DBLP (authors) type ngram (3);
write output to
asterix_nc1:"rttest/inverted-index-join-noeqjoin_ngram-edit-distance-inline.adm";
+set `compiler.internal.sanitycheck` "false";
+
select element {'aauthors':a.authors,'bauthors':b.authors,'ed':ed}
from DBLP as a,
DBLP as b
diff --git
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join-noeqjoin/ngram-fuzzyeq-edit-distance.sqlpp
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join-noeqjoin/ngram-fuzzyeq-edit-distance.sqlpp
index 1234824..30ea450 100644
---
a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join-noeqjoin/ngram-fuzzyeq-edit-distance.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/inverted-index-join-noeqjoin/ngram-fuzzyeq-edit-distance.sqlpp
@@ -55,8 +55,8 @@ create
[asterixdb] 10/25: [NO ISSUE][COMP] Add compiler property to skip assiging low budget
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit be592d91555389a389827bcfb5a243d0c3c4c804
Author: Ali Alsuliman
AuthorDate: Sun Feb 7 11:11:40 2021 -0800
[NO ISSUE][COMP] Add compiler property to skip assiging low budget
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Add a compiler property to allow users to instruct the compiler
not to assign low budget for queries such metadata queries.
Change-Id: Ibba711d76dbdc73c1072c999c05c43dbce30debc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9848
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Ali Alsuliman
Reviewed-by: Dmitry Lychagin
---
.../rules/SetAsterixMemoryRequirementsRule.java| 3 ++
.../apache/asterix/api/common/APIFramework.java| 3 +-
.../apache/asterix/test/common/TestExecutor.java | 32 +-
.../metadata_only_01.2.plans.sqlpp | 31 +++--
.../api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../cluster_state_1_full.1.regexadm| 1 +
.../cluster_state_1_less.1.regexadm| 1 +
.../misc/metadata_only_01/metadata_only_01.2.regex | 1 +
.../asterix/common/config/CompilerProperties.java | 12 +++-
.../common/config/OptimizationConfUtil.java| 3 ++
.../algebricks/core/config/AlgebricksConfig.java | 1 +
.../rewriter/base/PhysicalOptimizationConfig.java | 11 +++-
12 files changed, 82 insertions(+), 18 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
index abe4b71..8de1fb5 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java
@@ -52,6 +52,9 @@ public final class SetAsterixMemoryRequirementsRule extends
SetMemoryRequirement
}
private boolean forceMinMemoryBudget(AsterixOptimizationContext context) {
+if (!context.getPhysicalOptimizationConfig().getMinMemoryAllocation())
{
+return false;
+}
Int2ObjectMap> dataSourceMap =
context.getDataSourceMap();
if (dataSourceMap.isEmpty()) {
return false;
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index baf083d..9088db6 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -135,7 +135,8 @@ public class APIFramework {
CompilerProperties.COMPILER_SORT_PARALLEL_KEY,
CompilerProperties.COMPILER_SORT_SAMPLES_KEY,
CompilerProperties.COMPILER_INDEXONLY_KEY,
CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY,
CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY,
CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY,
-CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY,
FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
+CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY,
+CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY,
FunctionUtil.IMPORT_PRIVATE_FUNCTIONS,
FuzzyUtils.SIM_FUNCTION_PROP_NAME,
FuzzyUtils.SIM_THRESHOLD_PROP_NAME,
StartFeedStatement.WAIT_FOR_COMPLETION,
FeedActivityDetails.FEED_POLICY_NAME,
FeedActivityDetails.COLLECT_LOCATIONS,
SqlppQueryRewriter.INLINE_WITH_OPTION,
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index e202ddf..44038cc 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -42,6 +42,8 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.text.MessageFormat;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -161,6 +163,7 @@ public class TestExecutor {
private static final Pattern HANDLE_VARIABLE_PATTERN =
Pattern.compile("handlevariable=(\\w+)");
private static final Pattern RESULT_VARIABLE_PATTERN =
Pattern.compile("resultvariable=(\\w+)");
private static final Pattern COMPARE_UNORDERED_ARRAY_PATTERN =
[asterixdb] 08/25: [NO ISSUE][TX] Stop reading logs when an empty file is encountered
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1f238097fb756556628f2027f47acd4390ace36f
Author: Murtadha Hubail
AuthorDate: Mon Feb 8 00:07:44 2021 +0300
[NO ISSUE][TX] Stop reading logs when an empty file is encountered
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Whenever an empty txn log file is encountered, do not
attempt to read the next log file since an empty log
file indicates that no more logs exist.
- Add test case.
Change-Id: I8d95c9e444dcdea21c4af35779d4c0c38f6577e5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9883
Reviewed-by: Murtadha Hubail
Reviewed-by: Ian Maxon
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../org/apache/asterix/test/txn/LogManagerTest.java| 2 +-
.../apache/asterix/test/txn/RecoveryManagerTest.java | 18 ++
.../management/service/logging/LogReader.java | 2 +-
3 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/LogManagerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/LogManagerTest.java
index 495a967..685d983 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/LogManagerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/LogManagerTest.java
@@ -201,7 +201,7 @@ public class LogManagerTest {
return txnCtx;
}
-private static void prepareNextLogFile(LogManager logManager) throws
Exception {
+public static void prepareNextLogFile(LogManager logManager) throws
Exception {
Method ensureLastPageFlushed;
Method prepareNextLogFile;
String targetMethod = null;
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
index 65b10a0..34e48c2 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
@@ -22,8 +22,10 @@ import java.io.File;
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.common.TestDataUtil;
+import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.config.StorageProperties;
import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
+import org.apache.asterix.transaction.management.service.logging.LogManager;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -104,4 +106,20 @@ public class RecoveryManagerTest {
final long countAfterRecovery =
TestDataUtil.getDatasetCount(datasetName);
Assert.assertEquals(countBeforeRecovery, countAfterRecovery);
}
+
+@Test
+public void recoveryWithEmptyLogFile() throws Exception {
+String datasetName = "ds";
+TestDataUtil.createIdOnlyDataset(datasetName);
+TestDataUtil.upsertData(datasetName, 10);
+final INcApplicationContext ncAppCtx = (INcApplicationContext)
integrationUtil.ncs[0].getApplicationContext();
+final LogManager logManager = (LogManager)
ncAppCtx.getTransactionSubsystem().getLogManager();
+// do ungraceful shutdown to enforce recovery
+integrationUtil.deinit(false);
+// create empty txn log file
+LogManagerTest.prepareNextLogFile(logManager);
+// ensure recovery completes
+integrationUtil.init(false, TEST_CONFIG_FILE_PATH);
+TestDataUtil.upsertData(datasetName, 10);
+}
}
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogReader.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogReader.java
index 30caab7..53a2897 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogReader.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogReader.java
@@ -163,7 +163,7 @@ public class LogReader implements ILogReader {
*/
private boolean refillLogReadBuffer() {
try {
-if (readLSN % logFileSize == logFile.size()) {
+if (logFile.size() > 0 && readLSN % logFileSize == logFile.size())
{
readLSN += logFileSize - (readLSN % logFileSize);
getLogFile();
}
[asterixdb] 16/25: Merge branch 'gerrit/stabilization-5949a1cb71'
This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit 143eaa5dc7fe0bbb3bbd12dbd43afe16e262d118 Merge: 0381a66 5a377cf Author: Michael Blow AuthorDate: Thu Feb 11 13:46:41 2021 -0500 Merge branch 'gerrit/stabilization-5949a1cb71' Change-Id: Ib72cfa1a734dc260e283ea81da71e25c8e370097
[asterixdb] 09/25: [ASTERIXDB-2826][EXT]: S3 allow using custom non-S3 regions
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0381a66782891e2abd6bf9290fff007d2e363b0f
Author: Hussain Towaileb
AuthorDate: Sun Feb 7 18:08:06 2021 +0300
[ASTERIXDB-2826][EXT]: S3 allow using custom non-S3 regions
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Currently, if an external dataset is created using a
non-S3 region, querying the external dataset fails.
This change allows using non-S3 regions which is possible
for other S3-compatible storages to have.
Change-Id: If7f89eb2adbf9fb9a00d83c339099da05ab0ad06
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9843
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Hussain Towaileb
Reviewed-by: Murtadha Hubail
---
.../non-s3-region/external_dataset.000.ddl.sqlpp | 41 ++
.../non-s3-region/external_dataset.099.ddl.sqlpp | 20 +++
.../runtimets/testsuite_external_dataset.xml | 5 +++
.../asterix/external/util/ExternalDataUtils.java | 13 +--
4 files changed, 67 insertions(+), 12 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.000.ddl.sqlpp
new file mode 100644
index 000..3b22f11
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.000.ddl.sqlpp
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Creating an external dataset should not fail when non-s3-region is used
(some-new-region)
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="some-new-region"),
+("serviceEndpoint"="http://localhost:8001;),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.099.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.099.ddl.sqlpp
new file mode 100644
index 000..548e632
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/aws/s3/non-s3-region/external_dataset.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
index 87ba14d..9cded35 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset.xml
+++
[asterixdb] 06/25: [ASTERIXDB-2828][COMP] Error with GROUP BY ROLLUP and WITH
This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit df73054f93fa80f4a400bc671497baab26dff010 Author: Dmitry Lychagin AuthorDate: Fri Feb 5 15:58:19 2021 -0800 [ASTERIXDB-2828][COMP] Error with GROUP BY ROLLUP and WITH - user model changes: no - storage format changes: no - interface changes: no Details: - Fix internal error when WITH clause is used with GROUP BY ROLLUP - Additional testcases for grouping sets Change-Id: Ieb38a50a42f27d74a90d0f848664d4c7549658b0 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9845 Integration-Tests: Jenkins Tested-by: Jenkins Reviewed-by: Dmitry Lychagin Reviewed-by: Ali Alsuliman --- .../grouping-sets-1/grouping-sets-1.16.query.sqlpp | 25 ++ .../grouping-sets-1/grouping-sets-1.17.query.sqlpp | 35 +++ .../grouping-sets-1/grouping-sets-1.18.query.sqlpp | 27 ++ .../grouping-sets-1/grouping-sets-1.19.query.sqlpp | 29 ++ .../grouping-sets-1/grouping-sets-1.16.adm | 7 + .../grouping-sets-1/grouping-sets-1.17.adm | 7 + .../grouping-sets-1/grouping-sets-1.18.adm | 7 + .../grouping-sets-1/grouping-sets-1.19.adm | 7 + .../grouping-sets-1/grouping-sets-1.16.ast | 156 ++ .../grouping-sets-1/grouping-sets-1.17.ast | 342 + .../grouping-sets-1/grouping-sets-1.18.ast | 193 .../grouping-sets-1/grouping-sets-1.19.ast | 208 + .../rules/subplan/PushSubplanIntoGroupByRule.java | 5 +- 13 files changed, 1047 insertions(+), 1 deletion(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/grouping-sets-1/grouping-sets-1.16.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/grouping-sets-1/grouping-sets-1.16.query.sqlpp new file mode 100644 index 000..eb3ffc3 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/grouping-sets-1/grouping-sets-1.16.query.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; + +select two, four, array_sum((select value gi.tenk.ten from g as gi)) as agg_sum +from tenk +group by rollup(two,four) group as g +order by two, four; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/grouping-sets-1/grouping-sets-1.17.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/grouping-sets-1/grouping-sets-1.17.query.sqlpp new file mode 100644 index 000..388a9f5 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/grouping-sets-1/grouping-sets-1.17.query.sqlpp @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; + +select two, four, sum(ten) as agg_sum +from ( + select two, four, ten + from tenk + where hundred < 20 + + union all + + select two, four, ten + from tenk + where hundred >= 80 +) x +group by rollup(two,four) group as g +order by two, four; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/grouping-sets-1/grouping-sets-1.18.query.sqlpp
[asterixdb] 15/25: [NO ISSUE][TX] Stop reading logs when an empty file is encountered
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5a377cff7adf015eab330fd9f8698c6ca8fcc6e0
Author: Murtadha Hubail
AuthorDate: Mon Feb 8 00:07:44 2021 +0300
[NO ISSUE][TX] Stop reading logs when an empty file is encountered
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Whenever an empty txn log file is encountered, do not
attempt to read the next log file since an empty log
file indicates that no more logs exist.
- Add test case.
(cherry picked from commit 1f238097fb756556628f2027f47acd4390ace36f)
Change-Id: Ia7642e238dbc5f2d927a8a1d640b0586f091416b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9963
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Michael Blow
---
.../org/apache/asterix/test/txn/LogManagerTest.java| 2 +-
.../apache/asterix/test/txn/RecoveryManagerTest.java | 18 ++
.../management/service/logging/LogReader.java | 2 +-
3 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/LogManagerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/LogManagerTest.java
index 495a967..685d983 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/LogManagerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/LogManagerTest.java
@@ -201,7 +201,7 @@ public class LogManagerTest {
return txnCtx;
}
-private static void prepareNextLogFile(LogManager logManager) throws
Exception {
+public static void prepareNextLogFile(LogManager logManager) throws
Exception {
Method ensureLastPageFlushed;
Method prepareNextLogFile;
String targetMethod = null;
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
index 65b10a0..34e48c2 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
@@ -22,8 +22,10 @@ import java.io.File;
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.common.TestDataUtil;
+import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.config.StorageProperties;
import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
+import org.apache.asterix.transaction.management.service.logging.LogManager;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -104,4 +106,20 @@ public class RecoveryManagerTest {
final long countAfterRecovery =
TestDataUtil.getDatasetCount(datasetName);
Assert.assertEquals(countBeforeRecovery, countAfterRecovery);
}
+
+@Test
+public void recoveryWithEmptyLogFile() throws Exception {
+String datasetName = "ds";
+TestDataUtil.createIdOnlyDataset(datasetName);
+TestDataUtil.upsertData(datasetName, 10);
+final INcApplicationContext ncAppCtx = (INcApplicationContext)
integrationUtil.ncs[0].getApplicationContext();
+final LogManager logManager = (LogManager)
ncAppCtx.getTransactionSubsystem().getLogManager();
+// do ungraceful shutdown to enforce recovery
+integrationUtil.deinit(false);
+// create empty txn log file
+LogManagerTest.prepareNextLogFile(logManager);
+// ensure recovery completes
+integrationUtil.init(false, TEST_CONFIG_FILE_PATH);
+TestDataUtil.upsertData(datasetName, 10);
+}
}
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogReader.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogReader.java
index 30caab7..53a2897 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogReader.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/logging/LogReader.java
@@ -163,7 +163,7 @@ public class LogReader implements ILogReader {
*/
private boolean refillLogReadBuffer() {
try {
-if (readLSN % logFileSize == logFile.size()) {
+if (logFile.size() > 0 && readLSN % logFileSize == logFile.size())
{
readLSN += logFileSize - (readLSN % logFileSize);
getLogFile();
}
[asterixdb] 23/25: [NO ISSUE][OTH] Make ExecuteStatementRequestMessage extensible
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d337bb7ec6109e401cbeb0b6536212968ac0b10b
Author: Ali Alsuliman
AuthorDate: Thu Feb 11 17:15:37 2021 -0800
[NO ISSUE][OTH] Make ExecuteStatementRequestMessage extensible
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Make ExecuteStatementRequestMessage extensible.
Change-Id: I481a3bf16bbaf1b11e1073898f48b2d89a26c897
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9986
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Ali Alsuliman
Reviewed-by: Murtadha Hubail
---
.../asterix/api/http/server/NCQueryServiceServlet.java | 16
.../asterix/api/http/server/QueryServiceServlet.java | 8
.../app/message/ExecuteStatementRequestMessage.java | 11 ---
3 files changed, 20 insertions(+), 15 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index 99b6be1..0b0f262 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -68,7 +68,7 @@ public class NCQueryServiceServlet extends
QueryServiceServlet {
}
@Override
-protected void executeStatement(IRequestReference requestReference, String
statementsText,
+protected void executeStatement(IServletRequest request, IRequestReference
requestReference, String statementsText,
SessionOutput sessionOutput, ResultProperties resultProperties,
IStatementExecutor.StatementProperties statementProperties,
IStatementExecutor.Stats stats,
QueryServiceRequestParameters param, RequestExecutionState
executionState,
@@ -85,7 +85,7 @@ public class NCQueryServiceServlet extends
QueryServiceServlet {
long timeout = param.getTimeout();
int stmtCategoryRestrictionMask =
org.apache.asterix.app.translator.RequestParameters
.getStatementCategoryRestrictionMask(param.isReadOnly());
-ExecuteStatementRequestMessage requestMsg =
createRequestMessage(requestReference, statementsText,
+ExecuteStatementRequestMessage requestMsg =
createRequestMessage(request, requestReference, statementsText,
sessionOutput, resultProperties, param,
optionalParameters, statementParameters, ncCtx,
responseFuture, queryLanguage, handleUrl,
stmtCategoryRestrictionMask, false);
executionState.start();
@@ -129,12 +129,12 @@ public class NCQueryServiceServlet extends
QueryServiceServlet {
buildResponseResults(responsePrinter, sessionOutput,
responseMsg.getExecutionPlans(), warnings);
}
-protected ExecuteStatementRequestMessage
createRequestMessage(IRequestReference requestReference,
-String statementsText, SessionOutput sessionOutput,
ResultProperties resultProperties,
-QueryServiceRequestParameters param, Map
optionalParameters,
-Map statementParameters, INCServiceContext ncCtx,
MessageFuture responseFuture,
-ILangExtension.Language queryLanguage, String handleUrl, int
stmtCategoryRestrictionMask,
-boolean forceDropDataset) {
+protected ExecuteStatementRequestMessage
createRequestMessage(IServletRequest request,
+IRequestReference requestReference, String statementsText,
SessionOutput sessionOutput,
+ResultProperties resultProperties, QueryServiceRequestParameters
param,
+Map optionalParameters, Map
statementParameters, INCServiceContext ncCtx,
+MessageFuture responseFuture, ILangExtension.Language
queryLanguage, String handleUrl,
+int stmtCategoryRestrictionMask, boolean forceDropDataset) {
return new ExecuteStatementRequestMessage(ncCtx.getNodeId(),
responseFuture.getFutureId(), queryLanguage,
statementsText, sessionOutput.config(),
resultProperties.getNcToCcResultProperties(),
param.getClientContextID(), handleUrl, optionalParameters,
statementParameters,
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index dea46f2..f394794 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -301,9 +301,9 @@ public class QueryServiceServlet extends
[asterixdb] 14/25: [ASTERIXDB-2738][STO] Create Mask File Before Merge Operations
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit aa0f9a9338e7ded2b2c519e6dd9d24c7c078bb0c
Author: Murtadha Hubail
AuthorDate: Thu May 21 21:29:46 2020 +0300
[ASTERIXDB-2738][STO] Create Mask File Before Merge Operations
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Before starting a merge operation, create a mask file
(.mask_C_startSeq_endSeq) for the merged component to
indicate that this component isn't valid yet.
- On the merge operation successful completion, delete the
merged component mask file.
- In the case of any unexpected failure during the merge
operation, all files of the failed merged component will
be deleted on node startup/shutdown, including the mask
file.
- Halt on any IO opeartion failure.
- Add a test case that ensures only masked merged components
are deleted but not the original components that were
supposed to be merged.
(cherry picked from commit fe0f6fa181b03195cda8ff3585734ce10f466467)
Change-Id: I7c07daf033c5a1a00bd1b33debc8a4d7c7ce1e59
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9964
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Michael Blow
---
.../org/apache/asterix/app/nc/HaltCallback.java| 5 +-
.../PersistentLocalResourceRepositoryTest.java | 53 ++
.../ioopcallbacks/LSMIOOperationCallback.java | 32 -
.../asterix/common/storage/ResourceReference.java | 14 ++
.../replication/messaging/ComponentMaskTask.java | 4 +-
.../PersistentLocalResourceRepository.java | 14 +-
.../java/org/apache/hyracks/util/ExitUtil.java | 2 +-
7 files changed, 103 insertions(+), 21 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
index 17a4f46..9802001 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
@@ -20,7 +20,6 @@ package org.apache.asterix.app.nc;
import org.apache.hyracks.storage.am.lsm.common.api.IIoOperationFailedCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
-import
org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.util.ExitUtil;
import org.apache.logging.log4j.LogManager;
@@ -42,8 +41,6 @@ public class HaltCallback implements
IIoOperationFailedCallback {
@Override
public void operationFailed(ILSMIOOperation operation, Throwable t) {
LOGGER.error("Operation {} has failed", operation, t);
-if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
-ExitUtil.halt(ExitUtil.EC_FLUSH_FAILED);
-}
+ExitUtil.halt(ExitUtil.EC_IO_OPERATION_FAILED);
}
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
index 00d2d3d..50118fc 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
@@ -171,6 +171,59 @@ public class PersistentLocalResourceRepositoryTest {
Assert.assertFalse(indexMetadataMaskFile.exists());
}
+@Test
+public void deleteMaskedMergedFiles() throws Exception {
+final INcApplicationContext ncAppCtx = (INcApplicationContext)
integrationUtil.ncs[0].getApplicationContext();
+final String nodeId = ncAppCtx.getServiceContext().getNodeId();
+final String datasetName = "ds";
+TestDataUtil.createIdOnlyDataset(datasetName);
+final Dataset dataset = TestDataUtil.getDataset(integrationUtil,
datasetName);
+final String indexPath = TestDataUtil.getIndexPath(integrationUtil,
dataset, nodeId);
+FileReference indexDirRef = ncAppCtx.getIoManager().resolve(indexPath);
+int compSeqStart = 100;
+int validComponentSequence = 103;
+// advance valid component seq in checkpoint
+PersistentLocalResourceRepository localResourceRepository =
+(PersistentLocalResourceRepository)
ncAppCtx.getLocalResourceRepository();
+LocalResource localResource = localResourceRepository.get(indexPath);
+DatasetResourceReference drr =
[asterixdb] 11/25: Merge commit 'gerrit/mad-hatter'
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7707f1f02f59293e1aa3a899846cd54c676d02a8
Merge: be592d9 0381a66
Author: Hussain Towaileb
AuthorDate: Tue Feb 9 15:17:02 2021 +0300
Merge commit 'gerrit/mad-hatter'
Change-Id: I0a1bf2f476c13d1a7dfc7a1ffc13858367ed870a
.../apache/asterix/test/txn/LogManagerTest.java| 2 +-
.../asterix/test/txn/RecoveryManagerTest.java | 18 +++
.../non-s3-region/external_dataset.000.ddl.sqlpp | 41 +++
.../non-s3-region/external_dataset.099.ddl.sqlpp | 20
.../runtimets/testsuite_external_dataset_s3.xml| 5 +
asterixdb/asterix-external-data/pom.xml| 4 +
.../record/reader/aws/AwsS3InputStreamFactory.java | 125 -
.../external/util/ExternalDataConstants.java | 3 +-
.../asterix/external/util/ExternalDataUtils.java | 97 ++--
.../management/service/logging/LogReader.java | 2 +-
asterixdb/pom.xml | 5 +
11 files changed, 256 insertions(+), 66 deletions(-)
diff --cc
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
index 59d7fae,34e48c2..4f8df3d
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
@@@ -22,7 -22,10 +22,9 @@@ import java.io.File
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.common.TestDataUtil;
+ import org.apache.asterix.common.api.INcApplicationContext;
-import org.apache.asterix.common.config.StorageProperties;
import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
+ import org.apache.asterix.transaction.management.service.logging.LogManager;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
diff --cc
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp
index 000,3b22f11..3b22f11
mode 00,100644..100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.000.ddl.sqlpp
diff --cc
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.099.ddl.sqlpp
index 000,548e632..548e632
mode 00,100644..100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.099.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/non-s3-region/external_dataset.099.ddl.sqlpp
diff --cc
asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index e0a28ba,000..6557230
mode 100644,00..100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@@ -1,304 -1,0 +1,309 @@@
+
+
+
+
+
+
+
+common/json/json
+
+
+
+
+create-with-session-token
+
+
+
+
+
+common/json/gz
+
+
+
+
+
+common/json/mixed
+
+
+
+
+
+common/csv/csv
+
+
+
+
+common/csv/gz
+
+
+
+
+common/csv/mixed
+
+
+
+
+
+common/tsv/tsv
+
+
+
+
+
+common/tsv/gz
+
+
+
+
+
+common/tsv/mixed
+
+
+
+
+
+common/empty-string-definition
+
+
+
+
+
+common/over-1000-objects
+
+
+
+
+
+common/malformed-json
+Parsing error at malformed-data/duplicate-fields.json
line 1 field field: Duplicate field 'field'
+Parsing error at malformed-data/malformed-json.json
line 1 field field: Unexpected character ('}' (code 125)): was expecting
double-quote to start field name
+Parsing error at malformed-data/malformed-json-2.json
line 4 field array_f: Unexpected character (']' (code 93)): expected a valid
value (JSON String, Number, Array, Object or token 'null', 'true' or
'false')
+Parsing error at
malformed-data/malformed-jsonl-1.json line 3 field field2: Unrecognized token
'truee': was expecting (JSON String, Number, Array, Object or token 'null',
'true' or 'false')
+Parsing error at
[asterixdb] 25/25: Merge branch 'gerrit/cheshire-cat'
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2c22ebe95d210c047fc58a1247509aed5854048f
Merge: ee4a056 56759ec
Author: Michael Blow
AuthorDate: Fri Feb 12 18:48:21 2021 -0500
Merge branch 'gerrit/cheshire-cat'
Change-Id: If9b2e4f8b46d2f3b03917b74b08f98694fba4ece
.../org/apache/asterix/active/ActiveManager.java | 2 +-
.../rules/SetAsterixMemoryRequirementsRule.java| 3 +
.../apache/asterix/api/common/APIFramework.java| 3 +-
.../api/http/server/NCQueryServiceServlet.java | 16 +-
.../api/http/server/QueryServiceServlet.java | 8 +-
.../message/ExecuteStatementRequestMessage.java| 11 +-
.../app/replication/NcLifecycleCoordinator.java| 24 +-
.../apache/asterix/test/common/TestExecutor.java | 32 +-
.../apache/asterix/test/txn/LogManagerTest.java| 2 +-
.../asterix/test/txn/RecoveryManagerTest.java | 18 +
.../ngram-edit-distance-inline.sqlpp | 2 +
.../ngram-fuzzyeq-edit-distance.sqlpp | 2 +-
.../olist-edit-distance-inline.sqlpp | 2 +
...pidx-with-join-edit-distance-check-idx_01.sqlpp | 2 +
...x-with-join-edit-distance-check-idx_01_ps.sqlpp | 2 +-
.../ngram-edit-distance-check_02.sqlpp | 2 +
.../ngram-edit-distance-check_03.sqlpp | 2 +
.../ngram-edit-distance-check_04.sqlpp | 2 +
.../ngram-edit-distance_02.sqlpp | 2 +
.../ngram-edit-distance_03.sqlpp | 2 +
.../ngram-edit-distance_04.sqlpp | 2 +
.../ngram-fuzzyeq-edit-distance_01.sqlpp | 2 +-
.../ngram-fuzzyeq-edit-distance_03.sqlpp | 2 +-
.../olist-edit-distance-check_02.sqlpp | 2 +
.../olist-edit-distance-check_03.sqlpp | 2 +
.../olist-edit-distance-check_04.sqlpp | 2 +
.../olist-edit-distance_02.sqlpp | 2 +
.../olist-edit-distance_03.sqlpp | 2 +
.../olist-edit-distance_04.sqlpp | 2 +
.../olist-fuzzyeq-edit-distance_02.sqlpp | 2 +-
.../olist-fuzzyeq-edit-distance_03.sqlpp | 2 +-
...x-with-join-edit-distance-check-idx_01_ps.sqlpp | 1 +
...x-with-join-edit-distance-check-idx_01_ps.sqlpp | 1 +
...x-with-join-edit-distance-check-idx_01_ps.sqlpp | 1 +
.../non-s3-region/external_dataset.000.ddl.sqlpp} | 24 +-
.../non-s3-region/external_dataset.099.ddl.sqlpp} | 9 +-
.../grouping-sets-1.16.query.sqlpp}| 12 +-
.../grouping-sets-1.17.query.sqlpp}| 18 +-
.../grouping-sets-1.18.query.sqlpp}| 14 +-
.../grouping-sets-1.19.query.sqlpp}| 14 +-
...probe-pidx-with-join-invidx-sidx2.3.query.sqlpp | 1 +
.../metadata_only_01.2.plans.sqlpp}| 21 +-
...probe-pidx-with-join-invidx-sidx2.3.query.sqlpp | 1 +
...probe-pidx-with-join-invidx-sidx2.3.query.sqlpp | 1 +
...probe-pidx-with-join-invidx-sidx2.3.query.sqlpp | 1 +
.../{q49/q49.1.ddl.sqlpp => q05/q05.1.ddl.sqlpp} | 367 ++---
.../q49.2.update.sqlpp => q05/q05.2.update.sqlpp} | 20 +-
.../queries_sqlpp/tpcds/q05/q05.3.query.sqlpp | 155
.../queries_sqlpp/tpcds/q15/q15.1.ddl.sqlpp| 1 +
.../queries_sqlpp/tpcds/q15/q15.2.update.sqlpp | 8 +-
.../{q26/q26.1.ddl.sqlpp => q18/q18.1.ddl.sqlpp} | 151 +++
.../q15.2.update.sqlpp => q18/q18.2.update.sqlpp} | 11 +-
.../queries_sqlpp/tpcds/q18/q18.3.query.sqlpp | 50 +++
.../queries_sqlpp/tpcds/q20/q20.1.ddl.sqlpp| 1 +
.../queries_sqlpp/tpcds/q20/q20.2.update.sqlpp | 6 +-
.../queries_sqlpp/tpcds/q20/q20.3.query.sqlpp | 2 +-
.../{q37/q37.1.ddl.sqlpp => q22/q22.1.ddl.sqlpp} | 40 --
.../q20.2.update.sqlpp => q22/q22.2.update.sqlpp} | 7 +-
.../q20.2.update.sqlpp => q22/q22.3.query.sqlpp} | 17 +-
.../queries_sqlpp/tpcds/q25/q25.1.ddl.sqlpp| 2 +
.../queries_sqlpp/tpcds/q26/q26.1.ddl.sqlpp| 1 +
.../queries_sqlpp/tpcds/q26/q26.2.update.sqlpp | 10 +-
.../{q29/q29.1.ddl.sqlpp => q27/q27.1.ddl.sqlpp} | 80 +---
.../q26.2.update.sqlpp => q27/q27.2.update.sqlpp} | 11 +-
.../q26.2.update.sqlpp => q27/q27.3.query.sqlpp} | 30 +-
.../queries_sqlpp/tpcds/q29/q29.1.ddl.sqlpp| 2 +
.../{q29/q29.1.ddl.sqlpp => q36/q36.1.ddl.sqlpp} | 65
.../q20.2.update.sqlpp => q36/q36.2.update.sqlpp} | 7 +-
.../queries_sqlpp/tpcds/q36/q36.3.query.sqlpp | 48 +++
.../queries_sqlpp/tpcds/q37/q37.1.ddl.sqlpp| 1 +
.../queries_sqlpp/tpcds/q49/q49.1.ddl.sqlpp| 1 +
.../queries_sqlpp/tpcds/q49/q49.2.update.sqlpp | 14 +-
.../queries_sqlpp/tpcds/q49/q49.3.query.sqlpp | 10 +-
.../queries_sqlpp/tpcds/q57/q57.1.ddl.sqlpp| 1 +
.../queries_sqlpp/tpcds/q57/q57.3.query.sqlpp | 4 +-
[asterixdb] 22/25: Merge branch 'gerrit/mad-hatter'
This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit f72e4cfaa0f2678f6e3ce3237b6c58bc94bad45c Merge: 39eb84f 8ac45b4 Author: Michael Blow AuthorDate: Fri Feb 12 17:31:26 2021 -0500 Merge branch 'gerrit/mad-hatter' Change-Id: I6d2c3a1fae5dd5310885a3d65797b0ebc80c9937 .../asterix/common/config/ExternalProperties.java | 3 +- .../hyracks/control/common/config/OptionTypes.java | 258 + 2 files changed, 159 insertions(+), 102 deletions(-) diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java index c415437,642cbd6..9bef65d --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java @@@ -46,8 -47,8 +47,8 @@@ public class ExternalProperties extend CC_JAVA_OPTS(STRING, "-Xmx1024m", "The JVM options passed to the cluster controller process by managix"), NC_JAVA_OPTS(STRING, "-Xmx1024m", "The JVM options passed to the node controller process(es) by managix"), MAX_WEB_REQUEST_SIZE( - NONNEGATIVE_INTEGER, + POSITIVE_INTEGER_BYTE_UNIT, -StorageUtil.getIntSizeInBytes(50, StorageUtil.StorageUnit.MEGABYTE), +StorageUtil.getIntSizeInBytes(200, StorageUtil.StorageUnit.MEGABYTE), "The maximum accepted web request size in bytes"), REQUESTS_ARCHIVE_SIZE(NONNEGATIVE_INTEGER, 50, "The maximum number of archived requests to maintain");
[asterixdb] 03/25: [NO ISSUE][NET] Ensure ssl socket is connected before write
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0dbb0c2c39639262068bdaa8bb6d571fadb3f1da
Author: Murtadha Hubail
AuthorDate: Thu Feb 4 19:58:47 2021 +0300
[NO ISSUE][NET] Ensure ssl socket is connected before write
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Ensure the SSL socket is connected before attempting a
write/wrap operation.
- Only attempt to send goodbye message to replicas when
the socket channel is still connected.
- Always attempt to close the ssl socket channel even when
the close handshake fails.
Change-Id: I07fbcd76be29853c94cb133485d83034ceee9cb3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9825
Integration-Tests: Jenkins
Reviewed-by: Michael Blow
Tested-by: Jenkins
---
.../org/apache/asterix/replication/api/PartitionReplica.java | 4 ++--
.../java/org/apache/hyracks/ipc/sockets/SslSocketChannel.java | 11 +--
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
index 3226299..6b97306 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
@@ -113,10 +113,10 @@ public class PartitionReplica implements
IPartitionReplica {
public synchronized void close() {
try {
-if (sc != null) {
+if (NetworkingUtil.isHealthy(sc)) {
sendGoodBye();
-NetworkUtil.closeQuietly(sc);
}
+NetworkUtil.closeQuietly(sc);
} finally {
sc = null;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/sockets/SslSocketChannel.java
b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/sockets/SslSocketChannel.java
index 02a1a02..74deefe 100644
---
a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/sockets/SslSocketChannel.java
+++
b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/sockets/SslSocketChannel.java
@@ -20,6 +20,7 @@ package org.apache.hyracks.ipc.sockets;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
import java.nio.channels.SocketChannel;
import javax.net.ssl.SSLEngine;
@@ -142,6 +143,9 @@ public class SslSocketChannel implements ISocketChannel {
while (src.hasRemaining()) {
// chunk src to encrypted ssl records of pocket size
outEncryptedData.clear();
+if (!socketChannel.isConnected()) {
+throw new ClosedChannelException();
+}
final SSLEngineResult result = engine.wrap(src, outEncryptedData);
switch (result.getStatus()) {
case OK:
@@ -186,8 +190,11 @@ public class SslSocketChannel implements ISocketChannel {
public synchronized void close() throws IOException {
if (socketChannel.isOpen()) {
engine.closeOutbound();
-new SslHandshake(this).handshake();
-socketChannel.close();
+try {
+new SslHandshake(this).handshake();
+} finally {
+socketChannel.close();
+}
}
}
[asterixdb] 20/25: [NO ISSUE][CONFIG] Avoid integer overflow on option parse
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6eb42a0ebf010b69c8f4e763d90f89cfbe46ea1a
Author: Michael Blow
AuthorDate: Wed Dec 9 10:11:00 2020 -0500
[NO ISSUE][CONFIG] Avoid integer overflow on option parse
- Do not use Integer.parseUnsignedInt() for nonnegative integer option
parsing, as
this will result in negative numbers for values > 0x7fff
- Renamed (OptionTypes.)UNSIGNED_INTEGER to NONNEGATIVE_INTEGER for clarity
- Add POSITIVE_INTEGER_BYTE_UNIT & POSITIVE_LONG_BYTE_UNIT option types
(includes cherry-pick of commit 7a99fcefe3)
Change-Id: I03bc9e92c2b6ce48977e93966f6cb9160c4df893
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10003
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Michael Blow
Reviewed-by: Murtadha Hubail
---
.../asterix/common/config/CompilerProperties.java | 7 +-
.../asterix/common/config/ExternalProperties.java | 17 +-
.../asterix/common/config/MetadataProperties.java | 6 +-
.../common/config/ReplicationProperties.java | 4 +-
.../asterix/common/config/StorageProperties.java | 6 +-
.../common/config/TransactionProperties.java | 6 +-
.../hyracks/control/common/config/OptionTypes.java | 271 -
.../control/common/controllers/CCConfig.java | 20 +-
.../control/common/controllers/NCConfig.java | 26 +-
9 files changed, 202 insertions(+), 161 deletions(-)
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 9428e6f..8860495 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -23,7 +23,6 @@ import static
org.apache.hyracks.control.common.config.OptionTypes.INTEGER;
import static
org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT;
import static
org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
import static
org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER;
-import static
org.apache.hyracks.control.common.config.OptionTypes.UNSIGNED_INTEGER;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
@@ -31,6 +30,7 @@ import
org.apache.hyracks.algebricks.core.config.AlgebricksConfig;
import org.apache.hyracks.api.config.IOption;
import org.apache.hyracks.api.config.IOptionType;
import org.apache.hyracks.api.config.Section;
+import org.apache.hyracks.control.common.config.OptionTypes;
import org.apache.hyracks.util.StorageUtil;
public class CompilerProperties extends AbstractProperties {
@@ -68,7 +68,10 @@ public class CompilerProperties extends AbstractProperties {
+ "other integer values dictate the number of query
execution parallel partitions. The system will "
+ "fall back to use the number of all available CPU
cores in the cluster as the degree of parallelism "
+ "if the number set by a user is too large or too
small"),
-COMPILER_STRINGOFFSET(UNSIGNED_INTEGER, 0, "Position of a first
character in a String/Binary (0 or 1)"),
+COMPILER_STRINGOFFSET(
+OptionTypes.getRangedIntegerType(0, 1),
+0,
+"Position of a first character in a String/Binary (0 or 1)"),
COMPILER_SORT_PARALLEL(BOOLEAN, AlgebricksConfig.SORT_PARALLEL,
"Enabling/Disabling full parallel sort"),
COMPILER_SORT_SAMPLES(
POSITIVE_INTEGER,
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
index 1533c9f..642cbd6 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java
@@ -19,9 +19,10 @@
package org.apache.asterix.common.config;
import static org.apache.hyracks.control.common.config.OptionTypes.LEVEL;
+import static
org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
import static
org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER;
+import static
org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER_BYTE_UNIT;
import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
-import static
org.apache.hyracks.control.common.config.OptionTypes.UNSIGNED_INTEGER;
import
[asterixdb] 17/25: [NO ISSUE][REP] Notify nodes on replica failure
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 33eea75a950ba3b82612d938d0b892d116db3f5d
Author: Murtadha Hubail
AuthorDate: Thu Feb 11 13:36:26 2021 +0300
[NO ISSUE][REP] Notify nodes on replica failure
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Whenever a node fails, notify remaining active nodes
that the failed node's replica has failed.
Change-Id: I12551bd543cd4b664101e8f4d4f44f3124de3d54
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9944
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../app/replication/NcLifecycleCoordinator.java| 24 +-
.../replication/INcLifecycleCoordinator.java | 5 +-
.../messaging/ReplicaFailedMessage.java| 54 ++
.../asterix/runtime/utils/ClusterStateManager.java | 23 +++--
4 files changed, 100 insertions(+), 6 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index 7576e0d..d22e9fc 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.app.replication;
+import static org.apache.hyracks.api.exceptions.ErrorCode.NODE_FAILED;
+
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -49,6 +52,7 @@ import
org.apache.asterix.common.replication.INCLifecycleMessage;
import org.apache.asterix.common.replication.INcLifecycleCoordinator;
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.replication.messaging.ReplicaFailedMessage;
import org.apache.hyracks.api.application.ICCServiceContext;
import org.apache.hyracks.api.client.NodeStatus;
import org.apache.hyracks.api.control.IGatekeeper;
@@ -80,12 +84,15 @@ public class NcLifecycleCoordinator implements
INcLifecycleCoordinator {
}
@Override
-public void notifyNodeFailure(String nodeId) throws HyracksDataException {
+public void notifyNodeFailure(String nodeId, InetSocketAddress
replicaAddress) throws HyracksDataException {
pendingStartupCompletionNodes.remove(nodeId);
clusterManager.updateNodeState(nodeId, false, null);
if (nodeId.equals(metadataNodeId)) {
clusterManager.updateMetadataNode(metadataNodeId, false);
}
+if (replicaAddress != null) {
+notifyFailedReplica(clusterManager, nodeId, replicaAddress);
+}
clusterManager.refreshState();
}
@@ -229,4 +236,19 @@ public class NcLifecycleCoordinator implements
INcLifecycleCoordinator {
throw HyracksDataException.create(e);
}
}
+
+private void notifyFailedReplica(IClusterStateManager clusterManager,
String nodeID,
+InetSocketAddress replicaAddress) {
+LOGGER.info("notify replica failure of nodeId {} at {}", nodeID,
replicaAddress);
+Set ncs = clusterManager.getParticipantNodes(true);
+ReplicaFailedMessage message =
+new ReplicaFailedMessage(replicaAddress,
HyracksDataException.create(NODE_FAILED, nodeID));
+for (String nodeId : ncs) {
+try {
+messageBroker.sendApplicationMessageToNC(message, nodeId);
+} catch (Exception e) {
+LOGGER.info("failed to notify replica failure to node {}",
nodeID);
+}
+}
+}
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/INcLifecycleCoordinator.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/INcLifecycleCoordinator.java
index 1a7c3c8..9a3b125 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/INcLifecycleCoordinator.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/INcLifecycleCoordinator.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.common.replication;
+import java.net.InetSocketAddress;
+
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -35,9 +37,10 @@ public interface INcLifecycleCoordinator {
* Defines the logic of a {@link INcLifecycleCoordinator} when a node
leaves the cluster.
*
* @param nodeId
+ * @param replicaAddress
* @throws
[asterixdb] 18/25: Merge branch 'gerrit/mad-hatter'
This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit eccf4baa03f03eadc337763c312429f1abd7aa4f Merge: 33eea75 143eaa5 Author: Michael Blow AuthorDate: Thu Feb 11 14:09:43 2021 -0500 Merge branch 'gerrit/mad-hatter' Change-Id: I68c0a52780a10c96030165fa5c34426d88f5b57d
[asterixdb] 01/25: [NO ISSUE][*DB][MISC] Remove leftover managix XJC invocations
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 9f19cef6bb20a20173f6fe2ba14127ba8bf5a7e6
Author: Michael Blow
AuthorDate: Tue Feb 2 11:30:40 2021 -0500
[NO ISSUE][*DB][MISC] Remove leftover managix XJC invocations
Change-Id: I76847f32ac7f275bdf53854366ff22eacb1db84f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9805
Integration-Tests: Jenkins
Tested-by: Michael Blow
Reviewed-by: Michael Blow
Reviewed-by: Murtadha Hubail
---
asterixdb/asterix-common/pom.xml | 108 ---
1 file changed, 108 deletions(-)
diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml
index 760e027..d475caf 100644
--- a/asterixdb/asterix-common/pom.xml
+++ b/asterixdb/asterix-common/pom.xml
@@ -38,114 +38,6 @@
-org.jvnet.jaxb2.maven2
-maven-jaxb2-plugin
-
-
-configuration
-
- generate
-
-
-
--Xsetters
--Xvalue-constructor
-
-
-
- org.jvnet.jaxb2_commons
- jaxb2-basics
- 0.6.2
-
-
- org.jvnet.jaxb2_commons
- jaxb2-value-constructor
- 3.0
-
-
- src/main/resources/schema
-
-asterix-conf.xsd
-
-
org.apache.asterix.common.configuration
- src/main/resources/schema
-
-jaxb-bindings.xjb
-
-
${project.build.directory}/generated-sources/configuration
-
-
-
-cluster
-
- generate
-
-
-
--Xsetters
--Xvalue-constructor
-
-
-
- org.jvnet.jaxb2_commons
- jaxb2-basics
- 0.6.2
-
-
- org.jvnet.jaxb2_commons
- jaxb2-value-constructor
- 3.0
-
-
- src/main/resources/schema
-
-cluster.xsd
-
-
org.apache.asterix.event.schema.cluster
-
${project.build.directory}/generated-sources/cluster
- src/main/resources/schema
-
-jaxb-bindings.xjb
-
-
-
-
-yarn_cluster
-
- generate
-
-
-
--Xsetters
--Xvalue-constructor
-
-
-
- org.jvnet.jaxb2_commons
- jaxb2-basics
- 0.6.2
-
-
- org.jvnet.jaxb2_commons
- jaxb2-value-constructor
- 3.0
-
-
- src/main/resources/schema
-
-yarn_cluster.xsd
-
-
org.apache.asterix.event.schema.yarnCluster
-
${project.build.directory}/generated-sources/yarn_cluster
- src/main/resources/schema
-
-jaxb-bindings.xjb
-
-
-
-
-
-
org.apache.maven.plugins
maven-jar-plugin
[asterixdb] branch master updated (ee4a056 -> 2c22ebe)
This is an automated email from the ASF dual-hosted git repository. mblow pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git. from ee4a056 Merge branch 'gerrit/cheshire-cat' new bd2f43f [NO ISSUE][COMP] Add TPC-DS ROLLUP tests new 9f19cef [NO ISSUE][*DB][MISC] Remove leftover managix XJC invocations new 0dbb0c2 [NO ISSUE][NET] Ensure ssl socket is connected before write new 4e7d48b Merge branch 'gerrit/mad-hatter' new df73054 [ASTERIXDB-2828][COMP] Error with GROUP BY ROLLUP and WITH new c65fa86 [NO ISSUE][COMP] Additional plan sanity check new be592d9 [NO ISSUE][COMP] Add compiler property to skip assiging low budget new 9f454cd [ASTERIXDB-2827][EXT]: S3 external dataset: properly fallback to old API new 1f23809 [NO ISSUE][TX] Stop reading logs when an empty file is encountered new 0381a66 [ASTERIXDB-2826][EXT]: S3 allow using custom non-S3 regions new 7707f1f Merge commit 'gerrit/mad-hatter' new 5faef51 [ASTERIXDB-2819][IO] Make absolute & normalize IODevice paths new 33eea75 [NO ISSUE][REP] Notify nodes on replica failure new dc12fc0 Fixup merge commit 73d9c0f74e new aa0f9a9 [ASTERIXDB-2738][STO] Create Mask File Before Merge Operations new 5a377cf [NO ISSUE][TX] Stop reading logs when an empty file is encountered new 143eaa5 Merge branch 'gerrit/stabilization-5949a1cb71' new eccf4ba Merge branch 'gerrit/mad-hatter' new 39eb84f [NO ISSUE][*DB] Log at FATAL level when halting due to failure to stop active runtime new d337bb7 [NO ISSUE][OTH] Make ExecuteStatementRequestMessage extensible new 6eb42a0 [NO ISSUE][CONFIG] Avoid integer overflow on option parse new 8ac45b4 [ASTERIXDB-2815][COMP] DISTINCT in subquery gives wrong result new f72e4cf Merge branch 'gerrit/mad-hatter' new 56759ec Merge "Merge branch 'gerrit/mad-hatter'" into cheshire-cat new 2c22ebe Merge branch 'gerrit/cheshire-cat' The 25 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/asterix/active/ActiveManager.java | 2 +- .../rules/SetAsterixMemoryRequirementsRule.java| 3 + .../apache/asterix/api/common/APIFramework.java| 3 +- .../api/http/server/NCQueryServiceServlet.java | 16 +- .../api/http/server/QueryServiceServlet.java | 8 +- .../message/ExecuteStatementRequestMessage.java| 11 +- .../app/replication/NcLifecycleCoordinator.java| 24 +- .../apache/asterix/test/common/TestExecutor.java | 32 +- .../apache/asterix/test/txn/LogManagerTest.java| 2 +- .../asterix/test/txn/RecoveryManagerTest.java | 18 + .../ngram-edit-distance-inline.sqlpp | 2 + .../ngram-fuzzyeq-edit-distance.sqlpp | 2 +- .../olist-edit-distance-inline.sqlpp | 2 + ...pidx-with-join-edit-distance-check-idx_01.sqlpp | 2 + ...x-with-join-edit-distance-check-idx_01_ps.sqlpp | 2 +- .../ngram-edit-distance-check_02.sqlpp | 2 + .../ngram-edit-distance-check_03.sqlpp | 2 + .../ngram-edit-distance-check_04.sqlpp | 2 + .../ngram-edit-distance_02.sqlpp | 2 + .../ngram-edit-distance_03.sqlpp | 2 + .../ngram-edit-distance_04.sqlpp | 2 + .../ngram-fuzzyeq-edit-distance_01.sqlpp | 2 +- .../ngram-fuzzyeq-edit-distance_03.sqlpp | 2 +- .../olist-edit-distance-check_02.sqlpp | 2 + .../olist-edit-distance-check_03.sqlpp | 2 + .../olist-edit-distance-check_04.sqlpp | 2 + .../olist-edit-distance_02.sqlpp | 2 + .../olist-edit-distance_03.sqlpp | 2 + .../olist-edit-distance_04.sqlpp | 2 + .../olist-fuzzyeq-edit-distance_02.sqlpp | 2 +- .../olist-fuzzyeq-edit-distance_03.sqlpp | 2 +- ...x-with-join-edit-distance-check-idx_01_ps.sqlpp | 1 + ...x-with-join-edit-distance-check-idx_01_ps.sqlpp | 1 + ...x-with-join-edit-distance-check-idx_01_ps.sqlpp | 1 + .../non-s3-region/external_dataset.000.ddl.sqlpp} | 24 +- .../non-s3-region}/external_dataset.099.ddl.sqlpp | 0 .../grouping-sets-1.16.query.sqlpp}| 12 +- .../grouping-sets-1.17.query.sqlpp}| 18 +- .../grouping-sets-1.18.query.sqlpp}| 14 +- .../grouping-sets-1.19.query.sqlpp}| 14 +- ...probe-pidx-with-join-invidx-sidx2.3.query.sqlpp | 1 + .../metadata_only_01.2.plans.sqlpp}| 21 +- ...probe-pidx-with-join-invidx-sidx2.3.query.sqlpp | 1 + ...probe-pidx-with-join-invidx-sidx2.3.query.sqlpp | 1 + ...probe-pidx-with-join-invidx-sidx2.3.query.sqlpp |
[asterixdb] 04/25: Merge branch 'gerrit/mad-hatter'
This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit 4e7d48b5c39c95ed1d47a7d672051941dbe095ce Merge: bd2f43f 0dbb0c2 Author: Michael Blow AuthorDate: Fri Feb 5 15:37:18 2021 -0500 Merge branch 'gerrit/mad-hatter' Change-Id: Ib852b2318cc65dee4a78d3c57c533b9797fa24a9 asterixdb/asterix-common/pom.xml | 108 - .../asterix/replication/api/PartitionReplica.java | 4 +- .../hyracks/ipc/sockets/SslSocketChannel.java | 11 ++- 3 files changed, 11 insertions(+), 112 deletions(-)
[asterixdb] 05/25: [ASTERIXDB-2827][EXT]: S3 external dataset: properly fallback to old API
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 9f454cdb48537e1fc0f9dd7b66ad75470a8c514a
Author: Hussain Towaileb
AuthorDate: Fri Feb 5 16:49:11 2021 +0300
[ASTERIXDB-2827][EXT]: S3 external dataset: properly fallback to old API
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Properly fallback to old API if the new API is not supported.
if the old API fails as well, then report the error properly.
Change-Id: Ib453eb396def92218951b9e45a89b6c0f48a54f6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9844
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Hussain Towaileb
Reviewed-by: Michael Blow
---
asterixdb/NOTICE | 2 +-
asterixdb/asterix-external-data/pom.xml| 4 +
.../record/reader/aws/AwsS3InputStreamFactory.java | 122 -
.../external/util/ExternalDataConstants.java | 3 +-
.../asterix/external/util/ExternalDataUtils.java | 109 --
asterixdb/pom.xml | 5 +
hyracks-fullstack/NOTICE | 2 +-
7 files changed, 182 insertions(+), 65 deletions(-)
diff --git a/asterixdb/NOTICE b/asterixdb/NOTICE
index b4729a8..4aabe27 100644
--- a/asterixdb/NOTICE
+++ b/asterixdb/NOTICE
@@ -1,5 +1,5 @@
Apache AsterixDB
-Copyright 2015-2020 The Apache Software Foundation
+Copyright 2015-2021 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
diff --git a/asterixdb/asterix-external-data/pom.xml
b/asterixdb/asterix-external-data/pom.xml
index 8270d71..169bcb6 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -437,6 +437,10 @@
software.amazon.awssdk
+ aws-core
+
+
+ software.amazon.awssdk
http-client-spi
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index f3a36ff..0bc4c40 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -50,8 +50,11 @@ import org.apache.hyracks.api.util.CleanupUtils;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Object;
public class AwsS3InputStreamFactory implements IInputStreamFactory {
@@ -88,10 +91,6 @@ public class AwsS3InputStreamFactory implements
IInputStreamFactory {
this.configuration = configuration;
ICcApplicationContext ccApplicationContext = (ICcApplicationContext)
ctx.getApplicationContext();
-String container = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
-
-List filesOnly = new ArrayList<>();
-
// Ensure the validity of include/exclude
ExternalDataUtils.AwsS3.validateIncludeExclude(configuration);
@@ -126,35 +125,24 @@ public class AwsS3InputStreamFactory implements
IInputStreamFactory {
p = (matchers, key) -> true;
}
-S3Client s3Client =
ExternalDataUtils.AwsS3.buildAwsS3Client(configuration);
-
// Get all objects in a bucket and extract the paths to files
-ListObjectsV2Request.Builder listObjectsBuilder =
ListObjectsV2Request.builder().bucket(container);
-ExternalDataUtils.AwsS3.setPrefix(configuration, listObjectsBuilder);
-
-ListObjectsV2Response listObjectsResponse;
-boolean done = false;
-String newMarker = null;
+List filesOnly;
+String container = configuration.get(AwsS3.CONTAINER_NAME_FIELD_NAME);
+S3Client s3Client =
ExternalDataUtils.AwsS3.buildAwsS3Client(configuration);
try {
-while (!done) {
-// List the objects from the start, or from the last marker in
case of truncated result
-if (newMarker == null) {
-listObjectsResponse =
s3Client.listObjectsV2(listObjectsBuilder.build());
+filesOnly =
[asterixdb] 19/25: [NO ISSUE][*DB] Log at FATAL level when halting due to failure to stop active runtime
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 39eb84fadbd41911c63dc9891bb433c85954fd18
Author: Michael Blow
AuthorDate: Thu Feb 11 16:45:42 2021 -0500
[NO ISSUE][*DB] Log at FATAL level when halting due to failure to stop
active runtime
Change-Id: Ifa3d49704919417fd3c5b11538f6ec90bde03c73
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9987
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Michael Blow
Reviewed-by: Murtadha Hubail
---
.../src/main/java/org/apache/asterix/active/ActiveManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index 458008b..ba04967 100644
---
a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++
b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -179,7 +179,7 @@ public class ActiveManager {
try {
stopIfRunning(runtime, content.getTimeout(),
content.getUnit());
} catch (Throwable th) {
-LOGGER.warn("Failed to stop runtime: {}", runtimeId, th);
+LOGGER.fatal("Failed to stop runtime: {}", runtimeId, th);
ExitUtil.halt(ExitUtil.EC_UNCAUGHT_THROWABLE);
}
});
[asterixdb] 12/25: [ASTERIXDB-2819][IO] Make absolute & normalize IODevice paths
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5faef51562ce896bcb973dc321009808712e1659
Author: Ian Maxon
AuthorDate: Mon Jan 25 20:37:18 2021 -0800
[ASTERIXDB-2819][IO] Make absolute & normalize IODevice paths
This improves the issue of not canonicalizing IOManager paths
by at least making them absolute and removing redundant ./ and ../
parts of the path.
Change-Id: Ib71fd775eb8e4b1df548350152a7331316abd5b5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9444
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Michael Blow
---
.../src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java | 3 ++-
.../src/main/java/org/apache/hyracks/control/nc/io/IOManager.java | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
index a2d28e1..c826c2f 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
@@ -20,6 +20,7 @@ package org.apache.hyracks.api.io;
import java.io.File;
import java.io.Serializable;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
@@ -80,7 +81,7 @@ public class IODeviceHandle implements Serializable {
public static List getDevices(String[] ioDevices) {
List devices = new ArrayList<>();
for (String ioDevice : ioDevices) {
-String devPath = ioDevice.trim();
+String devPath =
Paths.get(ioDevice.trim()).toAbsolutePath().normalize().toString();
devices.add(new IODeviceHandle(new File(devPath), "."));
}
return devices;
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index da78a0d..dee61a4 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -414,9 +414,9 @@ public class IOManager implements IIOManager {
}
public IODeviceHandle getDevice(String fullPath) {
-Path full = Paths.get(fullPath);
+Path full = Paths.get(fullPath).normalize();
for (IODeviceHandle d : ioDevices) {
-if (full.startsWith(Paths.get(d.getMount().getAbsolutePath( {
+if
(full.startsWith(Paths.get(d.getMount().getAbsolutePath()).normalize())) {
return d;
}
}
[asterixdb] 13/25: Fixup merge commit 73d9c0f74e
This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit dc12fc01407314eefe40e2296fb840d9d2d2d8d7 Author: Michael Blow AuthorDate: Tue Feb 2 08:52:24 2021 -0500 Fixup merge commit 73d9c0f74e Change-Id: If5634b8c7360176e2a77d95ff39598eee4953108 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9803 Integration-Tests: Jenkins Reviewed-by: Hussain Towaileb Tested-by: Michael Blow Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9943 Reviewed-by: Michael Blow --- hyracks-fullstack/pom.xml | 25 - 1 file changed, 25 deletions(-) diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml index b895312..62cfbd5 100644 --- a/hyracks-fullstack/pom.xml +++ b/hyracks-fullstack/pom.xml @@ -283,31 +283,6 @@ maven-plugin-api 3.6.3 - -org.apache.maven -maven-model -3.6.3 - - -org.apache.maven -maven-artifact -3.6.3 - - -org.apache.maven -maven-compat -3.6.3 - - -org.apache.maven -maven-core -3.6.3 - - -org.apache.maven -maven-plugin-api -3.6.3 -
[asterixdb] 24/25: Merge "Merge branch 'gerrit/mad-hatter'" into cheshire-cat
This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit 56759ec7abd98ec9f1670dfe1b3dc61a396fff3d Merge: d337bb7 f72e4cf Author: Apache AsterixDB Gerrit AuthorDate: Fri Feb 12 23:37:48 2021 + Merge "Merge branch 'gerrit/mad-hatter'" into cheshire-cat .../asterix/common/config/ExternalProperties.java | 3 +- .../hyracks/control/common/config/OptionTypes.java | 258 + 2 files changed, 159 insertions(+), 102 deletions(-)
[asterixdb] 21/25: [ASTERIXDB-2815][COMP] DISTINCT in subquery gives wrong result
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 8ac45b4ba64c6020454fd72dd78c7e43dc4a90d7
Author: Dmitry Lychagin
AuthorDate: Wed Jan 13 16:05:09 2021 -0800
[ASTERIXDB-2815][COMP] DISTINCT in subquery gives wrong result
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix DISTINCT operator handling during subplan removal
- Added testcases
Change-Id: Idfe0aa09faba9bdad0c9c71e96d1facf07f401c0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9585
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Ali Alsuliman
(cherry picked from commit b7d6ddccf8477c74913b1b52b2536e0dd8975dcf)
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9984
Reviewed-by: Dmitry Lychagin
---
.../RemoveLeftOuterUnnestForLeftOuterJoinRule.java | 3 +-
.../subplan/InlineAllNtsInSubplanVisitor.java | 7 +-
...nlineLeftNtsInSubplanJoinFlatteningVisitor.java | 7 +-
.../SubplanSpecialFlatteningCheckVisitor.java | 2 +-
.../queries/subquery/query-ASTERIXDB-2815.sqlpp| 43 +
.../results/subquery/query-ASTERIXDB-2815.plan | 74 ++
.../query-ASTERIXDB-2815.1.ddl.sqlpp | 33 ++
.../query-ASTERIXDB-2815.2.update.sqlpp| 50 +++
.../query-ASTERIXDB-2815.3.query.sqlpp | 30 +
.../query-ASTERIXDB-2815.3.adm | 2 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++
.../operators/logical/DistinctOperator.java| 15 ++---
.../visitors/CardinalityInferenceVisitor.java | 2 +
13 files changed, 257 insertions(+), 16 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveLeftOuterUnnestForLeftOuterJoinRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveLeftOuterUnnestForLeftOuterJoinRule.java
index 364816b..775a1df 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveLeftOuterUnnestForLeftOuterJoinRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveLeftOuterUnnestForLeftOuterJoinRule.java
@@ -91,7 +91,8 @@ public class RemoveLeftOuterUnnestForLeftOuterJoinRule
implements IAlgebraicRewr
Triple checkGbyResult
=
checkUnnestAndGby(outerUnnest, gbyOperator);
// The argument for listify and not(is-missing(...)) check should be
variables.
-if (!isVariableReference(checkGbyResult.second) ||
!isVariableReference(checkGbyResult.third)) {
+if (!checkGbyResult.first || checkGbyResult.second == null ||
!isVariableReference(checkGbyResult.second)
+|| checkGbyResult.third == null ||
!isVariableReference(checkGbyResult.third)) {
return false;
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
index 78c4c5e..f7cbb9d 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
@@ -626,10 +626,11 @@ class InlineAllNtsInSubplanVisitor implements
IQueryOperatorVisitor distinctVarList = op.getDistinctByVarList();
for (LogicalVariable keyVar : correlatedKeyVars) {
-if (!distinctVarList.contains(keyVar)) {
-distinctVarList.add(keyVar);
+if (!op.isDistinctByVar(keyVar)) {
+VariableReferenceExpression keyVarRef = new
VariableReferenceExpression(keyVar);
+keyVarRef.setSourceLocation(op.getSourceLocation());
+op.getExpressions().add(new MutableObject<>(keyVarRef));
}
}
context.computeAndSetTypeEnvironmentForOperator(op);
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
index 12596ff..8326f85 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
@@ -361,10 +361,11 @@ class InlineLeftNtsInSubplanJoinFlatteningVisitor
implements IQueryOperatorVisit
if (!rewritten || !underJoin) {
