>From Ali Alsuliman <[email protected]>:
Ali Alsuliman has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17963 )
Change subject: WIP: Sqlpp in columnar
......................................................................
WIP: Sqlpp in columnar
Change-Id: I7ffa525c6022d087074914d6e0955df18b386c3f
---
M asterixdb/asterix-app/pom.xml
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
A
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionColumnarIT.java
A
asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_columnar.xml
A asterixdb/asterix-app/src/test/resources/runtimets/ignore_columnar.txt
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/Info.java
M
hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorNodePushable.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
A asterixdb/asterix-app/src/test/resources/cc-columnar.conf
11 files changed, 299 insertions(+), 4 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/63/17963/1
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index 9eed769..838e5ec 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -447,7 +447,7 @@
<profile>
<id>asterix-gerrit-asterix-app</id>
<properties>
-
<test.excludes>**/CloudStorageTest.java,**/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/*StaticPartitioning*Test.java,**/*Ssl*Test.java,**/Podman*.java,**/*AnalyzedExecutionTest.java,**/SqlppProfiledExecutionTest.java</test.excludes>
+
<test.excludes>**/CloudStorageTest.java,**/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/SqlppExecutionColumnarIT.java,**/*StaticPartitioning*Test.java,**/*Ssl*Test.java,**/Podman*.java,**/*AnalyzedExecutionTest.java,**/SqlppProfiledExecutionTest.java</test.excludes>
<itest.excludes>**/*.java</itest.excludes>
</properties>
<build>
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionColumnarIT.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionColumnarIT.java
new file mode 100644
index 0000000..8d44eef
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionColumnarIT.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.runtime;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.test.base.AsterixTestHelper;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the SQL++ runtime tests with columnar storage format.
+ */
+@RunWith(Parameterized.class)
+public class SqlppExecutionColumnarIT {
+
+ protected static final String TEST_CONFIG_FILE_NAME =
"src/test/resources/cc-columnar.conf";
+ private static final String DELTA_RESULT_PATH = "results_columnar";
+ private static Set<String> IGNORED;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ final TestExecutor testExecutor = new TestExecutor(DELTA_RESULT_PATH);
+ LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor);
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY,
TEST_CONFIG_FILE_NAME);
+ IGNORED = new HashSet<>(
+ AsterixTestHelper.readTestListFile(new
File("src/test/resources/runtimets/ignore_columnar.txt")));
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameters(name = "SqlppColumnarExecutionTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests("only_sqlpp.xml",
"testsuite_sqlpp_columnar.xml");
+ }
+
+ protected TestCaseContext tcCtx;
+
+ public SqlppExecutionColumnarIT(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @Test
+ public void test() throws Exception {
+ if (!IGNORED.contains(tcCtx.toString())) {
+ LangExecutionUtil.test(tcCtx);
+ }
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/resources/cc-columnar.conf
b/asterixdb/asterix-app/src/test/resources/cc-columnar.conf
new file mode 100644
index 0000000..4957bdc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/cc-columnar.conf
@@ -0,0 +1,64 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/asterix_nc1]
+txn.log.dir=target/tmp/asterix_nc1/txnlog
+core.dump.dir=target/tmp/asterix_nc1/coredump
+iodevices=target/tmp/asterix_nc1/iodevice1,
+iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2
+nc.api.port=19004
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
+
+[nc/asterix_nc2]
+ncservice.port=9091
+txn.log.dir=target/tmp/asterix_nc2/txnlog
+core.dump.dir=target/tmp/asterix_nc2/coredump
+iodevices=target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2
+nc.api.port=19005
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
+
+[nc]
+credential.file=src/test/resources/security/passwd
+python.cmd.autolocate=true
+python.env=FOO=BAR=BAZ,BAR=BAZ
+address=127.0.0.1
+command=asterixnc
+app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
+jvm.args=-Xmx4096m
-Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
+storage.buffercache.size=128MB
+storage.memorycomponent.globalbudget=512MB
+
+[cc]
+address = 127.0.0.1
+app.class=org.apache.asterix.hyracks.bootstrap.CCApplication
+heartbeat.period=2000
+heartbeat.max.misses=25
+credential.file=src/test/resources/security/passwd
+
+[common]
+log.dir = logs/
+log.level = INFO
+compiler.framesize=32KB
+compiler.sortmemory=320KB
+compiler.groupmemory=160KB
+compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
+compiler.windowmemory=192KB
+messaging.frame.size=4096
+messaging.frame.count=512
+storage.buffercache.pagesize=32KB
+storage.format=column
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/ignore_columnar.txt
b/asterixdb/asterix-app/src/test/resources/runtimets/ignore_columnar.txt
new file mode 100644
index 0000000..271a5a8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/ignore_columnar.txt
@@ -0,0 +1,93 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# different result
+objects: load-record-fields
+explain: explain_field_access
+explain: explain_field_access_closed
+
+## different result
+
+# unsupported types
+objects/get-object-fields: tiny-social-example
+objects/get-object-fields: tiny-social-example-no-complex-types
+objects/get-object-fields: tiny-social-example-only-records
+objects/get-object-fields: tiny-social-example-only-lists
+objects/get-object-field-value: tiny-social-example
+objects: object_pairs-2
+flwor: at01
+flwor: at02
+flwor: at03
+flwor: at07
+flwor: query-ASTERIXDB-2446-2
+sorting: range_hint
+sorting: records
+## unsupported types
+
+# multiset schema
+flwor: query-ASTERIXDB-883
+## multiset schema
+
+# union
+objects: object_length
+objects: object_names
+objects: object_remove
+objects: object_rename
+objects: object_replace
+objects: object_add
+objects: object_put
+objects: object_values
+objects: pairs
+sorting: arrays
+objects/get-object-fields: missing-fields
+objects: object_unwrap
+aggregate: issue395
+aggregate: avg_double
+aggregate: avg_double_null
+aggregate: avg_float
+aggregate: avg_float_null
+aggregate: avg_int16
+aggregate: avg_int16_null
+aggregate: avg_int32
+aggregate: avg_int32_null
+aggregate: avg_int64
+aggregate: avg_int64_null
+## union
+
+# type is null
+aggregate: avg_empty_01
+## type
+
+#
+geojson: datatype
+geojson: index
+geojson: two-geometries
+temporal: interval_bin_gby_0
+temporal: interval_bin_gby_1
+temporal: insert_from_ext_ds_2
+temporal/interval_joins: interval_after
+temporal/interval_joins: interval_before
+temporal/interval_joins: interval_covered_by
+temporal/interval_joins: interval_covers
+temporal/interval_joins: interval_ended_by
+temporal: overlap_bins_gby_3
+temporal: agg_min
+temporal: agg_max
+temporal: overlap_bins_gby_1
+temporal: overlap_bins_gby_0
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_columnar.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_columnar.xml
new file mode 100644
index 0000000..5b22470
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_columnar.xml
@@ -0,0 +1,28 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<!DOCTYPE test-suite [
+ <!ENTITY SqlppQueries SYSTEM "sqlpp_queries.xml">
+ <!ENTITY ObjectsQueries SYSTEM
"queries_sqlpp/objects/ObjectsQueries.xml">
+ <!ENTITY AsyncDeferredQueries SYSTEM
"queries_sqlpp/async-deferred/AsyncDeferredQueries.xml">
+ ]>
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org"
ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp"
QueryFileExtension=".sqlpp" SourceLocation="true">
+ &ObjectsQueries;
+ &AsyncDeferredQueries;
+ &SqlppQueries;
+</test-suite>
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index 34f5114..e257496 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -632,6 +632,15 @@
}
indexCheckpointManagerProvider.close(DatasetResourceReference.of(indexInfo.getLocalResource()));
indexInfo.setOpen(false);
+ } else {
+ ILSMIndex index = indexInfo.getIndex();
+ ILSMOperationTracker opTracker = index.getOperationTracker();
+ synchronized (opTracker) {
+ if (index.isActive()) {
+ index.deactivate(false);
+ }
+ }
+
indexCheckpointManagerProvider.close(DatasetResourceReference.of(indexInfo.getLocalResource()));
}
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/Info.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/Info.java
index fa0f14c..bf59f00 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/Info.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/Info.java
@@ -33,7 +33,8 @@
}
public void untouch() {
- referenceCount.decrementAndGet();
+ referenceCount.updateAndGet(i -> i > 0 ? i - 1 : i);
+ // referenceCount.decrementAndGet();
}
public int getReferenceCount() {
diff --git
a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorNodePushable.java
b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorNodePushable.java
index 785a330..932da99 100644
---
a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorNodePushable.java
+++
b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorNodePushable.java
@@ -44,10 +44,11 @@
@Override
public void initialize() throws HyracksDataException {
FrameTupleAppender appender = new FrameTupleAppender(new
VSizeFrame(ctx));
- if (fieldSlots != null && tupleData != null && tupleSize > 0)
+ if (fieldSlots != null && tupleData != null && tupleSize > 0) {
appender.append(fieldSlots, tupleData, 0, tupleSize);
- writer.open();
+ }
try {
+ writer.open();
appender.write(writer, false);
} catch (Throwable th) {
writer.fail();
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
index 406dbc6..cd26376 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
@@ -80,6 +80,7 @@
protected ISearchPredicate searchPred;
protected final IIndexDataflowHelper[] indexHelpers;
+ protected final boolean[] indexHelpersOpen;
protected IIndex[] indexes;
protected IIndexAccessor[] indexAccessors;
protected IIndexCursor[] cursors;
@@ -137,6 +138,7 @@
storagePartitionId2Index.put(partitions[i], i);
}
this.indexHelpers = new IIndexDataflowHelper[partitions.length];
+ this.indexHelpersOpen = new boolean[partitions.length];
this.indexes = new IIndex[partitions.length];
this.indexAccessors = new IIndexAccessor[partitions.length];
this.cursors = new IIndexCursor[partitions.length];
@@ -199,6 +201,7 @@
IIndexAccessParameters[] iaps = new
IndexAccessParameters[partitions.length];
for (int i = 0; i < partitions.length; i++) {
+ indexHelpersOpen[i] = true;
indexHelpers[i].open();
indexes[i] = indexHelpers[i].getIndexInstance();
searchCallbacks[i] = searchCallbackFactory
@@ -365,6 +368,8 @@
failure = ResourceReleaseUtils.close(cursors[i], failure);
failure = CleanupUtils.destroy(failure, cursors[i],
indexAccessors[i]);
failure = ResourceReleaseUtils.close(indexHelpers[i],
failure);
+ } else if (indexHelpersOpen[i]) {
+ failure = ResourceReleaseUtils.close(indexHelpers[i],
failure);
}
} catch (Throwable th) {// NOSONAR ensure closing other indexes
// subsequently, the failure will be thrown
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index 4756921..f8f895b 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -43,6 +43,8 @@
*/
public interface ILSMIndex extends IIndex {
+ boolean isActive();
+
boolean isAtomic();
/**
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 33fd38d..e28ac9f 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -175,6 +175,11 @@
}
@Override
+ public synchronized boolean isActive() {
+ return isActive && !isDeactivating;
+ }
+
+ @Override
public boolean isAtomic() {
return atomic;
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17963
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I7ffa525c6022d087074914d6e0955df18b386c3f
Gerrit-Change-Number: 17963
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <[email protected]>
Gerrit-MessageType: newchange
