(asterixdb) branch master updated: [NO ISSUE][*DB] Refactoring AssignRuntime Factory
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2374748ec2 [NO ISSUE][*DB] Refactoring AssignRuntime Factory
2374748ec2 is described below
commit 2374748ec276ff4d6f4f97e62c130f3a7633f36c
Author: Ritik Raj
AuthorDate: Tue Apr 23 02:40:16 2024 +0530
[NO ISSUE][*DB] Refactoring AssignRuntime Factory
Change-Id: Ieb583580f7bc8a40a15839ce15c492aa0bfb410c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18249
Integration-Tests: Jenkins
Reviewed-by: Peeyush Gupta
Tested-by: Murtadha Hubail
---
.../operators/std/AssignRuntimeFactory.java| 173 -
1 file changed, 97 insertions(+), 76 deletions(-)
diff --git
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java
index 5343069a4f..300fac64d4 100644
---
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java
+++
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java
@@ -39,10 +39,9 @@ import
org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
public class AssignRuntimeFactory extends
AbstractOneInputOneOutputRuntimeFactory {
private static final long serialVersionUID = 1L;
-
-private int[] outColumns;
-private IScalarEvaluatorFactory[] evalFactories;
-private final boolean flushFramesRapidly;
+protected int[] outColumns;
+protected IScalarEvaluatorFactory[] evalFactories;
+protected final boolean flushFramesRapidly;
/**
* @param outColumns
@@ -64,6 +63,14 @@ public class AssignRuntimeFactory extends
AbstractOneInputOneOutputRuntimeFactor
this.flushFramesRapidly = flushFramesRapidly;
}
+public int[] getOutColumns() {
+return outColumns;
+}
+
+public IScalarEvaluatorFactory[] getEvalFactories() {
+return evalFactories;
+}
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -88,92 +95,106 @@ public class AssignRuntimeFactory extends
AbstractOneInputOneOutputRuntimeFactor
@Override
public AbstractOneInputOneOutputOneFramePushRuntime
createOneOutputPushRuntime(final IHyracksTaskContext ctx)
throws HyracksDataException {
-IEvaluatorContext evalCtx = new EvaluatorContext(ctx);
-final int[] projectionToOutColumns = new int[projectionList.length];
-for (int j = 0; j < projectionList.length; j++) {
-projectionToOutColumns[j] = Arrays.binarySearch(outColumns,
projectionList[j]);
+return new AssignRuntime(ctx);
+}
+
+public int[] getProjectionList() {
+return projectionList;
+}
+
+public class AssignRuntime extends
AbstractOneInputOneOutputOneFramePushRuntime {
+private IPointable result;
+private IScalarEvaluator[] eval;
+protected ArrayTupleBuilder tupleBuilder;
+private final int[] projectionToOutColumns;
+private boolean first = true;
+protected int tupleIndex = 0;
+protected final IHyracksTaskContext ctx;
+protected final IEvaluatorContext evalCtx;
+
+public AssignRuntime(IHyracksTaskContext ctx) {
+this.ctx = ctx;
+this.evalCtx = new EvaluatorContext(ctx);
+projectionToOutColumns = new int[projectionList.length];
+for (int j = 0; j < projectionList.length; j++) {
+projectionToOutColumns[j] = Arrays.binarySearch(outColumns,
projectionList[j]);
+}
+tupleBuilder = new ArrayTupleBuilder(projectionList.length);
+eval = new IScalarEvaluator[evalFactories.length];
+result = VoidPointable.FACTORY.createPointable();
}
-return new AbstractOneInputOneOutputOneFramePushRuntime() {
-private IPointable result =
VoidPointable.FACTORY.createPointable();
-private IScalarEvaluator[] eval = new
IScalarEvaluator[evalFactories.length];
-private ArrayTupleBuilder tupleBuilder = new
ArrayTupleBuilder(projectionList.length);
-private boolean first = true;
-private int tupleIndex = 0;
-
-@Override
-public void open() throws HyracksDataException {
-if (first) {
-initAccessAppendRef(ctx);
-first = false;
-int n = evalFactories.length;
-for (int i = 0; i
(asterixdb) branch master updated: [ASTERIXDB-3386][CONF] Add runtime memory overhead config
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 48a3a9d4f8 [ASTERIXDB-3386][CONF] Add runtime memory overhead config
48a3a9d4f8 is described below
commit 48a3a9d4f8d223c163e00b6d04fea9186626713c
Author: Murtadha Hubail
AuthorDate: Mon Apr 22 22:41:01 2024 +0300
[ASTERIXDB-3386][CONF] Add runtime memory overhead config
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Add a new compiler configuration that allows specifying
a percentage of memory to be added to the job's required
memory to account for runtime memory overhead and default
it to 5%.
Change-Id: I33a1911ff6b0df64b273ba10a7626dd10cc4cca2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18246
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
Tested-by: Jenkins
---
.../java/org/apache/asterix/api/common/APIFramework.java| 11 +++
.../results/api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../cluster_state_1_full/cluster_state_1_full.1.regexadm| 1 +
.../cluster_state_1_less/cluster_state_1_less.1.regexadm| 1 +
.../misc/dataset-resources/dataset-resources.6.regex| 2 +-
.../misc/dataset-resources/dataset-resources.7.regex| 2 +-
.../test/resources/runtimets/results/misc/jobs/jobs.2.regex | 2 +-
.../test/resources/runtimets/results/misc/ping/ping.2.regex | 2 +-
.../apache/asterix/common/config/CompilerProperties.java| 13 -
9 files changed, 26 insertions(+), 9 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 3da25fe311..16f922f6d7 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -342,6 +342,7 @@ public class APIFramework {
getJobLocations(spec, nodeJobTracker,
computationLocations);
final IClusterCapacity jobRequiredCapacity =
ResourceUtils.getRequiredCapacity(plan,
jobLocations, physOptConf);
+addRuntimeMemoryOverhead(jobRequiredCapacity,
compilerProperties);
spec.setRequiredClusterCapacity(jobRequiredCapacity);
}
}
@@ -600,4 +601,14 @@ public class APIFramework {
return new
AlgebricksAbsolutePartitionConstraint(Arrays.stream(clusterLocations.getLocations())
.filter(jobParticipatingNodes::contains).toArray(String[]::new));
}
+
+private static void addRuntimeMemoryOverhead(IClusterCapacity
jobRequiredCapacity,
+CompilerProperties compilerProperties) {
+int runtimeMemoryOverheadPercentage =
compilerProperties.getRuntimeMemoryOverheadPercentage();
+if (runtimeMemoryOverheadPercentage > 0) {
+double multiplier = 1 + runtimeMemoryOverheadPercentage / 100.0;
+long aggregatedMemoryByteSize =
jobRequiredCapacity.getAggregatedMemoryByteSize();
+jobRequiredCapacity.setAggregatedMemoryByteSize((long)
(aggregatedMemoryByteSize * multiplier));
+}
+}
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 1bb6ef9c97..6888accc37 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -32,6 +32,7 @@
"compiler\.min\.memory\.allocation" : true,
"compiler\.parallelism" : 0,
"compiler.queryplanshape" : "zigzag",
+"compiler.runtime.memory.overhead" : 5,
"compiler\.sort\.parallel" : false,
"compiler\.sort\.samples" : 100,
"compiler\.sortmemory" : 327680,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index c2883ae935..31daf991a3 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -32,6 +32,7 @@
(asterixdb) branch master updated: [ASTERIXDB-3383][RT] resolving comments
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 44630dd476 [ASTERIXDB-3383][RT] resolving comments
44630dd476 is described below
commit 44630dd476f577c769b0316cfc871dc52eb4c195
Author: Ritik Raj
AuthorDate: Sun Apr 21 21:58:07 2024 +0530
[ASTERIXDB-3383][RT] resolving comments
Change-Id: I694d41db144eb8375171a0913cec837ac3fdab0e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18245
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Ritik Raj
Reviewed-by: Murtadha Hubail
---
.../main/java/org/apache/asterix/metadata/utils/DatasetUtil.java | 1 -
.../hyracks/storage/am/lsm/common/api/ILSMTupleFilterCallback.java | 7 +++
.../storage/am/lsm/common/api/ILSMTupleFilterCallbackFactory.java | 2 --
3 files changed, 3 insertions(+), 7 deletions(-)
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index becdbc1463..923dbd4547 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -533,7 +533,6 @@ public class DatasetUtil {
}
RecordDescriptor outputRecordDesc = new RecordDescriptor(outputSerDes,
outputTypeTraits);
-// get the Tuple filter callback
ILSMTupleFilterCallbackFactory tupleFilterCallbackFactory =
dataset.getTupleFilterCallbackFactory();
// This allows to project only the indexed fields instead of the
entirety of the record
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTupleFilterCallback.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTupleFilterCallback.java
index 05a9e1a947..c2229fbc35 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTupleFilterCallback.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTupleFilterCallback.java
@@ -26,14 +26,13 @@ import
org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
public interface ILSMTupleFilterCallback extends Serializable {
/**
- * Populates the state within the filter callback by fetching the state
from the provided "index"
- * and mapping it with the storage partition index, which will be utilized
for filtering
- * the tuple directed to the index "storagePartitionIdx".
+ * Initializes the filter callback
*/
void initialize(ILSMIndex index) throws HyracksDataException;
/**
- * Filter the received record based on the initialized ingestion state.
+ * This method is called on a tuple to evaluate whether the tuple meets
the criteria
+ * for filtration based on the specified filter condition.
*/
boolean filter(FrameTupleAccessor accessor, int tupleIdx);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTupleFilterCallbackFactory.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTupleFilterCallbackFactory.java
index dc271fd376..011d48f624 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTupleFilterCallbackFactory.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMTupleFilterCallbackFactory.java
@@ -24,8 +24,6 @@ import java.io.Serializable;
public interface ILSMTupleFilterCallbackFactory extends Serializable {
/**
* Creates a callback function that utilizes the field permutation of the
incoming tuple.
- * The field tuple contains information about the position of the record,
meta, and primary key,
- * facilitating the extraction of relevant information for filtering
purposes.
*/
ILSMTupleFilterCallback createTupleFilterCallback(int[] fieldPermutation);
}
(asterixdb) branch master updated: [ASTERIXDB-3383][RT] Introducing tuple filter to ingestion pipeline
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 414ea84d87 [ASTERIXDB-3383][RT] Introducing tuple filter to ingestion
pipeline
414ea84d87 is described below
commit 414ea84d87b9fc4952d40fdcdf3f192694cce07e
Author: Ritik Raj
AuthorDate: Thu Apr 18 00:47:29 2024 +0530
[ASTERIXDB-3383][RT] Introducing tuple filter to ingestion pipeline
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- excludes the tuple from being processed
based on the provided filter.
Change-Id: Id79607bdada1cd42949cccf43390a90dda092602
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18239
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Tested-by: Murtadha Hubail
---
.../asterix/app/bootstrap/TestNodeController.java | 3 +-
.../apache/asterix/metadata/entities/Dataset.java | 5 +++
.../entities/NoOpLSMTupleFilterCallback.java | 40 ++
.../NoOpLSMTupleFilterCallbackFactory.java | 33 ++
.../apache/asterix/metadata/utils/DatasetUtil.java | 6 +++-
.../LSMPrimaryUpsertOperatorDescriptor.java| 9 +++--
.../LSMPrimaryUpsertOperatorNodePushable.java | 16 -
.../am/lsm/common/api/ILSMTupleFilterCallback.java | 39 +
.../common/api/ILSMTupleFilterCallbackFactory.java | 31 +
9 files changed, 176 insertions(+), 6 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index fc992e3404..668ff13caa 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -52,6 +52,7 @@ import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.NoOpLSMTupleFilterCallbackFactory;
import org.apache.asterix.metadata.utils.DatasetUtil;
import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil;
import org.apache.asterix.om.types.ARecordType;
@@ -843,7 +844,7 @@ public class TestNodeController {
frameOpCallbackFactory == null ?
dataset.getFrameOpCallbackFactory(mdProvider)
: frameOpCallbackFactory,
MissingWriterFactory.INSTANCE, hasSecondaries,
NoOpTupleProjectorFactory.INSTANCE,
-tuplePartitionerFactory, partitionsMap);
+tuplePartitionerFactory, partitionsMap,
NoOpLSMTupleFilterCallbackFactory.INSTANCE);
RecordDescriptor upsertOutRecDesc =
getUpsertOutRecDesc(primaryIndexInfo.rDesc, dataset,
filterFields == null ? 0 : filterFields.length, recordType,
metaType);
// fix pk fields
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index 85fbad190f..9d41097111 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -116,6 +116,7 @@ import
org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFacto
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import
org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
import
org.apache.hyracks.storage.am.lsm.common.api.ILSMPageWriteCallbackFactory;
+import
org.apache.hyracks.storage.am.lsm.common.api.ILSMTupleFilterCallbackFactory;
import org.apache.hyracks.storage.common.IResourceFactory;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -877,4 +878,8 @@ public class Dataset implements IMetadataEntity,
IDataset {
public DatasetFormatInfo getDatasetFormatInfo() {
return datasetFormatInfo;
}
+
+public ILSMTupleFilterCallbackFactory getTupleFilterCallbackFactory() {
+return NoOpLSMTupleFilterCallbackFactory.INSTANCE;
+}
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NoOpLSMTupleFilterCallback.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NoOpLSMTupleFilterCallback.java
new file mode 100644
index 00..0957a96542
--- /dev/null
+++
b/asterixdb/asterix
(asterixdb) branch master updated: MB-61438: Supporting Plan for Id Extraction from Value
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9203c2fe9e MB-61438: Supporting Plan for Id Extraction from Value
9203c2fe9e is described below
commit 9203c2fe9e55c0cb1f6496737bdc5c94375bf98b
Author: Ritik Raj
AuthorDate: Sun Apr 7 20:18:48 2024 +0530
MB-61438: Supporting Plan for Id Extraction from Value
Change-Id: I2d4fd57af4f71a3b8cf45ed708a1500436565e6e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18229
Integration-Tests: Jenkins
Reviewed-by: Peeyush Gupta
Tested-by: Murtadha Hubail
---
.../apache/hyracks/data/std/util/ByteArrayAccessibleInputStream.java | 2 +-
.../dataflow/LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.java| 4
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git
a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ByteArrayAccessibleInputStream.java
b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ByteArrayAccessibleInputStream.java
index 7f1d451988..c1b4d9e1e2 100644
---
a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ByteArrayAccessibleInputStream.java
+++
b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ByteArrayAccessibleInputStream.java
@@ -29,7 +29,7 @@ public class ByteArrayAccessibleInputStream extends
ByteArrayInputStream {
public void setContent(byte[] buf, int offset, int length) {
this.buf = buf;
this.pos = offset;
-this.count = Math.min(offset + length, buf.length);
+this.count = Math.min(offset + length, buf == null ? 0 : buf.length);
this.mark = offset;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.java
index 26ce56af96..92813199b5 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.java
@@ -67,4 +67,8 @@ public class LSMTreeIndexInsertUpdateDeleteOperatorDescriptor
extends AbstractSi
recordDescProvider.getInputRecordDescriptor(getActivityId(),
0), op, modCallbackFactory,
tupleFilterFactory, tuplePartitionerFactory, partitionsMap);
}
+
+public int[] getFieldPermutation() {
+return fieldPermutation;
+}
}
(asterixdb) branch master updated: [ASTERIXDB-3372][COMP]: Validate minimum value for 'max-objects-per-file'
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 899ce953ca [ASTERIXDB-3372][COMP]: Validate minimum value for
'max-objects-per-file'
899ce953ca is described below
commit 899ce953ca0753cd7f796c1a3329c3865aad8a09
Author: Hussain Towaileb
AuthorDate: Mon Apr 1 20:31:24 2024 +0300
[ASTERIXDB-3372][COMP]: Validate minimum value for 'max-objects-per-file'
Change-Id: I13c4a516fa8cf76166c2b04aca4ce5c06fd69aa1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18216
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Hussain Towaileb
Reviewed-by: Wail Alkowaileet
---
.../empty-over.01.ddl.sqlpp| 39 ++
.../empty-over.02.update.sqlpp | 36
.../empty-over.03.update.sqlpp | 36
.../runtimets/testsuite_external_dataset_s3.xml| 7
.../asterix/common/exceptions/ErrorCode.java | 1 +
.../src/main/resources/asx_errormsg/en.properties | 1 +
.../external/util/ExternalDataConstants.java | 1 +
.../external/util/WriterValidationUtil.java| 12 +--
8 files changed, 130 insertions(+), 3 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.01.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.01.ddl.sqlpp
new file mode 100644
index 00..b68c38b195
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.01.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+("accessKeyId"="dummyAccessKey"),
+("secretAccessKey"="dummySecretKey"),
+("region"="us-west-2"),
+("serviceEndpoint"="http://127.0.0.1:8001;),
+("container"="playground"),
+
("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+("embed-filter-values" = "false"),
+("format"="json")
+);
+
+
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.02.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.02.update.sqlpp
new file mode 100644
index 00..02f6c96f1d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/bad-max-objects-per-file/empty-over.02.update.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("copy-to-result-with-over")
+WITH {
+"accessKeyId":"dummyAccessKey",
+"secretAccessKey":"dummySecr
(asterixdb) branch master updated: [ASTERIXDB-3370][COMP] Collect accessed entities during compilation
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new acf4d6338d [ASTERIXDB-3370][COMP] Collect accessed entities during
compilation
acf4d6338d is described below
commit acf4d6338d88e18f7fed4bbbe2ff5e84ceeb8a36
Author: AnkitPrabhu
AuthorDate: Thu Apr 4 00:18:24 2024 +0530
[ASTERIXDB-3370][COMP] Collect accessed entities during compilation
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Collect datasets/views/functions/link for copy-to statement
- Add visitor to collect accessed datasets/views/functions/copyto
Change-Id: Icb2529cd876bbbf40b92635aca0aa9693e9cd154
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18212
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Tested-by: Jenkins
---
.../asterix/app/translator/QueryTranslator.java| 19 +++--
.../lang/sqlpp/rewrites/SqlppQueryRewriter.java| 9 ++
.../rewrites/visitor/SqlppLoadAccessedDataset.java | 95 ++
.../metadata/declared/MetadataProvider.java| 15
.../asterix/metadata/entities/EntityDetails.java | 59 ++
5 files changed, 189 insertions(+), 8 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 6424280bbd..3e674fd282 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -448,16 +448,16 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
handleLibraryDropStatement(metadataProvider, stmt,
hcc, requestParameters);
break;
case CREATE_SYNONYM:
-handleCreateSynonymStatement(metadataProvider, stmt);
+handleCreateSynonymStatement(metadataProvider, stmt,
requestParameters);
break;
case SYNONYM_DROP:
-handleDropSynonymStatement(metadataProvider, stmt);
+handleDropSynonymStatement(metadataProvider, stmt,
requestParameters);
break;
case CREATE_VIEW:
handleCreateViewStatement(metadataProvider, stmt,
stmtRewriter, requestParameters);
break;
case VIEW_DROP:
-handleViewDropStatement(metadataProvider, stmt);
+handleViewDropStatement(metadataProvider, stmt,
requestParameters);
break;
case LOAD:
if (stats.getProfileType() == Stats.ProfileType.FULL) {
@@ -2963,7 +2963,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
}
-public void handleViewDropStatement(MetadataProvider metadataProvider,
Statement stmt) throws Exception {
+public void handleViewDropStatement(MetadataProvider metadataProvider,
Statement stmt,
+IRequestParameters requestParameters) throws Exception {
ViewDropStatement stmtDrop = (ViewDropStatement) stmt;
SourceLocation sourceLoc = stmtDrop.getSourceLocation();
String viewName = stmtDrop.getViewName().getValue();
@@ -2976,14 +2977,14 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
lockUtil.dropDatasetBegin(lockManager, metadataProvider.getLocks(),
databaseName, dataverseName, viewName);
try {
-doDropView(metadataProvider, stmtDrop, databaseName,
dataverseName, viewName);
+doDropView(metadataProvider, stmtDrop, databaseName,
dataverseName, viewName, requestParameters);
} finally {
metadataProvider.getLocks().unlock();
}
}
protected boolean doDropView(MetadataProvider metadataProvider,
ViewDropStatement stmtViewDrop, String databaseName,
-DataverseName dataverseName, String viewName) throws Exception {
+DataverseName dataverseName, String viewName, IRequestParameters
requestParameters) throws Exception {
SourceLocation sourceLoc = stmtViewDrop.getSourceLocation();
MetadataTransactionContext mdTxnCtx =
MetadataManager.INSTANCE.beginTransaction();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
@@ -3787,7 +3788,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
}
-protected void handleCreateSynonymStatement(MetadataProvider
(asterixdb) branch master updated: [NO ISSUE]: Support Reading Single Depth Files/Folder from S3 Container
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 381c707930 [NO ISSUE]: Support Reading Single Depth Files/Folder from
S3 Container
381c707930 is described below
commit 381c7079309572c35eee46142c5fcbcec1151de0
Author: utsavCbase
AuthorDate: Thu Mar 21 13:38:27 2024 +0530
[NO ISSUE]: Support Reading Single Depth Files/Folder from S3 Container
Change-Id: Iae9d85eb9899419e63c86322cd8da2273adf89c6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18206
Reviewed-by: Utsav Singh
Reviewed-by: Hussain Towaileb
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../external_dataset/ExternalDatasetTestUtils.java | 38 +++
.../aws/AwsS3ExternalDatasetTest.java | 1 +
.../asterix/external/util/aws/s3/S3Utils.java | 57 ++
3 files changed, 96 insertions(+)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index 55a515c879..34d209eb7a 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -20,6 +20,7 @@ package org.apache.asterix.test.external_dataset;
import static
org.apache.asterix.test.external_dataset.avro.AvroFileConverterUtil.AVRO_GEN_BASEDIR;
import static
org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.BOM_FILE_CONTAINER;
+import static
org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.BROWSE_CONTAINER;
import static
org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.DYNAMIC_PREFIX_AT_START_CONTAINER;
import static
org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.FIXED_DATA_CONTAINER;
import static
org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil.BINARY_GEN_BASEDIR;
@@ -78,6 +79,7 @@ public class ExternalDatasetTestUtils {
private static Uploader fixedDataLoader;
private static Uploader mixedDataLoader;
private static Uploader bomFileLoader;
+private static Uploader browseDataLoader;
protected TestCaseContext tcCtx;
@@ -148,6 +150,16 @@ public class ExternalDatasetTestUtils {
ExternalDatasetTestUtils.bomFileLoader = bomFileLoader;
}
+public static void setUploaders(Uploader playgroundDataLoader, Uploader
dynamicPrefixAtStartDataLoader,
+Uploader fixedDataLoader, Uploader mixedDataLoader, Uploader
bomFileLoader, Uploader browseDataLoader) {
+ExternalDatasetTestUtils.playgroundDataLoader = playgroundDataLoader;
+ExternalDatasetTestUtils.dynamicPrefixAtStartDataLoader =
dynamicPrefixAtStartDataLoader;
+ExternalDatasetTestUtils.fixedDataLoader = fixedDataLoader;
+ExternalDatasetTestUtils.mixedDataLoader = mixedDataLoader;
+ExternalDatasetTestUtils.bomFileLoader = bomFileLoader;
+ExternalDatasetTestUtils.browseDataLoader = browseDataLoader;
+}
+
/**
* Creates a bucket and fills it with some files for testing purpose.
*/
@@ -183,6 +195,32 @@ public class ExternalDatasetTestUtils {
LOGGER.info("Files added successfully");
}
+public static void prepareBrowseContainer() {
+/*
+file hierarchy inside browse container
+browse/1.json
+browse/2.json
+browse/level1/3.json
+browse/level1/4.json
+browse/level1/level2/5.json
+browse/level2/level3/6.json
+ */
+// -- todo:Utsav add a test for Browse S3 path which returns multiple
folders, skipped for now as S3 mock server does not support this.
+LOGGER.info("Adding JSON files to " + BROWSE_CONTAINER);
+browseDataLoader.upload("1.json", "{\"id\":" + 1 + "}");
+browseDataLoader.upload("2.json", "{\"id\":" + 2 + "}");
+browseDataLoader.upload("level1/3.json", "{\"id\":" + 3 + "}");
+browseDataLoader.upload("level1/4.json", "{\"id\":" + 4 + "}");
+browseDataLoader.upload("level1/level2/5.json", "{\"id\":" + 5 + "}");
+browseDataLoader.upload("level2/level3/6.json", "{\"id\":" + 6 + "}");
+
+//Adding 1000+ files
+for (int i = 1; i <= 1500; i++) {
+browseDataLoader.upload("level3/" + i + ".json&quo
(asterixdb) branch master updated: [NO ISSUE][EXT] Configurable Gzip compression level
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 0a485b [NO ISSUE][EXT] Configurable Gzip compression level
0a485b is described below
commit 0a485b6ea8739debd3602b2f323e0f35d244
Author: Murtadha Hubail
AuthorDate: Mon Mar 18 02:21:16 2024 +0300
[NO ISSUE][EXT] Configurable Gzip compression level
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add an option "gzipCompressionLevel" to specify the
compression level (1-9) to be used when gzip is used
with COPY TO.
- Default gzip compression level to -1 (library default)
when not specified.
- Make the compression buffer size equal to a frame size.
Change-Id: I6e80691e232269620d76e6b6f414cff6856f3232
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18204
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Wail Alkowaileet
---
.../copy-to/query/query.02.update.sqlpp| 3 +-
.../external/util/ExternalDataConstants.java | 1 +
.../asterix/external/util/ExternalDataUtils.java | 4 +++
.../external/util/WriterValidationUtil.java| 23
.../GzipExternalFileCompressStreamFactory.java | 17 ++--
.../metadata/declared/MetadataProvider.java| 2 +-
.../metadata/provider/ExternalWriterProvider.java | 32 --
7 files changed, 69 insertions(+), 13 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp
index 6df40c3fcb..1d7253b21b 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp
@@ -36,7 +36,8 @@ WITH {
"serviceEndpoint":"http://127.0.0.1:8001;,
"container":"playground",
"format":"json",
-"compression":"gzip"
+"compression":"gzip",
+"gzipCompressionLevel": "1"
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index fc8f527047..79252ada33 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -304,6 +304,7 @@ public class ExternalDataConstants {
* Compression constants
*/
public static final String KEY_COMPRESSION_GZIP = "gzip";
+public static final String KEY_COMPRESSION_GZIP_COMPRESSION_LEVEL =
"gzipCompressionLevel";
/**
* Writer Constants
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index ae3b567f79..b02122d719 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -1085,4 +1085,8 @@ public class ExternalDataUtils {
return ExternalDataConstants.KEY_PATH;
}
}
+
+public static boolean isGzipCompression(String compression) {
+return
ExternalDataConstants.KEY_COMPRESSION_GZIP.equalsIgnoreCase(compression);
+}
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
index 00fe855195..b348c1e21c 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.external.util;
+import static
org.apache.asterix.common.exceptions.ErrorCode.INVALID_REQ_PARAM_VAL;
+import static
org.apache.asterix.external.util.ExternalDataConstants.KEY_FORMAT;
+
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -59,6 +62,9 @@ public class WriterValidationUtil {
checkSupported(ExternalDataConstants.KEY_WRITER_COMPRESSION,
compres
(asterixdb) branch master updated: [NO ISSUE][EXT] Change default max objects per file
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new ae06ade5da [NO ISSUE][EXT] Change default max objects per file
ae06ade5da is described below
commit ae06ade5daf5099065f7ad181012f284c92f1edb
Author: Murtadha Hubail
AuthorDate: Fri Mar 15 01:51:20 2024 +0300
[NO ISSUE][EXT] Change default max objects per file
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Change the default of maximum object per file in
COPY TO from 1K to 10K to avoid writing many small
files to external storage.
Change-Id: I793251dd17be808eb3b9faf62504e422bc644634
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18202
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Hussain Towaileb
---
.../java/org/apache/asterix/external/util/ExternalDataConstants.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index c0af89fa65..fc8f527047 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -310,7 +310,7 @@ public class ExternalDataConstants {
*/
public static final String KEY_WRITER_MAX_RESULT = "max-objects-per-file";
public static final String KEY_WRITER_COMPRESSION = "compression";
-public static final int WRITER_MAX_RESULT_DEFAULT = 1000;
+public static final int WRITER_MAX_RESULT_DEFAULT = 1;
public static final Set WRITER_SUPPORTED_FORMATS;
public static final Set WRITER_SUPPORTED_ADAPTERS;
public static final Set WRITER_SUPPORTED_COMPRESSION;
(asterixdb) branch master updated: [ASTERIXDB-3364][STO] Use bootstrap marker during metadata bootstrap
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 909122c3b8 [ASTERIXDB-3364][STO] Use bootstrap marker during metadata
bootstrap
909122c3b8 is described below
commit 909122c3b8e5ed33e048967a0bce0f4eede5a42a
Author: Murtadha Hubail
AuthorDate: Tue Mar 12 02:29:04 2024 +0300
[ASTERIXDB-3364][STO] Use bootstrap marker during metadata bootstrap
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Before bootstrapping the metadata catalog for the first time,
put a bootstrap marker (file) in the blob storage to indicate
that the metadata hasn't been fully bootstrapped.
- Delete the bootstrap marker from the blob storage on successful
metadata bootstrap attempts.
- When checking the system state on missing checkpoints, check
for the bootstrap marker to detect failed bootstrap attempts.
- Delete any existing metadata files from the blob storage if
the bootstrap marker is found to start from a clean state.
Change-Id: I805bf1cbdfb58690876da0c68840ed2c365742a4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18199
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Wail Alkowaileet
---
.../nc/task/CloudToLocalStorageCachingTask.java| 6 ++--
.../asterix/app/nc/task/MetadataBootstrapTask.java | 26 ++-
.../asterix/cloud/AbstractCloudIOManager.java | 39 ++
.../asterix/cloud/LocalPartitionBootstrapper.java | 3 +-
.../cloud/bulk/DeleteBulkCloudOperation.java | 4 +--
.../common/cloud/IPartitionBootstrapper.java | 3 +-
.../asterix/common/utils/StorageConstants.java | 1 +
.../asterix/common/utils/StoragePathUtil.java | 15 +
8 files changed, 83 insertions(+), 14 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
index a44a695718..6b3257d870 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.asterix.common.api.INCLifecycleTask;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.cloud.IPartitionBootstrapper;
+import org.apache.asterix.common.transactions.Checkpoint;
import
org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
import org.apache.hyracks.api.control.CcId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -58,9 +59,10 @@ public class CloudToLocalStorageCachingTask implements
INCLifecycleTask {
String nodeId = applicationContext.getServiceContext().getNodeId();
LOGGER.info("Initializing Node {} with storage partitions: {}",
nodeId, storagePartitions);
+Checkpoint latestCheckpoint =
applicationContext.getTransactionSubsystem().getCheckpointManager().getLatest();
IPartitionBootstrapper bootstrapper =
applicationContext.getPartitionBootstrapper();
-bootstrapper.bootstrap(storagePartitions, lrs.getOnDiskPartitions(),
metadataNode, metadataPartitionId,
-cleanup);
+bootstrapper.bootstrap(storagePartitions, lrs.getOnDiskPartitions(),
metadataNode, metadataPartitionId, cleanup,
+latestCheckpoint == null);
}
@Override
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java
index f58f871207..246983de78 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MetadataBootstrapTask.java
@@ -21,8 +21,11 @@ package org.apache.asterix.app.nc.task;
import org.apache.asterix.common.api.INCLifecycleTask;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.hyracks.api.control.CcId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.service.IControllerService;
public class MetadataBootstrapTask implements INCLifecycleTask {
@@ -40,12 +43,33 @@ pu
(asterixdb) branch master updated: [ASTERIXDB-3362][MISC] Update formatter language versions
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new d0c119756c [ASTERIXDB-3362][MISC] Update formatter language versions d0c119756c is described below commit d0c119756c3c13f422a209350ae0fa34460cbee6 Author: Ian Maxon AuthorDate: Mon Mar 4 16:14:13 2024 -0800 [ASTERIXDB-3362][MISC] Update formatter language versions Change-Id: I603ae267efd137d4e9f3491be2a6bdcb1179acdc Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18188 Integration-Tests: Jenkins Tested-by: Jenkins Reviewed-by: Ian Maxon Reviewed-by: Peeyush Gupta --- asterixdb/AsterixCodeFormatProfile.xml | 6 +++--- hyracks-fullstack/AsterixCodeFormatProfile.xml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/asterixdb/AsterixCodeFormatProfile.xml b/asterixdb/AsterixCodeFormatProfile.xml index 4205bb6317..b9ebb729bb 100644 --- a/asterixdb/AsterixCodeFormatProfile.xml +++ b/asterixdb/AsterixCodeFormatProfile.xml @@ -67,7 +67,7 @@ - + @@ -183,7 +183,7 @@ - + @@ -256,7 +256,7 @@ - + diff --git a/hyracks-fullstack/AsterixCodeFormatProfile.xml b/hyracks-fullstack/AsterixCodeFormatProfile.xml index 4205bb6317..b9ebb729bb 100644 --- a/hyracks-fullstack/AsterixCodeFormatProfile.xml +++ b/hyracks-fullstack/AsterixCodeFormatProfile.xml @@ -67,7 +67,7 @@ - + @@ -183,7 +183,7 @@ - + @@ -256,7 +256,7 @@ - +
(asterixdb) branch master updated: [ASTERIXDB-3357][COMP][RT] Compiler and runtime support for COPY TO statement
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7250ba6730 [ASTERIXDB-3357][COMP][RT] Compiler and runtime support for
COPY TO statement
7250ba6730 is described below
commit 7250ba6730a6565d86ddf9bb794cb8ac32ef4386
Author: Hussain Towaileb
AuthorDate: Tue Jan 30 06:21:39 2024 +0300
[ASTERIXDB-3357][COMP][RT] Compiler and runtime support for COPY TO
statement
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
This change provides the compiler and runtime support
for COPY TO statement to write to different destinations.
Change-Id: Icea1ff9f32fe49ee83d0739f5c5a305c3345faa7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18169
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Hussain Towaileb
Reviewed-by: Wail Alkowaileet
---
.../translator/LangExpressionToPlanTranslator.java | 27 --
.../writer/AbstractCloudExternalFileWriter.java| 10 ++--
.../asterix/cloud/writer/S3ExternalFileWriter.java | 4 +-
.../cloud/writer/S3ExternalFileWriterFactory.java | 35 ++--
.../external/writer/LocalFSExternalFileWriter.java | 6 +--
.../writer/LocalFSExternalFileWriterFactory.java | 31 ++-
.../writer/printer/TextualExternalFilePrinter.java | 4 +-
.../printer/TextualExternalFilePrinterFactory.java | 12 ++---
...ilePrinter.java => TextualExternalPrinter.java} | 17 ++
...ory.java => TextualExternalPrinterFactory.java} | 18 +++
.../lang/common/statement/CopyToStatement.java | 15 +++---
.../common/visitor/AbstractInlineUdfsVisitor.java | 4 ++
.../lang/common/visitor/FormatPrintVisitor.java| 6 +--
.../AbstractSqlppExpressionScopingVisitor.java | 3 ++
.../base/AbstractSqlppSimpleExpressionVisitor.java | 1 +
.../metadata/declared/MetadataProvider.java| 15 --
.../metadata/provider/ExternalWriterProvider.java | 24 -
.../runtime/writer/DynamicPathResolver.java| 2 +-
...ExternalWriter.java => ExternalFileWriter.java} | 4 +-
...Factory.java => ExternalFileWriterFactory.java} | 8 +--
...ation.java => ExternalWriterConfiguration.java} | 4 +-
.../runtime/writer/IExternalFileWriterFactory.java | 12 ++---
...ava => IExternalFileWriterFactoryProvider.java} | 4 +-
...ernalFilePrinter.java => IExternalPrinter.java} | 2 +-
...erFactory.java => IExternalPrinterFactory.java} | 6 +--
...y.java => IExternalWriterFactoryValidator.java} | 11 ++--
.../core/algebra/metadata/IMetadataProvider.java | 4 ++
.../algebra/operators/logical/WriteOperator.java | 30 ++-
.../logical/visitors/OperatorDeepCopyVisitor.java | 4 +-
.../visitors/SubstituteVariableVisitor.java| 1 +
.../logical/visitors/UsedVariableVisitor.java | 4 ++
.../operators/physical/SinkWritePOperator.java | 31 +--
.../rules/SetAlgebricksPhysicalOperatorsRule.java | 4 +-
.../operators/writer/IWriterPartitioner.java | 18 +++
.../operators/writer/KeyWriterPartitioner.java | 18 +--
.../operators/writer/NoOpWriterPartitioner.java| 23 ++--
.../writer/SinkExternalWriterRuntime.java | 32 ++-
.../writer/SinkExternalWriterRuntimeFactory.java | 52 ++
.../operators/writer/WriterPartitioner.java| 62 ++
39 files changed, 366 insertions(+), 202 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 7dd0217192..0949478da8 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -361,7 +361,7 @@ abstract class LangExpressionToPlanTranslator
return translate(expr, outputDatasetName, (ICompiledDmlStatement)
stmt, null, resultMetadata);
}
-private ILogicalPlan translateCopyTo(Query expr,
CompiledStatements.ICompiledStatement stmt,
+public ILogicalPlan translateCopyTo(Query expr,
CompiledStatements.ICompiledStatement stmt,
IResultMetadata resultMetadata) throws AlgebricksException {
CompiledStatements.CompiledCopyToStatement copyTo =
(CompiledStatements.CompiledCopyToStatement) stmt;
MutableObject base = new MutableObject<>(new
EmptyTupleSourceOperator());
@@ -423,8 +423,7 @@ abstract class LangExpressionToPlanTranslator
// astPathExpressions has at least one expression see CopyToStatement
constructor
List astPathExpres
(asterixdb) branch master updated: [ASTERIXDB-3353][EXT] Support reading avro from localfs.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new de7c56a756 [ASTERIXDB-3353][EXT] Support reading avro from localfs.
de7c56a756 is described below
commit de7c56a7563e0eb94b89321abefb19a55a849cd0
Author: ayush-couchbase
AuthorDate: Tue Feb 13 16:13:42 2024 +0530
[ASTERIXDB-3353][EXT] Support reading avro from localfs.
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Read avro files from local storage.
Change-Id: Ibdacf4e6b156a3b6ef15b4420f4102c122f8bf1e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18153
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Wail Alkowaileet
---
.../asterix/external/api/IRecordReaderFactory.java | 3 +-
.../reader/stream/AbstractStreamRecordReader.java | 59 +++
.../record/reader/stream/AvroRecordReader.java | 145 ++
.../record/reader/stream/StreamRecordReader.java | 38 +
.../reader/stream/StreamRecordReaderFactory.java | 30 ++--
.../stream/DiscretizedMultipleInputStream.java | 127 +++
.../asterix/external/parser/AvroDataParser.java| 170 +
java => AbstractGenericDataParserFactory.java} | 54 +--
.../parser/factory/AvroDataParserFactory.java | 68 +
.../parser/factory/JSONDataParserFactory.java | 49 +-
.../provider/StreamRecordReaderProvider.java | 18 ++-
.../context/ExternalReaderRuntimeDataContext.java | 6 +-
.../external/util/ExternalDataConstants.java | 3 +-
apache.asterix.external.api.IDataParserFactory | 3 +-
14 files changed, 621 insertions(+), 152 deletions(-)
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
index 64be02e390..4708f52743 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordReaderFactory.java
@@ -21,6 +21,7 @@ package org.apache.asterix.external.api;
import java.util.List;
import java.util.Set;
+import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -28,7 +29,7 @@ public interface IRecordReaderFactory extends
IExternalDataSourceFactory {
IRecordReader createRecordReader(IExternalDataRuntimeContext
context) throws HyracksDataException;
-Class getRecordClass();
+Class getRecordClass() throws AsterixException;
@Override
default DataSourceType getDataSourceType() {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
new file mode 100644
index 00..eb1d754f62
--- /dev/null
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/stream/AbstractStreamRecordReader.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.stream;
+
+import static
org.apache.asterix.external.util.ExternalDataConstants.EMPTY_STRING;
+import static
org.apache.asterix.external.util.ExternalDataConstants.KEY_REDACT_WARNINGS;
+
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import org.apache.asterix.external.api.AsterixInputStream;
+import org.apache.asterix.external.api.IRecordReader;
+import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.hyracks.api.context.IHyra
(asterixdb) branch master updated: [NO ISSUE] [MISC] Misc. Changes
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 0cc88e8c14 [NO ISSUE] [MISC] Misc. Changes
0cc88e8c14 is described below
commit 0cc88e8c14dca2fc2bddff378e0d04ab2c0ef7f7
Author: Ritik Raj
AuthorDate: Thu Feb 15 02:46:11 2024 +0530
[NO ISSUE] [MISC] Misc. Changes
Details:
1. Resolving Dataset Creation Issue in case of Standalone collection
Extensions
2. added record field to addMeta in DataParserFactory for Extensions
Change-Id: I30d4b7633cef47fcbb9d340407f0fc5483c647a5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18164
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
---
.../org/apache/asterix/app/translator/QueryTranslator.java | 7 ++-
.../asterix/external/api/IRecordWithMetadataParser.java | 2 +-
.../external/dataflow/FeedWithMetaDataFlowController.java| 2 +-
.../asterix/external/parser/RecordWithMetadataParser.java| 2 +-
.../asterix/external/parser/test/RecordWithMetaTest.java | 2 +-
.../asterix/metadata/declared/MetadataManagerUtil.java | 12 ++--
6 files changed, 16 insertions(+), 11 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 30c7127e34..f20a03194d 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -981,7 +981,7 @@ public class QueryTranslator extends AbstractLangTranslator
implements IStatemen
compactionPolicy =
StorageConstants.DEFAULT_FILTERED_DATASET_COMPACTION_POLICY_NAME;
compactionPolicyProperties =
StorageConstants.DEFAULT_COMPACTION_POLICY_PROPERTIES;
}
-boolean isDatasetWithoutTypeSpec =
aRecordType.getFieldNames().length == 0 && metaRecType == null;
+boolean isDatasetWithoutTypeSpec =
isDatasetWithoutTypeSpec(dd, aRecordType, metaRecType);
// Validate dataset properties if the format is COLUMN
ColumnPropertiesValidationUtil.validate(sourceLoc,
datasetFormatInfo.getFormat(), compactionPolicy,
filterField);
@@ -1108,6 +1108,11 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
return Optional.of(dataset);
}
+protected boolean isDatasetWithoutTypeSpec(DatasetDecl datasetDecl,
ARecordType aRecordType,
+ARecordType metaRecType) {
+return aRecordType.getFieldNames().length == 0 && metaRecType == null;
+}
+
protected IDataset createDataset(DatasetDecl dd, String database,
DataverseName dataverseName, String datasetName,
String itemTypeDatabase, DataverseName itemTypeDataverseName,
String itemTypeName,
String metaItemTypeDatabase, DataverseName
metaItemTypeDataverseName, String metaItemTypeName,
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordWithMetadataParser.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordWithMetadataParser.java
index 751541d1b1..e04be9a4e1 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordWithMetadataParser.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordWithMetadataParser.java
@@ -24,7 +24,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
public interface IRecordWithMetadataParser extends IRecordDataParser {
-public void parseMeta(DataOutput out) throws HyracksDataException;
+void parseMeta(DataOutput out, IRawRecord record) throws
HyracksDataException;
void appendLastParsedPrimaryKeyToTuple(ArrayTupleBuilder tb) throws
HyracksDataException;
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedWithMetaDataFlowController.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedWithMetaDataFlowController.java
index 74bfe2671b..b78c27067c 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedWithMetaDataFlowController.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedWithMetaDataFlowController.java
@@ -39,7 +39,7 @@ public class FeedWithMetaDataFlowController extends
FeedRecordDataFlowControl
@Override
(asterixdb) branch master updated: [NO ISSUE][HYR] Enable extension of network security manager
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 528c28a8a5 [NO ISSUE][HYR] Enable extension of network security manager
new 08a7e4dca2 Merge branch 'gerrit/trinity' into 'master'
528c28a8a5 is described below
commit 528c28a8a50aeb32b49401106a5c964f3e785c46
Author: Michael Blow
AuthorDate: Sun Feb 11 23:01:50 2024 -0500
[NO ISSUE][HYR] Enable extension of network security manager
- remove some premature resolution of configured hostnames to
ip address
Change-Id: Idad460b5894eeed5ef9b43d666d10cfd2e1e4cd6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18159
Reviewed-by: Murtadha Hubail
Tested-by: Michael Blow
---
.../org/apache/asterix/test/storage/DeallocatableTest.java | 6 +++---
.../asterix/replication/management/NetworkingUtil.java | 14 --
.../java/org/apache/hyracks/api/comm/NetworkAddress.java | 11 +--
.../org/apache/hyracks/client/result/ResultSetReader.java | 6 +-
.../hyracks/control/cc/ClusterControllerService.java | 12 ++--
.../apache/hyracks/control/nc/NodeControllerService.java | 12 ++--
.../control/nc/work/ReportPartitionAvailabilityWork.java | 11 ++-
.../org/apache/hyracks/control/nc/work/StartTasksWork.java | 11 +++
.../hyracks/ipc/security/NetworkSecurityManager.java | 2 +-
9 files changed, 27 insertions(+), 58 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DeallocatableTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DeallocatableTest.java
index cfd251b025..368b17bef2 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DeallocatableTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DeallocatableTest.java
@@ -22,9 +22,9 @@ import java.nio.ByteBuffer;
import java.util.concurrent.TimeUnit;
import org.apache.asterix.app.bootstrap.TestNodeController;
-import org.apache.asterix.replication.management.NetworkingUtil;
import org.apache.asterix.test.common.TestHelper;
import org.apache.asterix.test.runtime.LangExecutionUtil;
+import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.ConnectorDescriptorId;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
@@ -63,8 +63,8 @@ public class DeallocatableTest {
final IHyracksTaskContext ctx = nc.createTestContext(jobId, 0,
true);
final ConnectorDescriptorId codId = new ConnectorDescriptorId(1);
final PartitionId pid = new
PartitionId(ctx.getJobletContext().getJobId(), codId, 1, 1);
-final ChannelControlBlock ccb = ncs.getNetworkManager()
-
.connect(NetworkingUtil.getSocketAddress(ncs.getNetworkManager().getLocalNetworkAddress()));
+NetworkAddress netAddr =
ncs.getNetworkManager().getLocalNetworkAddress();
+final ChannelControlBlock ccb =
ncs.getNetworkManager().connect(netAddr.toResolvedInetSocketAddress());
final NetworkOutputChannel networkOutputChannel = new
NetworkOutputChannel(ccb, 0);
final MaterializingPipelinedPartition mpp =
new MaterializingPipelinedPartition(ctx,
ncs.getPartitionManager(), pid, taId, ncs.getExecutor());
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
index b38f0aa8d3..9caaa79ce9 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/NetworkingUtil.java
@@ -21,18 +21,14 @@ package org.apache.asterix.replication.management;
import java.io.EOFException;
import java.io.IOException;
import java.net.InetAddress;
-import java.net.InetSocketAddress;
import java.net.NetworkInterface;
-import java.net.SocketAddress;
import java.net.SocketException;
-import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.SocketChannel;
import java.util.Enumeration;
-import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.network.ISocketChannel;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -131,16 +127,6 @@ public class NetworkingUtil {
fileChannel.transferFrom(socketChannel, pos, fileSize);
}
-public static InetSocketAddress getSocketAddress(SocketChannel
socketChannel
(asterixdb) branch master updated: [NO ISSUE][STO] Add API to ensure minimum LSN
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new d22d545a33 [NO ISSUE][STO] Add API to ensure minimum LSN
d22d545a33 is described below
commit d22d545a336a07baba7578586f8cd606663d1ce5
Author: Murtadha Hubail
AuthorDate: Thu Nov 9 02:25:17 2023 +0300
[NO ISSUE][STO] Add API to ensure minimum LSN
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add API in the log manager that can be used to advance the
append LSN to a specific LSN.
- Add API to get the maximum LSN used in specific storage partitions.
Change-Id: Ic5f91f08a75f3737e2b33373cfd9500d78ba83d2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17934
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../org/apache/asterix/app/nc/RecoveryManager.java | 25 ++
.../asterix/common/transactions/ILogManager.java | 10 +++--
.../common/transactions/IRecoveryManager.java | 9
.../management/service/logging/LogManager.java | 14 ++--
.../service/recovery/CheckpointManager.java| 3 ++-
5 files changed, 56 insertions(+), 5 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index f6eb123df3..a7d30a3405 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -519,6 +519,31 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
return minRemoteLSN;
}
+@Override
+public long getPartitionsMaxLSN(Set partitions) throws
HyracksDataException {
+final IIndexCheckpointManagerProvider idxCheckpointMgrProvider =
appCtx.getIndexCheckpointManagerProvider();
+long maxLSN = 0;
+for (Integer partition : partitions) {
+final List partitionResources =
localResourceRepository.getResources(resource -> {
+DatasetLocalResource dsResource = (DatasetLocalResource)
resource.getResource();
+return dsResource.getPartition() == partition;
+},
Collections.singleton(partition)).values().stream().map(DatasetResourceReference::of)
+.collect(Collectors.toList());
+for (DatasetResourceReference indexRef : partitionResources) {
+try {
+final IIndexCheckpointManager idxCheckpointMgr =
idxCheckpointMgrProvider.get(indexRef);
+if (idxCheckpointMgr.isValidIndex()) {
+long indexMaxLsn =
idxCheckpointMgrProvider.get(indexRef).getLowWatermark();
+maxLSN = Math.max(maxLSN, indexMaxLsn);
+}
+} catch (Exception e) {
+LOGGER.warn("Failed to get max LSN of resource {}",
indexRef, e);
+}
+}
+}
+return maxLSN;
+}
+
@Override
public synchronized void replayReplicaPartitionLogs(Set
partitions, boolean flush)
throws HyracksDataException {
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ILogManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ILogManager.java
index d7e0885464..9d89c07577 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ILogManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ILogManager.java
@@ -99,7 +99,13 @@ public interface ILogManager {
public void closeLogFile(TxnLogFile logFileRef, FileChannel fileChannel)
throws IOException;
/**
- * Deletes all current log files and start the next log file partition
+ * Deletes all current log files and start the next log file partition
after {@code minLSN}
*/
-void renewLogFiles();
+void renewLogFiles(long minLSN);
+
+/**
+ * Ensures the next lsn of this log manager is greater than {@code lsn}
+ * @param lsn
+ */
+void ensureMinLSN(long lsn);
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
index 8a5f34eb07..aef22154f3 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecover
(asterixdb) branch master updated: [NO ISSUE][CLUS] no replica failure messages on disabled replication
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 51b0283d95 [NO ISSUE][CLUS] no replica failure messages on disabled
replication
51b0283d95 is described below
commit 51b0283d9575f670ca00aa359610f54b2a25e93c
Author: Murtadha Hubail
AuthorDate: Mon Oct 30 15:42:13 2023 +0300
[NO ISSUE][CLUS] no replica failure messages on disabled replication
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Do not send replica failure messages to NCs on node failures
when replication is disabled.
Change-Id: Ided98c584f9ba7ed81683005da86da5eab4da5d1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17887
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
Tested-by: Jenkins
---
.../org/apache/asterix/app/replication/NcLifecycleCoordinator.java | 3 +++
1 file changed, 3 insertions(+)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index 4135f356b3..dde2371d31 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
@@ -332,6 +332,9 @@ public class NcLifecycleCoordinator implements
INcLifecycleCoordinator {
private void notifyFailedReplica(IClusterStateManager clusterManager,
String nodeID,
InetSocketAddress replicaAddress) {
+if (!replicationEnabled) {
+return;
+}
LOGGER.info("notify replica failure of nodeId {} at {}", nodeID,
replicaAddress);
Set ncs = clusterManager.getParticipantNodes(true);
ReplicaFailedMessage message =
[asterixdb] branch master updated: [NO ISSUE][OTH] Added license override for zstd-jni-1.5.5-1
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new 457b8ad6c6 [NO ISSUE][OTH] Added license override for zstd-jni-1.5.5-1 457b8ad6c6 is described below commit 457b8ad6c69a3e0a1b666282f6bcb0cac7fce46c Author: Ritik Raj AuthorDate: Fri Oct 13 20:24:39 2023 +0530 [NO ISSUE][OTH] Added license override for zstd-jni-1.5.5-1 - user model changes: no - storage format changes: no - interface changes: no Change-Id: Idd83208d2d92240999319e929e60f07422c8715f Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17865 Integration-Tests: Jenkins Reviewed-by: Murtadha Hubail Tested-by: Ali Alsuliman --- asterixdb/src/main/appended-resources/supplemental-models.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml index 60530e99c9..eb94233acd 100644 --- a/asterixdb/src/main/appended-resources/supplemental-models.xml +++ b/asterixdb/src/main/appended-resources/supplemental-models.xml @@ -2321,9 +2321,9 @@ com.github.luben zstd-jni - 1.5.0-1 - 1.5.0-1 -1.5.0-1 + 1.5.0-1,1.5.5-1 + 1.5.0-1,1.5.5-1 + 1.5.0-1,1.5.5-1
[asterixdb] branch master updated: [NO ISSUE][STO] Download metadata files on lazy caching
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 56bca65d94 [NO ISSUE][STO] Download metadata files on lazy caching
56bca65d94 is described below
commit 56bca65d9485c303d57e2e9bbd974a85aefe359a
Author: Murtadha Hubail
AuthorDate: Tue Oct 10 03:09:54 2023 +0300
[NO ISSUE][STO] Download metadata files on lazy caching
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Download metadata files for the node storage partitions
on bootstrap on lazy caching.
- Use the local + uncached files list to perform list operations
rather than calling the cloud API.
- Delay partition clean up on promotion for cloud deployments.
Change-Id: I8ba767d5ffa0257a429af8c455bacea3df7ff7a7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17856
Reviewed-by: Wail Alkowaileet
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../org/apache/asterix/app/cc/GlobalTxManager.java | 3 ++-
.../org/apache/asterix/app/nc/ReplicaManager.java | 3 +--
.../app/replication/NcLifecycleCoordinator.java| 5 ++--
.../apache/asterix/cloud/LazyCloudIOManager.java | 28 +++
.../apache/asterix/cloud/lazy/IParallelCacher.java | 4 +++
.../asterix/cloud/lazy/NoOpParallelCacher.java | 8 ++
.../apache/asterix/cloud/lazy/ParallelCacher.java | 31 +++---
.../lazy/accessor/ReplaceableCloudAccessor.java| 21 ---
.../asterix/common/utils/StorageConstants.java | 1 +
.../asterix/common/utils/StoragePathUtil.java | 8 ++
.../PersistentLocalResourceRepository.java | 1 +
.../hyracks/storage/common/LocalResource.java | 2 +-
12 files changed, 91 insertions(+), 24 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/GlobalTxManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/GlobalTxManager.java
index b4223bc4cb..0786895581 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/GlobalTxManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/GlobalTxManager.java
@@ -32,6 +32,7 @@ import org.apache.asterix.common.cluster.IGlobalTxManager;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.messaging.api.ICCMessageBroker;
import org.apache.asterix.common.transactions.IGlobalTransactionContext;
+import org.apache.asterix.common.utils.StorageConstants;
import
org.apache.asterix.transaction.management.service.transaction.GlobalTransactionContext;
import
org.apache.asterix.transaction.management.service.transaction.GlobalTxInfo;
import org.apache.hyracks.api.application.ICCServiceContext;
@@ -182,7 +183,7 @@ public class GlobalTxManager implements IGlobalTxManager {
@Override
public void rollback() throws Exception {
-Set txnLogFileRefs =
ioManager.list(ioManager.resolve("."));
+Set txnLogFileRefs =
ioManager.list(ioManager.resolve(StorageConstants.GLOBAL_TXN_DIR_NAME));
for (FileReference txnLogFileRef : txnLogFileRefs) {
IGlobalTransactionContext context = new
GlobalTransactionContext(txnLogFileRef, ioManager);
txnContextRepository.put(context.getJobId(), context);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 0684442bee..101b1b0fe2 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -129,9 +129,8 @@ public class ReplicaManager implements IReplicaManager {
LOGGER.warn("promoting partition {}", partition);
final PersistentLocalResourceRepository localResourceRepository =
(PersistentLocalResourceRepository)
appCtx.getLocalResourceRepository();
-localResourceRepository.cleanup(partition);
-localResourceRepository.clearResourcesCache();
if (!appCtx.isCloudDeployment()) {
+localResourceRepository.cleanup(partition);
final IRecoveryManager recoveryManager =
appCtx.getTransactionSubsystem().getRecoveryManager();
recoveryManager.replayReplicaPartitionLogs(Stream.of(partition).collect(Collectors.toSet()),
true);
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index 5d0901d1a5..4135f356b3 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/ast
[asterixdb] branch master updated: [NO ISSUE][STO] Allow concurrent modification on persisted resources
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 8e51145806 [NO ISSUE][STO] Allow concurrent modification on persisted
resources
8e51145806 is described below
commit 8e51145806fb54ea8536487b484dd05049462f8d
Author: Murtadha Hubail
AuthorDate: Fri Sep 29 22:09:59 2023 +0300
[NO ISSUE][STO] Allow concurrent modification on persisted resources
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- To allow index's resources to be created on different partitions
concurrently, replace the synchronization by read/write lock in
PersistentLocalResourceRepository.
- Any operation that might modify the persisted files will acquire
a read lock.
- Any operation that attempts to read the persisted files will acquire
a write lock to wait for any on-going modifications.
Change-Id: Id435bfc113a0b8e3e2a1f75712f0ded74ae0ee6f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17824
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Ali Alsuliman
---
.../PersistentLocalResourceRepository.java | 502 +
.../storage/am/common/build/IndexBuilder.java | 82 ++--
2 files changed, 355 insertions(+), 229 deletions(-)
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 1eef182c06..33cbf2dc04 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -39,6 +39,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -83,7 +84,6 @@ public class PersistentLocalResourceRepository implements
ILocalResourceReposito
private static final Logger LOGGER = LogManager.getLogger();
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
private static final String METADATA_FILE_MASK_NAME =
StorageConstants.MASK_FILE_PREFIX +
StorageConstants.METADATA_FILE_NAME;
private static final FilenameFilter LSM_INDEX_FILES_FILTER =
@@ -94,7 +94,6 @@ public class PersistentLocalResourceRepository implements
ILocalResourceReposito
(dir, name) -> name.equals(StorageConstants.METADATA_FILE_NAME);
private static final FilenameFilter METADATA_MASK_FILES_FILTER =
(dir, name) -> name.equals(METADATA_FILE_MASK_NAME);
-
private static final int MAX_CACHED_RESOURCES = 1000;
// Finals
@@ -102,11 +101,11 @@ public class PersistentLocalResourceRepository implements
ILocalResourceReposito
private final Cache resourceCache;
// Mutables
private boolean isReplicationEnabled = false;
-private Set filesToBeReplicated;
private IReplicationManager replicationManager;
private final List storageRoots;
private final IIndexCheckpointManagerProvider
indexCheckpointManagerProvider;
private final IPersistedResourceRegistry persistedResourceRegistry;
+private final ReentrantReadWriteLock resourcesAccessLock = new
ReentrantReadWriteLock(true);
public PersistentLocalResourceRepository(IIOManager ioManager,
IIndexCheckpointManagerProvider indexCheckpointManagerProvider,
@@ -135,23 +134,29 @@ public class PersistentLocalResourceRepository implements
ILocalResourceReposito
}
@Override
-public synchronized LocalResource get(String relativePath) throws
HyracksDataException {
-LocalResource resource = resourceCache.getIfPresent(relativePath);
-if (resource == null) {
-FileReference resourceFile = getLocalResourceFileByName(ioManager,
relativePath);
-resource = readLocalResource(resourceFile);
-if (resource != null) {
-resourceCache.put(relativePath, resource);
+public LocalResource get(String relativePath) throws HyracksDataException {
+beforeReadAccess();
+try {
+LocalResource resource = resourceCache.getIfPresent(relativePath);
+if (resource == null) {
+FileReference resourceFile =
getLocalResourceFileByName(ioManager, relativePath);
+resource = readLocalResource(resourceFile);
+if (resource !
[asterixdb] branch master updated: [NO ISSUE][OTH] Configure cached resource ids based on storage partitions
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 27f5467f8a [NO ISSUE][OTH] Configure cached resource ids based on
storage partitions
27f5467f8a is described below
commit 27f5467f8a15a51ec06f0bfe79dd917f9e336436
Author: Murtadha Hubail
AuthorDate: Thu Sep 28 18:47:21 2023 +0300
[NO ISSUE][OTH] Configure cached resource ids based on storage partitions
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- To ensure each node has enough cached resource ids to create at least
one resource per storage partition, configure the cached resource ids
block size based on the storage partition.
Change-Id: I531fd366168209a132e552306a0f2b667e6e0342
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17815
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Ali Alsuliman
---
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 8 +++-
.../runtime/transaction/GlobalResourceIdFactory.java | 20 +++-
.../transaction/GlobalResourceIdFactoryProvider.java | 6 --
3 files changed, 22 insertions(+), 12 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 630265ca18..9e796f2651 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -186,7 +186,8 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
cloudProperties = propertiesFactory.newCloudProperties();
ncExtensionManager = extensionManager;
componentProvider = new StorageComponentProvider();
-resourceIdFactory = new
GlobalResourceIdFactoryProvider(ncServiceContext).createResourceIdFactory();
+resourceIdFactory = new
GlobalResourceIdFactoryProvider(ncServiceContext, getResourceIdBlockSize())
+.createResourceIdFactory();
persistedResourceRegistry =
ncServiceContext.getPersistedResourceRegistry();
cacheManager = new CacheManager();
}
@@ -677,4 +678,9 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
public IPartitionBootstrapper getPartitionBootstrapper() {
return partitionBootstrapper;
}
+
+private int getResourceIdBlockSize() {
+return isCloudDeployment() ?
storageProperties.getStoragePartitionsCount()
+: ncServiceContext.getIoManager().getIODevices().size();
+}
}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactory.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactory.java
index 908663fe63..30877d95b2 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactory.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/transaction/GlobalResourceIdFactory.java
@@ -41,21 +41,23 @@ import it.unimi.dsi.fastutil.longs.LongPriorityQueues;
public class GlobalResourceIdFactory implements IResourceIdFactory {
private static final Logger LOGGER = LogManager.getLogger();
-private static final int RESOURCE_ID_INITIAL_BLOCK_SIZE = 24;
-private static final int MAX_BLOCK_SIZE = 35;
private final INCServiceContext serviceCtx;
-private final LongPriorityQueue resourceIds =
-LongPriorityQueues.synchronize(new
LongArrayFIFOQueue(RESOURCE_ID_INITIAL_BLOCK_SIZE));
+private final LongPriorityQueue resourceIds;
private final LinkedBlockingQueue
resourceIdResponseQ;
private final String nodeId;
-private volatile boolean reset = false;
+private final int initialBlockSize;
+private final int maxBlockSize;
private int currentBlockSize;
+private volatile boolean reset = false;
-public GlobalResourceIdFactory(INCServiceContext serviceCtx) {
+public GlobalResourceIdFactory(INCServiceContext serviceCtx, int
initialBlockSize) {
this.serviceCtx = serviceCtx;
this.resourceIdResponseQ = new LinkedBlockingQueue<>();
this.nodeId = serviceCtx.getNodeId();
-this.currentBlockSize = RESOURCE_ID_INITIAL_BLOCK_SIZE;
+this.initialBlockSize = initialBlockSize;
+maxBlockSize = initialBlockSize * 2;
+currentBlockSize = initialBlockSize;
+resourceIds = LongPriorityQueues.synchronize(new
LongArrayFIFOQueue(initialBlockSize));
}
public synchronized void addNewIds(ResourceIdRequestResponseMessage
resourceIdResponse)
@@ -120,8
[asterixdb] branch master updated: [NO ISSUE][OTH] Update AWS SDK to 2.20.135
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2f3947437d [NO ISSUE][OTH] Update AWS SDK to 2.20.135
2f3947437d is described below
commit 2f3947437decdea49198c2b08e3ba3ebed1c4590
Author: Murtadha Hubail
AuthorDate: Tue Aug 29 00:30:37 2023 +0300
[NO ISSUE][OTH] Update AWS SDK to 2.20.135
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: Ic9300f891bd167b6933086f0e3cd7f24537c2ac8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17750
Reviewed-by: Hussain Towaileb
Tested-by: Murtadha Hubail
---
asterixdb/pom.xml | 7 ++-
asterixdb/src/main/appended-resources/supplemental-models.xml | 4 ++--
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 95ff786418..161374c395 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -90,7 +90,7 @@
3.3.4
0.7.6.201602180812
2.19.0
-2.20.37
+2.20.135
1.12.3
1.12.402
12.22.0
@@ -1636,6 +1636,11 @@
s3-transfer-manager
${awsjavasdk.version}
+
+software.amazon.awssdk
+sts
+${awsjavasdk.version}
+
io.findify
diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml
b/asterixdb/src/main/appended-resources/supplemental-models.xml
index f04113a64b..d7cdf06a86 100644
--- a/asterixdb/src/main/appended-resources/supplemental-models.xml
+++ b/asterixdb/src/main/appended-resources/supplemental-models.xml
@@ -580,8 +580,8 @@
software.amazon.awssdk
third-party-jackson-core
-
2.17.218:META-INF/NOTICE.txt,2.20.37:META-INF/NOTICE.txt
-
2.17.218:META-INF/LICENSE.txt,2.20.37:META-INF/LICENSE.txt
+
2.17.218:META-INF/NOTICE.txt,2.20.37:META-INF/NOTICE.txt,2.20.135:META-INF/NOTICE.txt
+
2.17.218:META-INF/LICENSE.txt,2.20.37:META-INF/LICENSE.txt,2.20.135:META-INF/LICENSE.txt
[asterixdb] branch master updated: [NO ISSUE][OTH] Logging clean up
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 62aa90aab4 [NO ISSUE][OTH] Logging clean up
62aa90aab4 is described below
commit 62aa90aab4150c57a649f346dbfcd7fcede7bf8f
Author: Murtadha Hubail
AuthorDate: Mon Aug 7 21:27:26 2023 +0300
[NO ISSUE][OTH] Logging clean up
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: If5ed90c07d94083aad653847436afe1964dc9b42
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17702
Integration-Tests: Jenkins
Reviewed-by: Murtadha Al Hubail
Reviewed-by: Peeyush Gupta
Tested-by: Murtadha Al Hubail
---
.../apache/asterix/app/message/AtomicJobCommitMessage.java | 8 +---
.../asterix/app/message/AtomicJobCompletionMessage.java | 8 +---
.../app/message/AtomicJobRollbackCompleteMessage.java | 7 ---
.../asterix/app/message/AtomicJobRollbackMessage.java | 8 +---
.../org/apache/asterix/app/message/EnableMergeMessage.java | 8 +---
.../common/context/PrimaryIndexOperationTracker.java| 13 +++--
.../asterix/common/messaging/AtomicJobPreparedMessage.java | 7 ---
7 files changed, 35 insertions(+), 24 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobCommitMessage.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobCommitMessage.java
index 55ae22557f..fac023cffa 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobCommitMessage.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobCommitMessage.java
@@ -28,15 +28,12 @@ import
org.apache.asterix.common.messaging.api.INcAddressedMessage;
import org.apache.asterix.messaging.NCMessageBroker;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobId;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
/**
* Message sent from CC to all NCs asking to commit an atomic statement/job.
*/
public class AtomicJobCommitMessage implements INcAddressedMessage {
-private static final Logger LOGGER = LogManager.getLogger();
private static final long serialVersionUID = 1L;
private final JobId jobId;
private final List datasetIds;
@@ -65,4 +62,9 @@ public class AtomicJobCommitMessage implements
INcAddressedMessage {
throw HyracksDataException.create(e);
}
}
+
+@Override
+public String toString() {
+return "AtomicJobCommitMessage{" + "jobId=" + jobId + ", datasetIds="
+ datasetIds + '}';
+}
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobCompletionMessage.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobCompletionMessage.java
index 869e5d2522..a2a1e51c51 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobCompletionMessage.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobCompletionMessage.java
@@ -22,15 +22,12 @@ import
org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobId;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
/**
* Message sent from NC to CC on successful local commit of an atomic
statement/job.
*/
public class AtomicJobCompletionMessage implements ICcAddressedMessage {
-private static final Logger LOGGER = LogManager.getLogger();
private static final long serialVersionUID = 1L;
private final String nodeId;
private final JobId jobId;
@@ -44,4 +41,9 @@ public class AtomicJobCompletionMessage implements
ICcAddressedMessage {
public void handle(ICcApplicationContext appCtx) throws
HyracksDataException, InterruptedException {
appCtx.getGlobalTxManager().handleJobCompletionMessage(jobId, nodeId);
}
+
+@Override
+public String toString() {
+return "AtomicJobCompletionMessage{" + "jobId=" + jobId + ", nodeId='"
+ nodeId + '\'' + '}';
+}
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobRollbackCompleteMessage.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobRollbackCompleteMessage.java
index 90e1fbdf2f..85f9cf43cd 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AtomicJobRollbackCompleteMessage.java
+++
b/asterixdb/asterix-app/src/main/java/org/apac
[asterixdb] branch master updated: [ASTERIXDB-3227][COMP] Refactor QueryTranslator doCreateDatasetStatement()
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new b1b7d4f0cd [ASTERIXDB-3227][COMP] Refactor QueryTranslator
doCreateDatasetStatement()
b1b7d4f0cd is described below
commit b1b7d4f0cd36a6bd5c602064cf4b1b65d3652d3c
Author: Ritik Raj
AuthorDate: Tue Jul 18 21:37:15 2023 +0530
[ASTERIXDB-3227][COMP] Refactor QueryTranslator doCreateDatasetStatement()
- user model changes: no
- storage format changes: no
- interface changes: no
This small patch extracts dataset object creation in function
getDataset(..) out of
doCreateDatasetStatement(..)
Change-Id: I7da017d4bf1c56381033e5c07cb6f094bbd22aca
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17648
Integration-Tests: Jenkins
Reviewed-by: Hussain Towaileb
Tested-by: Hussain Towaileb
---
.../asterix/app/translator/QueryTranslator.java | 20
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index dc165a07fa..f390237072 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -90,6 +90,7 @@ import
org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.metadata.DatasetFullyQualifiedName;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.IDataset;
import org.apache.asterix.common.metadata.IMetadataLockUtil;
import org.apache.asterix.common.utils.JobUtils;
import org.apache.asterix.common.utils.JobUtils.ProgressState;
@@ -886,10 +887,9 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
// #. add a new dataset with PendingAddOp
-dataset = new Dataset(dataverseName, datasetName,
itemTypeDataverseName, itemTypeName,
-metaItemTypeDataverseName, metaItemTypeName, ngName,
compactionPolicy, compactionPolicyProperties,
-datasetDetails, dd.getHints(), dsType,
DatasetIdFactory.generateDatasetId(),
-MetadataUtil.PENDING_ADD_OP, compressionScheme,
datasetFormatInfo);
+dataset = (Dataset) createDataset(dd, dataverseName, datasetName,
itemTypeDataverseName, itemTypeName,
+metaItemTypeDataverseName, metaItemTypeName, dsType,
compactionPolicy, compactionPolicyProperties,
+compressionScheme, datasetFormatInfo, datasetDetails,
ngName);
MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(),
dataset);
if (itemTypeIsInline) {
@@ -981,6 +981,18 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
return Optional.of(dataset);
}
+protected IDataset createDataset(DatasetDecl dd, DataverseName
dataverseName, String datasetName,
+DataverseName itemTypeDataverseName, String itemTypeName,
DataverseName metaItemTypeDataverseName,
+String metaItemTypeName, DatasetType dsType, String
compactionPolicy,
+Map compactionPolicyProperties, String
compressionScheme,
+DatasetFormatInfo datasetFormatInfo, IDatasetDetails
datasetDetails, String ngName)
+throws AlgebricksException {
+return new Dataset(dataverseName, datasetName, itemTypeDataverseName,
itemTypeName, metaItemTypeDataverseName,
+metaItemTypeName, ngName, compactionPolicy,
compactionPolicyProperties, datasetDetails, dd.getHints(),
+dsType, DatasetIdFactory.generateDatasetId(),
MetadataUtil.PENDING_ADD_OP, compressionScheme,
+datasetFormatInfo);
+}
+
protected Triple
extractDatasetItemTypeName(DataverseName datasetDataverseName,
String datasetName, TypeExpression itemTypeExpr, boolean
isMetaItemType, SourceLocation sourceLoc)
throws CompilationException {
[asterixdb] branch master updated: [ASTERIXDB-3196][*DB] Skip upload on empty writes
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 1ac5c5b898 [ASTERIXDB-3196][*DB] Skip upload on empty writes
1ac5c5b898 is described below
commit 1ac5c5b898a97fdd648a06c339324d15f2b949cd
Author: Murtadha Hubail
AuthorDate: Fri Jun 23 18:12:39 2023 -0700
[ASTERIXDB-3196][*DB] Skip upload on empty writes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When no data was uploaded, skip completing the multipart
upload.
- Workaround S3Mock encoding issues by encoding/decoding keys
on list objects.
Change-Id: I31f9955e8a9c4897eba4f819746b2dd47fa7c50f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17614
Reviewed-by: Ali Alsuliman
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java| 6 ++
.../org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java | 9 +
.../org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java | 5 -
.../java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java| 2 +-
4 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
index 9104d9afc5..6c02bbc260 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3BufferedWriter.java
@@ -69,6 +69,9 @@ public class S3BufferedWriter implements ICloudBufferedWriter
{
@Override
public void finish() throws HyracksDataException {
+if (uploadId == null) {
+return;
+}
CompletedMultipartUpload completedMultipartUpload =
CompletedMultipartUpload.builder().parts(partQueue).build();
CompleteMultipartUploadRequest completeMultipartUploadRequest =
CompleteMultipartUploadRequest.builder()
.bucket(bucket).key(path).uploadId(uploadId).multipartUpload(completedMultipartUpload).build();
@@ -97,6 +100,9 @@ public class S3BufferedWriter implements
ICloudBufferedWriter {
@Override
public void abort() throws HyracksDataException {
+if (uploadId == null) {
+return;
+}
s3Client.abortMultipartUpload(
AbortMultipartUploadRequest.builder().bucket(bucket).key(path).uploadId(uploadId).build());
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index 281fe7707b..bbe8586540 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -55,7 +55,16 @@ public class S3ClientConfig {
return prefix;
}
+public boolean isEncodeKeys() {
+// to workaround https://github.com/findify/s3mock/issues/187 in our
S3Mock, we encode/decode keys
+return isS3Mock();
+}
+
public AwsCredentialsProvider createCredentialsProvider() {
return anonymousAuth ? AnonymousCredentialsProvider.create() :
DefaultCredentialsProvider.create();
}
+
+private boolean isS3Mock() {
+return endpoint != null && !endpoint.isEmpty();
+}
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 6d65f69039..0fb9c082cc 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.cloud.clients.aws.s3;
+import static org.apache.asterix.cloud.clients.aws.s3.S3Utils.encodeURI;
import static org.apache.asterix.cloud.clients.aws.s3.S3Utils.listS3Objects;
import java.io.File;
@@ -108,6 +109,7 @@ public class S3CloudClient implements ICloudClient {
@Override
public Set listObjects(String bucket, String path, FilenameFilter
filter) {
+path = config.isEncodeKeys() ? encodeURI(path) : path;
return filterAndGet(listS3Objects(s3Client, bucket, path), filter);
}
@@ -171,6 +173,7 @@ public class S3CloudClient implements ICloudClient {
@Override
public void copy(String bucket, String
[asterixdb] branch master updated: [ASTERIXDB-3196][*DB] Skip logs REDO on cloud deployments
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new af14efc2de [ASTERIXDB-3196][*DB] Skip logs REDO on cloud deployments
af14efc2de is described below
commit af14efc2dedf2cfd386ab88de0ef9c35d3e1308c
Author: Murtadha Hubail
AuthorDate: Thu Jun 22 20:36:17 2023 -0700
[ASTERIXDB-3196][*DB] Skip logs REDO on cloud deployments
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Since no updates logs are generated on cloud deployments,
we should skip logs REDO during partition promotion.
- Ensure cloud prefixes do not start with file separator.
Change-Id: If05ba888ea8327508d4cde00380a88d1f3494780
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17613
Integration-Tests: Jenkins
Reviewed-by: Ali Alsuliman
Tested-by: Jenkins
---
.../src/main/java/org/apache/asterix/app/nc/ReplicaManager.java| 6 --
.../src/main/java/org/apache/asterix/cloud/CloudIOManager.java | 4 ++--
.../main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java | 7 ++-
3 files changed, 12 insertions(+), 5 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 1372016d09..0684442bee 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -131,8 +131,10 @@ public class ReplicaManager implements IReplicaManager {
(PersistentLocalResourceRepository)
appCtx.getLocalResourceRepository();
localResourceRepository.cleanup(partition);
localResourceRepository.clearResourcesCache();
-final IRecoveryManager recoveryManager =
appCtx.getTransactionSubsystem().getRecoveryManager();
-
recoveryManager.replayReplicaPartitionLogs(Stream.of(partition).collect(Collectors.toSet()),
true);
+if (!appCtx.isCloudDeployment()) {
+final IRecoveryManager recoveryManager =
appCtx.getTransactionSubsystem().getRecoveryManager();
+
recoveryManager.replayReplicaPartitionLogs(Stream.of(partition).collect(Collectors.toSet()),
true);
+}
partitions.put(partition, new Object());
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
index b508b6c930..4939ab53fd 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/CloudIOManager.java
@@ -37,6 +37,7 @@ import java.util.Set;
import org.apache.asterix.cloud.clients.CloudClientProvider;
import org.apache.asterix.cloud.clients.ICloudClient;
+import org.apache.asterix.cloud.clients.aws.s3.S3Utils;
import org.apache.asterix.common.config.CloudProperties;
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -165,8 +166,7 @@ public class CloudIOManager extends IOManager {
if (file.getFile().isDirectory()) {
continue;
}
-
-String path = file.getRelativePath();
+String path = S3Utils.toCloudPrefix(file.getRelativePath());
if (!cloudFiles.contains(path)) {
// Delete local files that do not exist in cloud storage (the
ground truth for valid files)
localFilesIter.remove();
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
index e3e9e98eb2..4ea11b2ec2 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3Utils.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.cloud.clients.aws.s3;
+import java.io.File;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
@@ -39,7 +40,7 @@ public class S3Utils {
String newMarker = null;
ListObjectsV2Response listObjectsResponse;
ListObjectsV2Request.Builder listObjectsBuilder =
ListObjectsV2Request.builder().bucket(bucket);
-listObjectsBuilder.prefix(encodeURI(path));
+listObjectsBuilder.prefix(encodeURI(toCloudPrefix(path)));
while (true) {
// List the objects from the start, or from the last marker in
case of truncated result
if (newMarker == null) {
@@ -72,4 +73,8
[asterixdb] branch master updated: [ASTERIXDB-3196][*DB] Support S3 objects with special chars
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7e68132cc2 [ASTERIXDB-3196][*DB] Support S3 objects with special chars
7e68132cc2 is described below
commit 7e68132cc2539cc5345bb028d30bef7d19918a74
Author: Murtadha Hubail
AuthorDate: Mon Jun 19 13:22:49 2023 -0700
[ASTERIXDB-3196][*DB] Support S3 objects with special chars
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Encode/decode S3 URIs to support objects with special chars.
- Add test case.
Change-Id: I75879b117c8b5bb761e138e6657ae1261a08f8de
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17605
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Wail Alkowaileet
---
.../test/cloud_storage/CloudStorageTest.java | 6 +++---
.../cloud_storage/special-chars/test.000.ddl.sqlpp | 24 ++
.../special-chars/test.001.update.sqlpp| 22
.../special-chars/test.002.query.sqlpp | 23 +
.../cloud_storage/special-chars/test.999.ddl.sqlpp | 20 ++
.../cloud_storage/special-chars/result.002.adm | 1 +
.../runtimets/testsuite_cloud_storage.xml | 5 +
.../cloud/clients/aws/s3/S3CloudClient.java| 2 +-
.../asterix/cloud/clients/aws/s3/S3Utils.java | 23 ++---
9 files changed, 119 insertions(+), 7 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
index 505e239c85..f3fd8054d9 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageTest.java
@@ -26,7 +26,7 @@ import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.test.runtime.LangExecutionUtil;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.junit.AfterClass;
-import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -51,8 +51,8 @@ public class CloudStorageTest {
this.tcCtx = tcCtx;
}
-@Before
-public void setUp() throws Exception {
+@BeforeClass
+public static void setUp() throws Exception {
CloudUtils.startS3CloudEnvironment();
LangExecutionUtil.setUp(CONFIG_FILE_NAME, testExecutor);
System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY,
CONFIG_FILE_NAME);
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/special-chars/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/special-chars/test.000.ddl.sqlpp
new file mode 100644
index 00..18a98e38c2
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/special-chars/test.000.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE `part1`.`p%r t2` IF EXISTS;
+CREATE DATAVERSE `part1`.`p%r t2`;
+
+USE `part1`.`p%r t2`;
+CREATE COLLECTION `some@dataset` PRIMARY KEY (id: int);
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/special-chars/test.001.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/special-chars/test.001.update.sqlpp
new file mode 100644
index 00..3629d7f000
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/special-chars/test.001.update.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE
[asterixdb] branch master updated: [ASTERIXDB-3196][TX] Support atomic Txn with no WAL
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 63c7093fdb [ASTERIXDB-3196][TX] Support atomic Txn with no WAL
63c7093fdb is described below
commit 63c7093fdbf23083e4b940180196bee37e5d818f
Author: Murtadha Hubail
AuthorDate: Wed May 31 04:17:28 2023 +0300
[ASTERIXDB-3196][TX] Support atomic Txn with no WAL
- user model changes: yes
- storage format changes: no
- interface changes: yes
Details:
- Prevent concurrent metadata catalog modification.
- Introduce atomic txn without WAL that will persist
all records on commit or delete on abort.
- Compute-storage separation fixes.
Change-Id: Icfd034a4dc0b6464564e2129a166e4ceb0dc7b41
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17583
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../asterix/app/nc/IndexCheckpointManager.java | 7 +-
.../api/common/CloudStorageIntegrationUtil.java| 2 +-
.../TestPrimaryIndexOperationTrackerFactory.java | 11 ---
.../org/apache/asterix/cloud/CloudIOManager.java | 8 +-
.../context/PrimaryIndexOperationTracker.java | 27 +++
.../common/transactions/ITransactionContext.java | 14
.../common/transactions/ITransactionManager.java | 4 +
.../org/apache/asterix/metadata/MetadataNode.java | 26 +--
.../messaging/CheckpointPartitionIndexesTask.java | 2 +-
.../opcallbacks/NoOpModificationOpCallback.java| 46 +++
.../PersistentLocalResourceRepository.java | 27 +++
.../transaction/AbstractTransactionContext.java| 21 ++
.../transaction/AtomicNoWALTransactionContext.java | 88 ++
.../transaction/AtomicTransactionContext.java | 10 +++
.../transaction/EntityLevelTransactionContext.java | 5 ++
.../transaction/TransactionContextFactory.java | 2 +
.../service/transaction/TransactionManager.java| 22 +++---
.../java/org/apache/hyracks/api/io/IIOManager.java | 11 ++-
.../java/org/apache/hyracks/api/util/IoUtil.java | 17 +
.../apache/hyracks/control/nc/io/IOManager.java| 37 +
.../storage/am/common/build/IndexBuilder.java | 3 +-
.../buffercache/AbstractBufferedFileIOManager.java | 8 +-
.../storage/common/buffercache/BufferCache.java| 7 +-
.../compression/file/CompressedFileManager.java| 7 +-
.../common/file/CompressedBufferedFileHandle.java | 2 +-
.../am/common/AbstractIndexLifecycleTest.java | 2 -
26 files changed, 312 insertions(+), 104 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
index 2ed163812e..372cf69c7c 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
@@ -154,7 +154,7 @@ public class IndexCheckpointManager implements
IIndexCheckpointManager {
}
if (checkpoints.isEmpty()) {
LOGGER.warn("Couldn't find any checkpoint file for index {}.
Content of dir are {}.", indexPath,
-ioManager.getMatchingFiles(indexPath,
IoUtil.NO_OP_FILTER).toString());
+ioManager.list(indexPath, IoUtil.NO_OP_FILTER).toString());
throw new IllegalStateException("Couldn't find any checkpoints for
resource: " + indexPath);
}
checkpoints.sort(Comparator.comparingLong(IndexCheckpoint::getId).reversed());
@@ -182,7 +182,7 @@ public class IndexCheckpointManager implements
IIndexCheckpointManager {
private List getCheckpoints() throws
ClosedByInterruptException, HyracksDataException {
List checkpoints = new ArrayList<>();
-final Collection checkpointFiles =
ioManager.getMatchingFiles(indexPath, CHECKPOINT_FILE_FILTER);
+final Collection checkpointFiles =
ioManager.list(indexPath, CHECKPOINT_FILE_FILTER);
if (!checkpointFiles.isEmpty()) {
for (FileReference checkpointFile : checkpointFiles) {
try {
@@ -229,8 +229,7 @@ public class IndexCheckpointManager implements
IIndexCheckpointManager {
private void deleteHistory(long latestId, int historyToKeep) {
try {
-final Collection checkpointFiles =
-ioManager.getMatchingFiles(indexPath,
CHECKPOINT_FILE_FILTER);
+final Collection checkpointFiles =
ioManager.list(indexPath, CHECKPOINT_FILE_FILTER);
if (!checkpointFiles.isEmpty()) {
for (FileReference checkp
[asterixdb] branch master updated: [ASTERIXDB-3196][*DB] Cluster state for compute-storage separation
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new ae3c273a54 [ASTERIXDB-3196][*DB] Cluster state for compute-storage
separation
ae3c273a54 is described below
commit ae3c273a549a92501f47ccf8713cb2aba1beee58
Author: Murtadha Hubail
AuthorDate: Wed May 24 15:17:30 2023 +0300
[ASTERIXDB-3196][*DB] Cluster state for compute-storage separation
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Implement changes required to drive cluster state based on
compute-storage partitions map.
- Persist index checkpoints to cloud storage.
- Remove eager caching from NC startup tasks.
- Fixes for static data partitioning.
Change-Id: I217da04d06884d841c4a56aee3ab9815cc659de7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17553
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 3 +-
.../app/replication/NcLifecycleCoordinator.java| 7 ---
.../common/cluster/IClusterStateManager.java | 6 +++
.../cluster/StorageComputePartitionsMap.java | 11 +++-
.../metadata/declared/MetadataProvider.java| 9 +---
.../asterix/runtime/utils/ClusterStateManager.java | 62 +-
.../PersistentLocalResourceRepository.java | 4 +-
7 files changed, 67 insertions(+), 35 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index f60ed63872..7f0b529480 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -204,8 +204,7 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
storageProperties.getBufferCachePageSize(),
storageProperties.getBufferCacheNumPages());
lsmIOScheduler = createIoScheduler(storageProperties);
metadataMergePolicyFactory = new ConcurrentMergePolicyFactory();
-// TODO do we want to write checkpoints for cloud?
-indexCheckpointManagerProvider = new
IndexCheckpointManagerProvider(ioManager);
+indexCheckpointManagerProvider = new
IndexCheckpointManagerProvider(persistenceIOManager);
ILocalResourceRepositoryFactory
persistentLocalResourceRepositoryFactory =
new
PersistentLocalResourceRepositoryFactory(persistenceIOManager,
indexCheckpointManagerProvider,
persistedResourceRegistry);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index 4f9613d2a7..449dd27845 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
@@ -35,7 +35,6 @@ import java.util.Set;
import org.apache.asterix.app.nc.task.BindMetadataNodeTask;
import org.apache.asterix.app.nc.task.CheckpointTask;
-import org.apache.asterix.app.nc.task.CloudToLocalStorageCachingTask;
import org.apache.asterix.app.nc.task.ExportMetadataNodeTask;
import org.apache.asterix.app.nc.task.LocalRecoveryTask;
import org.apache.asterix.app.nc.task.LocalStorageCleanupTask;
@@ -52,7 +51,6 @@ import
org.apache.asterix.app.replication.message.RegistrationTasksResponseMessa
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
import org.apache.asterix.common.api.INCLifecycleTask;
import org.apache.asterix.common.cluster.IClusterStateManager;
-import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.common.messaging.api.ICCMessageBroker;
@@ -222,11 +220,6 @@ public class NcLifecycleCoordinator implements
INcLifecycleCoordinator {
tasks.add(new UpdateNodeStatusTask(NodeStatus.BOOTING,
nodeActivePartitions));
int metadataPartitionId =
clusterManager.getMetadataPartition().getPartitionId();
tasks.add(new LocalStorageCleanupTask(metadataPartitionId));
-
-if (((ICcApplicationContext)
(serviceContext.getControllerService()).getApplicationContext())
-.isCloudDeployment()) {
-tasks.add(new CloudToLocalStorageCachingTask(activePartitions));
-}
if (state == SystemState.CORRUPTED
[asterixdb] branch master updated: [NO ISSUE][LIC]: Fix failing license test
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new 775a90373b [NO ISSUE][LIC]: Fix failing license test 775a90373b is described below commit 775a90373b7d15d1e42dd8e8dcaa30ffc002f853 Author: Hussain Towaileb AuthorDate: Mon May 15 22:12:11 2023 +0300 [NO ISSUE][LIC]: Fix failing license test Change-Id: I3982d84ad13857cac82a8fa912c800883eb95a33 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17533 Reviewed-by: Michael Blow Integration-Tests: Jenkins Tested-by: Jenkins --- asterixdb/src/main/appended-resources/supplemental-models.xml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/asterixdb/src/main/appended-resources/supplemental-models.xml b/asterixdb/src/main/appended-resources/supplemental-models.xml index 3727fee2c6..baec366d70 100644 --- a/asterixdb/src/main/appended-resources/supplemental-models.xml +++ b/asterixdb/src/main/appended-resources/supplemental-models.xml @@ -555,8 +555,8 @@ software.amazon.awssdk third-party-jackson-core - 2.17.218:META-INF/NOTICE.txt - 2.17.218:META-INF/LICENSE.txt + 2.17.218:META-INF/NOTICE.txt,2.20.37:META-INF/NOTICE.txt + 2.17.218:META-INF/LICENSE.txt,2.20.37:META-INF/LICENSE.txt @@ -586,7 +586,6 @@ 0.21.10 0.21.10 -0.21.10
[asterixdb] branch master updated: [NO ISSUE][OTH] Move page size storage config to common config
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new 33751783f9 [NO ISSUE][OTH] Move page size storage config to common config 33751783f9 is described below commit 33751783f931e76c0fbe1fc54b319d548569097b Author: Murtadha Hubail AuthorDate: Mon May 15 19:09:50 2023 +0300 [NO ISSUE][OTH] Move page size storage config to common config - user model changes: no - storage format changes: no - interface changes: no Details: - Move page size storage config used during compilation to the properties common section to avoid warning of usage outside NC scope. Change-Id: I43d603feec63261ec14b723d0d39e4a71461750f Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17531 Integration-Tests: Jenkins Tested-by: Jenkins Reviewed-by: Murtadha Hubail Reviewed-by: Wail Alkowaileet --- asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf| 2 +- asterixdb/asterix-app/src/main/resources/cc.conf | 2 +- asterixdb/asterix-app/src/main/resources/cc2.conf | 2 +- asterixdb/asterix-app/src/main/resources/cc3.conf | 2 +- asterixdb/asterix-app/src/main/resources/cc4.conf | 2 +- asterixdb/asterix-app/src/main/resources/cc_no_cbo.conf | 2 +- asterixdb/asterix-app/src/main/resources/cc_static_partitioning.conf | 2 +- asterixdb/asterix-app/src/test/resources/cc-analyze.conf | 4 ++-- asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf| 2 +- asterixdb/asterix-app/src/test/resources/cc-multipart.conf| 2 +- asterixdb/asterix-app/src/test/resources/cc-single-lookup.conf| 3 +-- asterixdb/asterix-app/src/test/resources/cc-single.conf | 2 +- .../asterix-app/src/test/resources/cc-small-txn-log-partition.conf| 2 +- asterixdb/asterix-app/src/test/resources/cc-ssl.conf | 4 ++-- asterixdb/asterix-app/src/test/resources/cc-storage.conf | 2 +- asterixdb/asterix-app/src/test/resources/cc.conf | 2 +- .../runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm | 1 + .../results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 + .../results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 + .../main/java/org/apache/asterix/common/config/StorageProperties.java | 1 + 20 files changed, 22 insertions(+), 19 deletions(-) diff --git a/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf b/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf index 2b03c9a4c6..be9bd544ea 100644 --- a/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf +++ b/asterixdb/asterix-app/src/main/resources/cc-cloud-storage.conf @@ -37,7 +37,6 @@ address=127.0.0.1 command=asterixnc app.class=org.apache.asterix.hyracks.bootstrap.NCApplication jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" -storage.buffercache.pagesize=32KB storage.buffercache.size=128MB storage.memorycomponent.globalbudget=512MB storage.io.scheduler=greedy @@ -66,3 +65,4 @@ compiler.internal.sanitycheck=true messaging.frame.size=4096 messaging.frame.count=512 cloud.deployment=true +storage.buffercache.pagesize=32KB diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf index dc6e5a2075..b531940447 100644 --- a/asterixdb/asterix-app/src/main/resources/cc.conf +++ b/asterixdb/asterix-app/src/main/resources/cc.conf @@ -37,7 +37,6 @@ address=127.0.0.1 command=asterixnc app.class=org.apache.asterix.hyracks.bootstrap.NCApplication jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" -storage.buffercache.pagesize=32KB storage.buffercache.size=128MB storage.memorycomponent.globalbudget=512MB storage.io.scheduler=greedy @@ -65,3 +64,4 @@ compiler.sort.parallel=false compiler.internal.sanitycheck=true messaging.frame.size=4096 messaging.frame.count=512 +storage.buffercache.pagesize=32KB diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf index c746198745..3f456cfa7d 100644 --- a/asterixdb/asterix-app/src/main/resources/cc2.conf +++ b/asterixdb/asterix-app/src/main/resources/cc2.conf @@ -35,7 +35,6 @@ address=127.0.0.1 command=asterixnc app.class=org.apache.asterix.hyracks.bootstrap.NCApplication jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory" -storage.buffercache.pagesize=32KB storage.buffercache.size=128MB storage.memorycomponent.globalbudget=512MB @@ -60,3 +59,4 @@ compiler.windowmemory=192KB compiler.parall
[asterixdb] branch master updated: [NO-ISSUE][MTD] Incorrect deserailization of dataset primary key types
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 41fae6a4ec [NO-ISSUE][MTD] Incorrect deserailization of dataset
primary key types
41fae6a4ec is described below
commit 41fae6a4ec144d7e74556e3c768ddcbf73b70976
Author: Peeyush Gupta
AuthorDate: Mon May 15 11:39:25 2023 -0700
[NO-ISSUE][MTD] Incorrect deserailization of dataset primary key types
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
While deserializing dataset details, all primary key types were
incorrectly set to string.
Change-Id: I20636cb8535709f607e8748e1fdd9383ab269cf7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17532
Integration-Tests: Jenkins
Tested-by: Murtadha Hubail
Reviewed-by: Murtadha Hubail
---
.../metadata/entitytupletranslators/DatasetTupleTranslator.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index e659b8fe21..70dfde1acc 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -149,7 +149,6 @@ public class DatasetTupleTranslator extends
AbstractTupleTranslator {
.getValueByPos(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_PARTITIONKEY_FIELD_INDEX))
.getCursor();
List> partitioningKey = new ArrayList<>();
-List partitioningKeyType = new ArrayList<>();
while (cursor.next()) {
AOrderedList fieldNameList = (AOrderedList) cursor.get();
@@ -159,7 +158,6 @@ public class DatasetTupleTranslator extends
AbstractTupleTranslator {
nestedFieldName.add(((AString)
nestedFieldNameCursor.get()).getStringValue());
}
partitioningKey.add(nestedFieldName);
-partitioningKeyType.add(BuiltinType.ASTRING);
}
// Check if there is a primary key types field
@@ -223,7 +221,7 @@ public class DatasetTupleTranslator extends
AbstractTupleTranslator {
}
datasetDetails = new InternalDatasetDetails(fileStructure,
partitioningStrategy, partitioningKey,
-partitioningKey, keyFieldSourceIndicator,
partitioningKeyType, autogenerated,
+partitioningKey, keyFieldSourceIndicator,
primaryKeyTypes, autogenerated,
filterSourceIndicator, filterField);
break;
}
[asterixdb] branch master updated: [NO ISSUE][OTH] Fix bucket name in S3Mock
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 43b0b3ed28 [NO ISSUE][OTH] Fix bucket name in S3Mock
43b0b3ed28 is described below
commit 43b0b3ed28baca75e6b9f47ac0b72d95a837fa7d
Author: Murtadha Hubail
AuthorDate: Sun May 14 21:17:25 2023 +0300
[NO ISSUE][OTH] Fix bucket name in S3Mock
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Fixed the bucket name in S3Mock to match the bucket name
used in the mock cloud storage config.
- Remove unused partitions count from ICloudStorageConfiguration.
Change-Id: I16e3c8b16ccab843ff4737bc9cc4df488277045d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17529
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Hussain Towaileb
---
.../src/test/java/org/apache/asterix/api/common/CloudUtils.java| 3 ++-
.../asterix/cloud/storage/FileCloudStorageConfiguration.java | 7 ---
.../apache/asterix/cloud/storage/ICloudStorageConfiguration.java | 5 -
.../asterix/cloud/storage/MockCloudStorageConfiguration.java | 6 --
4 files changed, 2 insertions(+), 19 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
index 5f13e0b33b..293a58041a 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/CloudUtils.java
@@ -20,6 +20,7 @@ package org.apache.asterix.api.common;
import java.net.URI;
+import org.apache.asterix.cloud.storage.MockCloudStorageConfiguration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -36,7 +37,7 @@ public class CloudUtils {
private static final int MOCK_SERVER_PORT = 8001;
private static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:; +
MOCK_SERVER_PORT;
-private static final String CLOUD_STORAGE_BUCKET = "cloud-storage-bucket";
+private static final String CLOUD_STORAGE_BUCKET =
MockCloudStorageConfiguration.INSTANCE.getContainer();
private static final String MOCK_SERVER_REGION = "us-west-2";
private static S3Mock s3MockServer;
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java
index 7cee20a06d..ef75bba8e8 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/FileCloudStorageConfiguration.java
@@ -28,7 +28,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
public class FileCloudStorageConfiguration implements
ICloudStorageConfiguration {
private final String containerName;
-private final int storagePartitionsCount;
public FileCloudStorageConfiguration(File file) throws
HyracksDataException {
if (!file.exists()) {
@@ -38,7 +37,6 @@ public class FileCloudStorageConfiguration implements
ICloudStorageConfiguration
try {
List lines = FileUtils.readLines(file, "UTF-8");
this.containerName = lines.get(0);
-this.storagePartitionsCount = Integer.parseInt(lines.get(1));
} catch (IOException ex) {
throw HyracksDataException.create(ex);
}
@@ -48,9 +46,4 @@ public class FileCloudStorageConfiguration implements
ICloudStorageConfiguration
public String getContainer() {
return containerName;
}
-
-@Override
-public int getPartitionsCount() {
-return storagePartitionsCount;
-}
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java
index e140c7cd8a..356b2ef06d 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/ICloudStorageConfiguration.java
@@ -29,9 +29,4 @@ public interface ICloudStorageConfiguration {
* @return returns the container name used for the storage
*/
String getContainer();
-
-/**
- * @return returns the number of storage partitions
- */
-int getPartitionsCount();
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/storage/MockCloudStorageConfiguration.java
b/asteri
[asterixdb] branch master updated: [NO ISSUE][OTH] Move compile time storage config to common config
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new a08a5a4bbf [NO ISSUE][OTH] Move compile time storage config to common config a08a5a4bbf is described below commit a08a5a4bbf892f51e37e329bd6e39b49a2ee202f Author: Murtadha Hubail AuthorDate: Sat May 13 20:31:46 2023 +0300 [NO ISSUE][OTH] Move compile time storage config to common config - user model changes: no - storage format changes: no - interface changes: no Details: - Move the storage properties used during compilation to the properties common section to avoid warning of usage outside NC scope. Change-Id: Iac165a252fcecd2e23bdd47cc69a67e5253edd58 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17527 Integration-Tests: Jenkins Reviewed-by: Till Westmann Tested-by: Jenkins --- .../runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm | 3 +++ .../results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm | 3 +++ .../results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm | 3 +++ .../main/java/org/apache/asterix/common/config/StorageProperties.java | 3 +++ 4 files changed, 12 insertions(+) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index f28c30311f..bdd3f515ef 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -51,7 +51,10 @@ "replication\.strategy" : "none", "replication\.timeout" : 120, "ssl\.enabled" : false, +"storage.column.free.space.tolerance" : 0.15, +"storage.column.max.tuple.count" : 15000, "storage.compression.block" : "snappy", +"storage.format" : "row", "storage.global.cleanup.timeout" : 600, "storage.lsm.bloomfilter.falsepositiverate" : 0.01, "storage.partitioning" : "dynamic", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index 99001d865f..7c416e6622 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -51,7 +51,10 @@ "replication\.strategy" : "none", "replication\.timeout" : 120, "ssl\.enabled" : false, +"storage.column.free.space.tolerance" : 0.15, +"storage.column.max.tuple.count" : 15000, "storage.compression.block" : "snappy", +"storage.format" : "row", "storage.global.cleanup.timeout" : 600, "storage.lsm.bloomfilter.falsepositiverate" : 0.01, "storage.partitioning" : "dynamic", diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index f62aee38c9..a28cc11642 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -51,7 +51,10 @@ "replication\.strategy" : "none", "replication\.timeout" : 120, "ssl\.enabled" : false, +"storage.column.free.space.tolerance" : 0.15, +"storage.column.max.tuple.count" : 15000, "storage.compression.block" : "snappy", +"storage.format" : "row", "storage.global.cleanup.timeout" : 600, "storage.lsm.bloomfilter.falsepositiverate" : 0.01, "storage.partitioning" : "dynamic", diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java index d61ee14273..70cd4ff65d 100644 --- a/asterixdb/asterix-common/src/main/jav
[asterixdb] branch master updated: [NO ISSUE][OTH] Balance testing jobs
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new 13b6b1f690 [NO ISSUE][OTH] Balance testing jobs 13b6b1f690 is described below commit 13b6b1f690298df102bbd0c89bda2be1d4c72c07 Author: Murtadha Hubail AuthorDate: Wed May 10 01:38:47 2023 +0300 [NO ISSUE][OTH] Balance testing jobs - user model changes: no - storage format changes: no - interface changes: no Change-Id: Ifc00576fe3854ba39f65d779f77a8d42b78eb605 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17520 Integration-Tests: Jenkins Reviewed-by: Ali Alsuliman Tested-by: Jenkins --- asterixdb/asterix-app/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml index 8e32fa2275..6e8224d42f 100644 --- a/asterixdb/asterix-app/pom.xml +++ b/asterixdb/asterix-app/pom.xml @@ -388,7 +388,7 @@ asterix-gerrit-asterix-app - **/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/AqlExecutionTest.java,**/*Compression*Test.java,**/*Ssl*Test.java,**/Podman*.java + **/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/*StaticPartitioning*Test.java,**/*Ssl*Test.java,**/Podman*.java,**/*AnalyzedExecutionTest.java,**/SqlppProfiledExecutionTest.java **/*.java @@ -437,7 +437,7 @@ asterix-gerrit-ssl-compression - **/*Compression*Test.java,**/*Ssl*Test.java + **/*StaticPartitioning*Test.java,**/*Ssl*Test.java,**/*AnalyzedExecutionTest.java **/*.java false @@ -445,7 +445,7 @@ asterix-gerrit-verify-asterix-app -**/AqlExecutionTest.java +**/SqlppProfiledExecutionTest.java **/External*IT.java,**/SqlppExecution*IT.java,**/SqlppRQG*IT.java,**/RebalanceWithCancellationIT.java false
[asterixdb] branch master updated: [ASTERIXDB-3144][CONF] Configurable storage partitions count
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new a69202d649 [ASTERIXDB-3144][CONF] Configurable storage partitions count a69202d649 is described below commit a69202d64926facc239a52b9348299910e5f7740 Author: Murtadha Hubail AuthorDate: Tue May 9 17:06:45 2023 +0300 [ASTERIXDB-3144][CONF] Configurable storage partitions count - user model changes: no - storage format changes: no - interface changes: yes Details: - Make number of storage partitions used for static partitioning configurable. Change-Id: I1549b408562910c326764f366c95af345da9 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17519 Integration-Tests: Jenkins Tested-by: Jenkins Reviewed-by: Ali Alsuliman --- .../api/cluster_state_1/cluster_state_1.1.regexadm | 1 + .../cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 + .../cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 + .../asterix/common/cluster/IClusterStateManager.java | 6 ++ .../common/cluster/StorageComputePartitionsMap.java | 16 ++-- .../apache/asterix/common/config/StorageProperties.java | 16 +--- .../apache/asterix/common/utils/StorageConstants.java| 1 - .../asterix/metadata/utils/DataPartitioningProvider.java | 8 .../metadata/utils/StaticDataPartitioningProvider.java | 2 +- .../asterix/runtime/utils/ClusterStateManager.java | 5 + 10 files changed, 42 insertions(+), 15 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index 801cfb09bc..f28c30311f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -55,6 +55,7 @@ "storage.global.cleanup.timeout" : 600, "storage.lsm.bloomfilter.falsepositiverate" : 0.01, "storage.partitioning" : "dynamic", +"storage.partitions.count" : 8, "txn\.commitprofiler\.enabled" : false, "txn\.commitprofiler\.reportinterval" : 5, "txn\.dataset\.checkpoint\.interval" : 3600, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index e54bd70ba9..99001d865f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -55,6 +55,7 @@ "storage.global.cleanup.timeout" : 600, "storage.lsm.bloomfilter.falsepositiverate" : 0.01, "storage.partitioning" : "dynamic", +"storage.partitions.count" : 8, "txn\.commitprofiler\.enabled" : false, "txn\.commitprofiler\.reportinterval" : 5, "txn\.dataset\.checkpoint\.interval" : 3600, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index c5496b26ee..f62aee38c9 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -55,6 +55,7 @@ "storage.global.cleanup.timeout" : 600, "storage.lsm.bloomfilter.falsepositiverate" : 0.01, "storage.partitioning" : "dynamic", +"storage.partitions.count" : 8, "txn\.commitprofiler\.enabled" : false, "txn\.commitprofiler\.reportinterval" : 5, "txn\.dataset\.checkpoint\.interval" : 3600, diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java index 76802d9a1b..3d0fee8e22 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/ICluster
[asterixdb] branch master updated: [NO ISSUE][OTH] Add option to enable/disable ingestion events logging
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 4b8e5aaf86 [NO ISSUE][OTH] Add option to enable/disable ingestion
events logging
4b8e5aaf86 is described below
commit 4b8e5aaf869fc296bb015ade906b54e3fa93ff48
Author: Ali Alsuliman
AuthorDate: Mon May 8 20:55:26 2023 -0700
[NO ISSUE][OTH] Add option to enable/disable ingestion events logging
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Add new feed option (log-ingestion-events) to enable/disable
ingestion events logging and default it to true.
Change-Id: I890c37e2ad70a5c7742e97ebdc311c9eae04e0b4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17518
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Ali Alsuliman
---
.../asterix/app/function/FunctionReader.java | 4 +--
.../results/feeds/feeds_01/feeds_01.1.adm | 2 +-
.../adapter/factory/GenericAdapterFactory.java | 18 +++
.../asterix/external/api/AsterixInputStream.java | 6 ++--
.../apache/asterix/external/api/IRecordReader.java | 4 +--
.../dataflow/AbstractFeedDataFlowController.java | 6 ++--
.../dataflow/ChangeFeedDataFlowController.java | 4 +--
.../ChangeFeedWithMetaDataFlowController.java | 4 +--
.../dataflow/FeedRecordDataFlowController.java | 4 +--
.../dataflow/FeedStreamDataFlowController.java | 4 +--
.../dataflow/FeedWithMetaDataFlowController.java | 7 +++--
.../reader/hdfs/AbstractHDFSRecordReader.java | 4 +--
.../record/reader/http/HttpServerRecordReader.java | 4 +--
.../input/record/reader/rss/RSSRecordReader.java | 4 +--
.../record/reader/stream/StreamRecordReader.java | 6 ++--
.../reader/twitter/TwitterPullRecordReader.java| 4 +--
.../reader/twitter/TwitterPushRecordReader.java| 4 +--
.../input/stream/AsterixInputStreamReader.java | 4 +--
.../external/input/stream/LocalFSInputStream.java | 4 +--
.../provider/DataflowControllerProvider.java | 4 +--
.../external/util/ExternalDataConstants.java | 1 +
.../asterix/external/util/ExternalDataUtils.java | 12
.../asterix/external/util/FeedLogManager.java | 17 ++
.../asterix/external/util/FileSystemWatcher.java | 4 +--
.../asterix/external/util/IFeedLogManager.java | 36 ++
.../asterix/external/util/NoOpFeedLogManager.java} | 36 +-
.../record/reader/TestAsterixMembersReader.java| 4 +--
.../input/record/reader/kv/KVTestReader.java | 4 +--
28 files changed, 141 insertions(+), 74 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionReader.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionReader.java
index c73f8e8e31..f0ecd89b7d 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionReader.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionReader.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.asterix.external.api.IRecordReader;
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
-import org.apache.asterix.external.util.FeedLogManager;
+import org.apache.asterix.external.util.IFeedLogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public abstract class FunctionReader implements IRecordReader {
@@ -43,7 +43,7 @@ public abstract class FunctionReader implements
IRecordReader {
}
@Override
-public void setFeedLogManager(FeedLogManager feedLogManager) throws
HyracksDataException {
+public void setFeedLogManager(IFeedLogManager feedLogManager) throws
HyracksDataException {
throw new UnsupportedOperationException();
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
index 8ada33cd91..b0a09aa699 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
@@ -1 +1 @@
-{ "DataverseName": "feeds", "FeedName": "TweetFeed", "AdapterConfiguration":
{{ { "Name": "dataset-dataverse", "Value": "feeds" }, { "Name": "path",
"Value": "asterix_nc1://data/twitter/obamatweets.adm" }, { "Name": "feed",
"Value": "TweetFeed" }, { "Name": "adapter-name", "Value": "lo
[asterixdb] branch master updated: [ASTERIXDB-3177][CONF] Store ingestion logs outside of data storage dir
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new fcba67 [ASTERIXDB-3177][CONF] Store ingestion logs outside of data
storage dir
fcba67 is described below
commit fcba6796b1569073d1d1db05c024bc72b58e
Author: Murtadha Hubail
AuthorDate: Mon May 8 16:40:28 2023 +0300
[ASTERIXDB-3177][CONF] Store ingestion logs outside of data storage dir
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Store ingestion logs on the first iodevice of each node rather
than inside the data storage dir.
- Move columnar dataset rebalance test to rebalance test suite.
Change-Id: Ia5f836fa99f3c982b6420b092d40d88f5f6429c1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17517
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../src/test/resources/runtimets/rebalance.xml | 5 +++
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ---
.../asterix/common/utils/StorageConstants.java | 1 +
.../asterix/common/utils/StoragePathUtil.java | 9
.../adapter/factory/GenericAdapterFactory.java | 5 ++-
.../apache/asterix/external/util/FeedUtils.java| 36
.../metadata/utils/DataPartitioningProvider.java | 2 +-
.../hyracks/api/io/DefaultIoDeviceFileSplit.java | 48 ++
8 files changed, 74 insertions(+), 37 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/rebalance.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/rebalance.xml
index 511e27bdcb..a4158ea511 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/rebalance.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/rebalance.xml
@@ -99,5 +99,10 @@ ERROR_BODY: {"results":"to rebalance a particular dataset,
the parameter dataver
all_datasets
+
+
+rebalance
+
+
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 08f71f7a17..299e32a63f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -16274,11 +16274,6 @@
secondary-index/create-index/after-upsert-with-meta
-
-
-rebalance
-
-
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
index c26fe76156..5d6322b0bd 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
@@ -31,6 +31,7 @@ import
org.apache.hyracks.storage.am.lsm.common.impls.ConcurrentMergePolicyFacto
public class StorageConstants {
public static final String STORAGE_ROOT_DIR_NAME = "storage";
+public static final String INGESTION_LOGS_DIR_NAME = "ingestion_logs";
public static final String PARTITION_DIR_PREFIX = "partition_";
/**
* Any file that shares the same directory as the LSM index files must
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
index c87f3685e0..a9ed066009 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
@@ -30,6 +30,7 @@ import
org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit
import
org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.DefaultIoDeviceFileSplit;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.MappedFileSplit;
@@ -65,11 +66,19 @@ public class StoragePathUtil {
return new MappedFileSplit(partition.getActiveNodeId(), relativePath,
partition.getIODeviceNum());
}
+public static FileSplit getDefaultIoDeviceFileSpiltForNode(String nodeId,
String relativePath) {
+return new DefaultIoDeviceFileSplit(nodeId, relativePath);
+}
+
public static String prepareStoragePartitionPat
[asterixdb] branch master updated: [NO ISSUE][TEST] Remove SqlppExecutionWithCompressionTest
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new e18c9332a9 [NO ISSUE][TEST] Remove SqlppExecutionWithCompressionTest
e18c9332a9 is described below
commit e18c9332a91d8a78bfc0ffeaeb271c5cc487874a
Author: Murtadha Hubail
AuthorDate: Sun May 7 19:53:47 2023 +0300
[NO ISSUE][TEST] Remove SqlppExecutionWithCompressionTest
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Remove SqlppExecutionWithCompressionTest as it is currently
a redundant test since the default storage configuration has
compression enabled.
Change-Id: Iaf35ce1d7574ee4edfd4414ceb573feb4a668a80
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17516
Integration-Tests: Jenkins
Reviewed-by: Till Westmann
Tested-by: Jenkins
---
.../runtime/SqlppExecutionWithCompressionTest.java | 64 --
.../src/test/resources/cc-compression.conf | 60
2 files changed, 124 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionWithCompressionTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionWithCompressionTest.java
deleted file mode 100644
index 4be21426d1..00
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionWithCompressionTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.test.runtime;
-
-import java.util.Collection;
-
-import org.apache.asterix.test.common.TestExecutor;
-import org.apache.asterix.testframework.context.TestCaseContext;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-/**
- * Runs the SQL++ storage runtime tests with the Snappy compression.
- */
-@RunWith(Parameterized.class)
-public class SqlppExecutionWithCompressionTest {
-protected static final String TEST_CONFIG_FILE_NAME =
"src/test/resources/cc-compression.conf";
-
-@BeforeClass
-public static void setUp() throws Exception {
-LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, new TestExecutor());
-}
-
-@AfterClass
-public static void tearDown() throws Exception {
-LangExecutionUtil.tearDown();
-}
-
-@Parameters(name = "SqlppExecutionWithCompressionTest {index}: {0}")
-public static Collection tests() throws Exception {
-return LangExecutionUtil.tests("only_sqlpp.xml",
"testsuite_sqlpp.xml");
-}
-
-protected TestCaseContext tcCtx;
-
-public SqlppExecutionWithCompressionTest(TestCaseContext tcCtx) {
-this.tcCtx = tcCtx;
-}
-
-@Test
-public void test() throws Exception {
-LangExecutionUtil.test(tcCtx);
-}
-}
diff --git a/asterixdb/asterix-app/src/test/resources/cc-compression.conf
b/asterixdb/asterix-app/src/test/resources/cc-compression.conf
deleted file mode 100644
index 373d9bb770..00
--- a/asterixdb/asterix-app/src/test/resources/cc-compression.conf
+++ /dev/null
@@ -1,60 +0,0 @@
-; Licensed to the Apache Software Foundation (ASF) under one
-; or more contributor license agreements. See the NOTICE file
-; distributed with this work for additional information
-; regarding copyright ownership. The ASF licenses this file
-; to you under the Apache License, Version 2.0 (the
-; "License"); you may not use this file except in compliance
-; with the License. You may obtain a copy of the License at
-;
-; http://www.apache.org/licenses/LICENSE-2.0
-;
-; Unless required by applicable law or agreed to in writing,
-; software distributed under the License is distributed on an
-; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-; KIND, either express or implied. See the Lic
[asterixdb] branch master updated: [ASTERIXDB-3144][HYR][RT] Make dump_index() function support multiple partitions
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9acfce145e [ASTERIXDB-3144][HYR][RT] Make dump_index() function
support multiple partitions
9acfce145e is described below
commit 9acfce145e846edcf719a009b8b925d027354604
Author: Ali Alsuliman
AuthorDate: Fri May 5 17:21:15 2023 -0700
[ASTERIXDB-3144][HYR][RT] Make dump_index() function support multiple
partitions
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
This patch changes the dump_index() function to support
operating on multiple partitions.
Change-Id: I8754069a7340c0d9e3bf69e1fe5c94eb333b73b5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17513
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Ali Alsuliman
Reviewed-by: Murtadha Hubail
---
.../asterix/app/function/DumpIndexDatasource.java | 13 +++--
.../asterix/app/function/DumpIndexFunction.java| 14 +++--
.../asterix/app/function/DumpIndexReader.java | 62 ++
.../asterix/app/function/DumpIndexRewriter.java| 9 ++--
.../LSMSecondaryIndexBulkLoadNodePushable.java | 2 +-
5 files changed, 65 insertions(+), 35 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 691be4706c..bd72a66d66 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -39,27 +39,30 @@ public class DumpIndexDatasource extends FunctionDataSource
{
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
-private final AlgebricksAbsolutePartitionConstraint storageLocations;
+private final AlgebricksAbsolutePartitionConstraint constraint;
+private final int[][] partitionsMap;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory
indexDataflowHelperFactory,
RecordDescriptor recDesc, IBinaryComparatorFactory[]
comparatorFactories,
-AlgebricksAbsolutePartitionConstraint storageLocations) throws
AlgebricksException {
+AlgebricksAbsolutePartitionConstraint constraint, int[][]
partitionsMap) throws AlgebricksException {
super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
-this.storageLocations = storageLocations;
+this.constraint = constraint;
+this.partitionsMap = partitionsMap;
}
@Override
protected AlgebricksAbsolutePartitionConstraint
getLocations(IClusterStateManager csm) {
-return storageLocations;
+return constraint;
}
@Override
protected IDatasourceFunction createFunction(MetadataProvider
metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
-return new DumpIndexFunction(locations, indexDataflowHelperFactory,
recDesc, comparatorFactories);
+return new DumpIndexFunction(locations, indexDataflowHelperFactory,
recDesc, comparatorFactories,
+partitionsMap);
}
@Override
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexFunction.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexFunction.java
index 2fdbef3f75..fcfe3e0b96 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexFunction.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexFunction.java
@@ -31,25 +31,31 @@ import
org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
public class DumpIndexFunction extends AbstractDatasourceFunction {
-private static final long serialVersionUID = 1L;
+private static final long serialVersionUID = 2L;
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
+private final int[][] partitionsMap;
public DumpIndexFunction(AlgebricksAbsolutePartitionConstraint locations,
IndexDataflowHelperFactory indexDataflowHelperFactory,
RecordDescriptor recDesc,
-IBinaryComparatorFactory[] comparatorFactories) {
+IBinaryComparatorFactory[] comparatorFactories, int
[asterixdb] branch master updated: [ASTERIXDB-3144][RT] Run correlated index bulkload at storage parallelism
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new de51cc4799 [ASTERIXDB-3144][RT] Run correlated index bulkload at
storage parallelism
de51cc4799 is described below
commit de51cc479920e26e88f041d8c5d064764f788a42
Author: Murtadha Hubail
AuthorDate: Fri May 5 21:22:32 2023 +0300
[ASTERIXDB-3144][RT] Run correlated index bulkload at storage parallelism
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- To ensure correctness, always run correlated secondary index
bulkload jobs using storage parallelism regardless of the number
of compute partitions. This will ensure that each storage partition
will produce the corresponding secondary index components.
Change-Id: I5f38e4b06bcd91479bae544619bf3a96dde3c500
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17512
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Ali Alsuliman
---
.../apache/asterix/metadata/utils/DatasetUtil.java | 15 ++--
.../SecondaryCorrelatedBTreeOperationsHelper.java | 3 ++-
...aryCorrelatedInvertedIndexOperationsHelper.java | 3 ++-
.../SecondaryCorrelatedRTreeOperationsHelper.java | 3 ++-
.../utils/SecondaryIndexOperationsHelper.java | 27 ++
5 files changed, 42 insertions(+), 9 deletions(-)
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 130e39c357..e882129823 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -585,7 +585,20 @@ public class DatasetUtil {
MetadataProvider metadataProvider) throws AlgebricksException {
PartitioningProperties partitioningProperties =
metadataProvider.getPartitioningProperties(dataset);
AlgebricksPartitionConstraint primaryPartitionConstraint =
partitioningProperties.getConstraints();
+IOperatorDescriptor dummyKeyProviderOp =
createDummyKeyProviderOp(spec);
+
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec,
dummyKeyProviderOp,
+primaryPartitionConstraint);
+return dummyKeyProviderOp;
+}
+public static IOperatorDescriptor
createCorrelatedDummyKeyProviderOp(JobSpecification spec,
+AlgebricksPartitionConstraint apc) throws AlgebricksException {
+IOperatorDescriptor dummyKeyProviderOp =
createDummyKeyProviderOp(spec);
+
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec,
dummyKeyProviderOp, apc);
+return dummyKeyProviderOp;
+}
+
+private static IOperatorDescriptor
createDummyKeyProviderOp(JobSpecification spec) throws AlgebricksException {
// Build dummy tuple containing one field with a dummy value inside.
ArrayTupleBuilder tb = new ArrayTupleBuilder(1);
DataOutput dos = tb.getDataOutput();
@@ -602,8 +615,6 @@ public class DatasetUtil {
RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
ConstantTupleSourceOperatorDescriptor keyProviderOp = new
ConstantTupleSourceOperatorDescriptor(spec,
keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
tb.getSize());
-
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec,
keyProviderOp,
-primaryPartitionConstraint);
return keyProviderOp;
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
index c59613781b..2b948ef623 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
@@ -75,7 +75,8 @@ public class SecondaryCorrelatedBTreeOperationsHelper extends
SecondaryCorrelate
IndexUtil.bindJobEventListener(spec, metadataProvider);
// Create dummy key provider for feeding the primary index scan.
-IOperatorDescriptor keyProviderOp =
DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider);
+IOperatorDescriptor keyProviderOp =
+DatasetUtil.createCorrelatedDummyKeyProviderOp(spec,
primaryPartitionConstraint);
// Create primary index scan op.
IOperatorDescriptor
[asterixdb] branch master updated: [ASTERIXDB-3144][RT] Implement Static Partitioning
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new f400d33287 [ASTERIXDB-3144][RT] Implement Static Partitioning
f400d33287 is described below
commit f400d33287536d81408c1475e510c50bdd344c27
Author: Murtadha Hubail
AuthorDate: Fri May 5 03:31:47 2023 +0300
[ASTERIXDB-3144][RT] Implement Static Partitioning
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Implement static partitioning based on storage/compute
partitions map.
- Fixes for LSMPrimaryInsertOperatorNodePushable state
keeping for working on multiple storage partitions.
Change-Id: Ieca7ffb0f48e16fba4dc5beb0868c1ef8ac9245e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17509
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../asterix/app/cc/CcApplicationContext.java | 2 +-
.../asterix/runtime/ClusterStateManagerTest.java | 11 ++
.../asterix/common/cluster/ComputePartition.java | 37 +++
.../common/cluster/SplitComputeLocations.java | 41
.../cluster/StorageComputePartitionsMap.java | 93 +
.../asterix/common/utils/StorageConstants.java | 2 +
.../adapter/factory/GenericAdapterFactory.java | 1 +
.../metadata/declared/MetadataProvider.java| 5 +-
.../metadata/utils/DataPartitioningProvider.java | 90 +++-
.../utils/DynamicDataPartitioningProvider.java | 57 ++
.../utils/StaticDataPartitioningProvider.java | 116 +
.../LSMPrimaryInsertOperatorNodePushable.java | 38 +++
.../asterix/runtime/utils/ClusterStateManager.java | 20 +++-
.../PersistentLocalResourceRepository.java | 2 +-
14 files changed, 440 insertions(+), 75 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
index a2d99a0332..e4247f0dd9 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CcApplicationContext.java
@@ -157,7 +157,7 @@ public class CcApplicationContext implements
ICcApplicationContext {
requestTracker = new RequestTracker(this);
configValidator = configValidatorFactory.create();
this.adapterFactoryService = adapterFactoryService;
-dataPartitioningProvider = new DataPartitioningProvider(this);
+dataPartitioningProvider = DataPartitioningProvider.create(this);
}
@Override
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ClusterStateManagerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ClusterStateManagerTest.java
index a2a3b48393..53b9294e2b 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ClusterStateManagerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ClusterStateManagerTest.java
@@ -33,8 +33,10 @@ import
org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.cluster.IGlobalRecoveryManager;
import org.apache.asterix.common.config.MetadataProperties;
+import org.apache.asterix.common.config.StorageProperties;
import org.apache.asterix.common.metadata.IMetadataBootstrap;
import org.apache.asterix.common.utils.NcLocalCounters;
+import org.apache.asterix.common.utils.PartitioningScheme;
import org.apache.asterix.hyracks.bootstrap.CCApplication;
import org.apache.asterix.runtime.transaction.ResourceIdManager;
import org.apache.asterix.runtime.utils.BulkTxnIdFactory;
@@ -231,6 +233,9 @@ public class ClusterStateManagerTest {
MetadataProperties metadataProperties = mockMetadataProperties();
Mockito.when(ccApplicationContext.getMetadataProperties()).thenReturn(metadataProperties);
+StorageProperties storageProperties = mockStorageProperties();
+
Mockito.when(ccApplicationContext.getStorageProperties()).thenReturn(storageProperties);
+
ResourceIdManager resourceIdManager = new ResourceIdManager(csm);
Mockito.when(ccApplicationContext.getResourceIdManager()).thenReturn(resourceIdManager);
@@ -258,6 +263,12 @@ public class ClusterStateManagerTest {
return metadataProperties;
}
+private StorageProperties mockStorageProperties() {
+StorageProperties storageProperties =
Mockito.mock(StorageProperties.class);
+
Mockito.when(storageProperties.getPartitioningScheme()).thenReturn
[asterixdb] branch master updated: [ASTERIXDB-3174][COMP] Support CREATE DATASET AS SELECT Syntax
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 0814b0e559 [ASTERIXDB-3174][COMP] Support CREATE DATASET AS SELECT
Syntax
0814b0e559 is described below
commit 0814b0e559f1c677972db6e59b35e47b99c84a96
Author: Peeyush Gupta
AuthorDate: Wed May 3 11:08:12 2023 -0700
[ASTERIXDB-3174][COMP] Support CREATE DATASET AS SELECT Syntax
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
With this change we add support for CREATE DATASET AS SELECT syntax
that can be used to initialize a dataset during it creation by
upserting data from the result of a query.
Change-Id: I4d49b944ed11532ff45b6862345b11b9fae236c1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17508
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Peeyush Gupta
Reviewed-by: Murtadha Hubail
---
.../asterix/app/translator/QueryTranslator.java| 14
.../create-dataset-3/create-dataset-3.01.ddl.sqlpp | 24
.../create-dataset-3.02.update.sqlpp | 23 +++
.../create-dataset-3/create-dataset-3.03.ddl.sqlpp | 25 +
.../create-dataset-3.04.update.sqlpp | 25 +
.../create-dataset-3.05.query.sqlpp| 22 ++
.../create-dataset-3/create-dataset-3.06.ddl.sqlpp | 23 +++
.../create-dataset-3/create-dataset-3.07.ddl.sqlpp | 26 ++
.../ddl/create-dataset-3/create-dataset-3.05.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 6 +
.../asterix/lang/common/statement/DatasetDecl.java | 12 ++
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj| 13 +--
12 files changed, 212 insertions(+), 2 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 43218d0552..d2fd4be2be 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -151,6 +151,7 @@ import
org.apache.asterix.lang.common.statement.StopFeedStatement;
import org.apache.asterix.lang.common.statement.SynonymDropStatement;
import org.apache.asterix.lang.common.statement.TypeDecl;
import org.apache.asterix.lang.common.statement.TypeDropStatement;
+import org.apache.asterix.lang.common.statement.UpsertStatement;
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.statement.ViewDropStatement;
import org.apache.asterix.lang.common.struct.Identifier;
@@ -730,9 +731,22 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
doCreateDatasetStatement(metadataProvider, dd, dataverseName,
datasetName, itemTypeDataverseName,
itemTypeExpr, itemTypeName, metaItemTypeExpr,
metaItemTypeDataverseName, metaItemTypeName, hcc,
requestParameters);
+if (dd.getQuery() != null) {
+final IResultSet resultSet = requestParameters.getResultSet();
+final ResultDelivery resultDelivery =
requestParameters.getResultProperties().getDelivery();
+final Stats stats = requestParameters.getStats();
+IStatementRewriter stmtRewriter =
rewriterFactory.createStatementRewriter();
+final ResultMetadata outMetadata =
requestParameters.getOutMetadata();
+final Map stmtParams =
requestParameters.getStatementParameters();
+UpsertStatement upsertStmt =
+new UpsertStatement(dataverseName, datasetName,
dd.getQuery(), -1, null, null);
+handleInsertUpsertStatement(metadataProvider, upsertStmt, hcc,
resultSet, resultDelivery, outMetadata,
+stats, requestParameters, stmtParams, stmtRewriter);
+}
} finally {
metadataProvider.getLocks().unlock();
}
+
}
protected Optional
doCreateDatasetStatement(MetadataProvider metadataProvider, DatasetDecl dd,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.01.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.01.ddl.sqlpp
new file mode 100644
index 00..11b6f9ba46
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.01.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software
[asterixdb] branch master updated: [ASTERIXDB-3154] Adds apache iceberg table read support
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new f6d772cdad [ASTERIXDB-3154] Adds apache iceberg table read support f6d772cdad is described below commit f6d772cdad769ec9fc27c9f5402feeed453ba6c1 Author: hariuserx AuthorDate: Wed May 3 11:13:24 2023 -0700 [ASTERIXDB-3154] Adds apache iceberg table read support - user model changes: yes - Introduced two new DDL parameters “table-format” and “metadata-path” specific to iceberg - storage format changes: no - interface changes: no Details: These changes add support in AsterixDB to read apache iceberg tables (version 1) as external data. Iceberg table details can be specified in the external dataset DDL and all queries fetch the data from the latest iceberg snapshot. APE link: https://cwiki.apache.org/confluence/display/ASTERIXDB/APE+1%3A+Iceberg+API+Integration * External Adapters modified: AWS S3 and Hadoop * Removed kite-sdk dependency and extracted the schema inferrence utilities from kite-sdk Change-Id: I12df589a6dffdc5af4a5cace68a11729995ea9af Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17419 Integration-Tests: Jenkins Reviewed-by: Ian Maxon Tested-by: Jenkins --- asterixdb/asterix-app/pom.xml | 60 +- .../aws/AwsS3ExternalDatasetOnePartitionTest.java | 4 + .../aws/AwsS3ExternalDatasetTest.java | 186 +++ .../parquet/BinaryFileConverterUtil.java | 21 +- .../test/external_dataset/parquet/JsonUtil.java| 614 + .../test/external_dataset/parquet/SchemaUtil.java | 557 +++ .../test/runtime/SqlppHdfsExecutionTest.java | 73 +++ .../s3/iceberg-empty/iceberg-empty.00.ddl.sqlpp| 36 ++ .../s3/iceberg-empty/iceberg-empty.01.query.sqlpp | 22 + .../iceberg-metadata-invalid-location.00.ddl.sqlpp | 36 ++ ...ceberg-metadata-invalid-location.01.query.sqlpp | 22 + .../iceberg-load-selective-metadata.00.ddl.sqlpp | 36 ++ .../iceberg-load-selective-metadata.01.query.sqlpp | 22 + .../iceberg-mixed-data-format.00.ddl.sqlpp | 36 ++ .../iceberg-mixed-data-format.01.query.sqlpp | 22 + .../iceberg-modified-data.00.ddl.sqlpp | 36 ++ .../iceberg-modified-data.01.query.sqlpp | 22 + .../iceberg-multiple-data-files.00.ddl.sqlpp | 36 ++ .../iceberg-multiple-data-files.01.query.sqlpp | 22 + .../iceberg-unsupported-version.00.ddl.sqlpp | 36 ++ .../iceberg-unsupported-version.01.query.sqlpp | 22 + .../iceberg-read-from-latest-snapshot.00.ddl.sqlpp | 36 ++ ...ceberg-read-from-latest-snapshot.01.query.sqlpp | 22 + .../hdfs/iceberg/read-data.1.ddl.sqlpp | 33 ++ .../hdfs/iceberg/read-data.2.query.sqlpp | 22 + .../s3/iceberg-empty/iceberg-empty.01.adm | 0 .../iceberg-modified-data.01.adm | 3 + .../iceberg-multiple-data-files.01.adm | 6 + .../iceberg-read-from-latest-snapshot.01.adm | 3 + .../runtimets/results/hdfs/iceberg/read-data.2.adm | 3 + .../runtimets/testsuite_external_dataset_s3.xml| 45 ++ .../resources/runtimets/testsuite_sqlpp_hdfs.xml | 5 + .../asterix/common/exceptions/ErrorCode.java | 3 + .../src/main/resources/asx_errormsg/en.properties | 3 + asterixdb/asterix-external-data/pom.xml| 10 + .../aws/parquet/AwsS3ParquetReaderFactory.java | 3 +- .../external/util/ExternalDataConstants.java | 4 + .../asterix/external/util/ExternalDataUtils.java | 85 ++- .../asterix/external/util/aws/s3/S3Constants.java | 1 + asterixdb/asterix-server/pom.xml | 49 +- asterixdb/pom.xml | 14 +- .../appended-resources/supplemental-models.xml | 111 ...nt.com_airlift_aircompressor_0.21_notice.md.txt | 37 ++ ...hubusercontent.com_apache_orc_v1.8.0_NOTICE.txt | 8 + hyracks-fullstack/hyracks/hyracks-hdfs/pom.xml | 8 + 45 files changed, 2410 insertions(+), 25 deletions(-) diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml index baa3be1ced..a3518852d0 100644 --- a/asterixdb/asterix-app/pom.xml +++ b/asterixdb/asterix-app/pom.xml @@ -878,11 +878,65 @@ com.google.cloud google-cloud-storage - - org.kitesdk - kite-data-core + org.apache.iceberg + iceberg-core + 1.1.0 test + + org.apache.iceberg + iceberg-data + 1.1.0 + + + org.slf4j + slf4j-api + + + org.apache.parquet + parquet-avro
[asterixdb] branch master updated: [ASTERIXDB-3144][RT] Introduce DataPartitioningProvider
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 09f4cdc2a9 [ASTERIXDB-3144][RT] Introduce DataPartitioningProvider
09f4cdc2a9 is described below
commit 09f4cdc2a9c879169808380bf64f4bda1af89390
Author: Murtadha Hubail
AuthorDate: Wed May 3 01:30:41 2023 +0300
[ASTERIXDB-3144][RT] Introduce DataPartitioningProvider
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add storage partitioning scheme config (dyanmic or static) and
default it to dynamic.
- Introduce DataPartitioningProvider which encapsulates the logic
for dataset partitioning based on the partitioning scheme.
Change-Id: Ia2bbc716fb4c2e9abca06e8f8629b15bd48bc7f3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17503
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
Tested-by: Murtadha Hubail
---
.../operators/physical/BTreeSearchPOperator.java | 8 +-
.../operators/physical/InvertedIndexPOperator.java | 19 +-
.../rules/am/IntroduceSelectAccessMethodRule.java | 7 +-
.../asterix/app/cc/CcApplicationContext.java | 9 +
.../org/apache/asterix/utils/DataverseUtil.java| 13 +-
.../org/apache/asterix/utils/FlushDatasetUtil.java | 9 +-
.../asterix/app/bootstrap/TestNodeController.java | 16 +-
.../api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../cluster_state_1_full.1.regexadm| 1 +
.../cluster_state_1_less.1.regexadm| 1 +
.../common/cluster/PartitioningProperties.java | 56 +
.../asterix/common/config/StorageProperties.java | 12 +-
.../common/dataflow/ICcApplicationContext.java | 7 +
.../common/dataflow/IDataPartitioningProvider.java | 22 ++
.../asterix/common/utils/PartitioningScheme.java | 52 +
.../declared/DataSourcePartitioningProvider.java | 12 +-
.../metadata/declared/MetadataProvider.java| 232 +
.../metadata/utils/DataPartitioningProvider.java | 93 +
.../apache/asterix/metadata/utils/DatasetUtil.java | 79 ---
.../metadata/utils/SampleOperationsHelper.java | 25 ++-
.../utils/SecondaryIndexOperationsHelper.java | 33 ++-
.../utils/SecondaryTreeIndexOperationsHelper.java | 35 ++--
.../tests/am/btree/AbstractBTreeOperatorTest.java | 3 +-
.../tests/am/rtree/AbstractRTreeOperatorTest.java | 3 +-
24 files changed, 483 insertions(+), 265 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
index 8865bb2241..6c6b2aa233 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
@@ -21,6 +21,7 @@ package org.apache.asterix.algebra.operators.physical;
import java.util.ArrayList;
import java.util.List;
+import org.apache.asterix.common.cluster.PartitioningProperties;
import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -241,10 +242,9 @@ public class BTreeSearchPOperator extends
IndexSearchPOperator {
propsLocal.add(new LocalOrderProperty(orderColumns));
MetadataProvider mp = (MetadataProvider)
context.getMetadataProvider();
Dataset dataset =
mp.findDataset(searchIndex.getDataverseName(), searchIndex.getDatasetName());
-int[][] partitionsMap = mp.getPartitionsMap(dataset);
-pv[0] = new StructuralPropertiesVector(
-
UnorderedPartitionedProperty.ofPartitionsMap(searchKeyVars, domain,
partitionsMap),
-propsLocal);
+PartitioningProperties partitioningProperties =
mp.getPartitioningProperties(dataset);
+pv[0] = new
StructuralPropertiesVector(UnorderedPartitionedProperty.ofPartitionsMap(searchKeyVars,
+domain,
partitioningProperties.getComputeStorageMap()), propsLocal);
return new PhysicalRequirements(pv,
IPartitioningRequirementsCoordinator.NO_COORDINATION);
}
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index 5bdb2dba5a..20334bf277 100644
[asterixdb] branch master updated: [ASTERIXDB-3170][COMP] Support COPY Statment
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 066fd5657f [ASTERIXDB-3170][COMP] Support COPY Statment
066fd5657f is described below
commit 066fd5657f2a2a89de54c47b6d62facc858175ed
Author: Peeyush Gupta
AuthorDate: Sun Apr 30 13:03:00 2023 -0700
[ASTERIXDB-3170][COMP] Support COPY Statment
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
With this change, we add support for COPY statement, that can be
used to upsert data into a dataset from an external data source.
Change-Id: I612978472f090ab3c32e901aa37087ed5b7edf92
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17499
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Peeyush Gupta
Reviewed-by: Murtadha Hubail
---
.../base/ILangExpressionToPlanTranslator.java | 4 +-
.../asterix/translator/CompiledStatements.java | 44 ++
.../translator/LangExpressionToPlanTranslator.java | 94 +++---
.../apache/asterix/api/common/APIFramework.java| 13 +--
.../asterix/app/translator/QueryTranslator.java| 59 ++
.../queries_sqlpp/copy/copy-1/copy-1.1.ddl.sqlpp | 44 ++
.../copy/copy-1/copy-1.2.update.sqlpp | 65 +++
.../queries_sqlpp/copy/copy-1/copy-1.3.query.sqlpp | 23 ++
.../copy/copy-2/copy-2.0.container.sqlpp | 20 +
.../queries_sqlpp/copy/copy-2/copy-2.1.ddl.sqlpp | 25 ++
.../copy/copy-2/copy-2.2.update.sqlpp | 52
.../queries_sqlpp/copy/copy-2/copy-2.3.query.sqlpp | 23 ++
.../runtimets/results/copy/copy-1/copy-1.1.adm | 1 +
.../runtimets/results/copy/copy-2/copy-2.1.adm | 1 +
.../runtimets/testsuite_external_dataset_s3.xml| 7 ++
.../test/resources/runtimets/testsuite_sqlpp.xml | 7 ++
.../apache/asterix/lang/common/base/Statement.java | 1 +
.../lang/common/statement/CopyStatement.java | 90 +
.../lang/common/visitor/FormatPrintVisitor.java| 11 +++
.../base/AbstractQueryExpressionVisitor.java | 6 ++
.../lang/common/visitor/base/ILangVisitor.java | 3 +
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj| 34 +++-
22 files changed, 591 insertions(+), 36 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExpressionToPlanTranslator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExpressionToPlanTranslator.java
index fea93400f1..0116576cde 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExpressionToPlanTranslator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExpressionToPlanTranslator.java
@@ -51,10 +51,10 @@ public interface ILangExpressionToPlanTranslator {
*
* @param stmt,
*the compiled load statement.
- * @return a logical query plan for the load statement.
+ * @return a logical query plan for the Copy/Load statement.
* @throws AlgebricksException
*/
-public ILogicalPlan translateLoad(ICompiledDmlStatement stmt) throws
AlgebricksException;
+public ILogicalPlan translateCopyOrLoad(ICompiledDmlStatement stmt) throws
AlgebricksException;
/**
* @return the current minimum available variable id.
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index 088676c520..4981f0ef04 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -286,6 +286,50 @@ public class CompiledStatements {
}
}
+public static class CompiledCopyFromFileStatement extends
AbstractCompiledStatement
+implements ICompiledDmlStatement {
+private final DataverseName dataverseName;
+private final String datasetName;
+private final String adapter;
+private final Map properties;
+
+public CompiledCopyFromFileStatement(DataverseName dataverseName,
String datasetName, String adapter,
+Map properties) {
+this.dataverseName = dataverseName;
+this.datasetName = datasetName;
+this.adapter = adapter;
+this.properties = properties;
+}
+
+@Override
+public DataverseName getDataverseName() {
+return dataverseName;
+}
+
+@Override
+public String getDatasetName() {
+return datasetName
[asterixdb] branch master updated: [ASTERIXDB-3166][FUN] Error with get_object_fields with open list type
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new bacbacbfe7 [ASTERIXDB-3166][FUN] Error with get_object_fields with
open list type
bacbacbfe7 is described below
commit bacbacbfe77fdd7e3a1e8225834826cb0b6d97c7
Author: Peeyush Gupta
AuthorDate: Thu Apr 13 15:49:00 2023 -0700
[ASTERIXDB-3166][FUN] Error with get_object_fields with open list type
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
get_object_fields function causes null pointer exception when used
on a dataset containing an open list type.
Change-Id: Ie413d286738f571df81e2f450a3fb0b132dc5521
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17488
Reviewed-by: Peeyush Gupta
Reviewed-by: Ali Alsuliman
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../queries_sqlpp/objects/ObjectsQueries.xml | 5 +++
.../open-list-type/open-list-type.1.ddl.sqlpp | 34 +++
.../open-list-type/open-list-type.2.update.sqlpp | 34 +++
.../open-list-type/open-list-type.3.query.sqlpp| 22 ++
.../open-list-type/open-list-type.4.query.sqlpp| 22 ++
.../open-list-type/open-list-type.3.adm| 1 +
.../open-list-type/open-list-type.4.adm| 1 +
.../om/pointables/nonvisitor/AListPointable.java | 2 +-
.../functions/records/RecordFieldsUtil.java| 51 ++
9 files changed, 152 insertions(+), 20 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
index 77fc7c5682..1756378f84 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
@@ -82,6 +82,11 @@
missing-fields
+
+
+ open-list-type
+
+
documentation-example
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp
new file mode 100644
index 00..402cc11311
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.1.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Testing get-object-fields where some schema (complex) fields
are missing or null
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE t1 AS {
+ id: string
+};
+
+
+CREATE DATASET ds1(t1) PRIMARY KEY id;
+CREATE DATASET ds2(t1) PRIMARY KEY id;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp
new file mode 100644
index 00..d5e98b4992
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/get-object-fields/open-list-type/open-list-type.2.update.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless requir
[asterixdb] branch master updated: [NO ISSUE][OTH] Create dataset without type specification
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6ba5a2670a [NO ISSUE][OTH] Create dataset without type specification
6ba5a2670a is described below
commit 6ba5a2670a6de81f3ba1fece98fd7900311f4b04
Author: Peeyush Gupta
AuthorDate: Thu Apr 13 11:08:53 2023 -0700
[NO ISSUE][OTH] Create dataset without type specification
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
With this change, users can create datasets without providing type
specification. For example the following statement creates a user
datase with id as primary key of type string:
CREATE DATASET user PRIMARY KEY(id: string)
Change-Id: I333d2b0dbbccef51dd3a11c611af10ac6a665ee3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17485
Integration-Tests: Jenkins
Reviewed-by: Peeyush Gupta
Reviewed-by: Murtadha Hubail
Tested-by: Jenkins
---
.../operators/physical/BTreeSearchPOperator.java | 2 +
.../IntroduceSecondaryIndexInsertDeleteRule.java | 2 +
.../IntroducePrimaryIndexForAggregationRule.java | 3 +
.../rules/am/OptimizableOperatorSubTree.java | 2 +
.../translator/LangExpressionToPlanTranslator.java | 4 ++
.../asterix/translator/util/ValidateUtil.java | 77 +++---
.../asterix/app/function/QueryIndexRewriter.java | 2 +
.../asterix/app/translator/QueryTranslator.java| 15 -
.../org/apache/asterix/utils/RebalanceUtil.java| 1 +
.../create-dataset-1/create-dataset.01.ddl.sqlpp | 33 ++
.../create-dataset.02.update.sqlpp | 30 +
.../create-dataset-1/create-dataset.03.query.sqlpp | 23 +++
.../create-dataset-1/create-dataset.04.query.sqlpp | 23 +++
.../create-dataset-1/create-dataset.05.query.sqlpp | 23 +++
.../create-dataset-1/create-dataset.06.query.sqlpp | 23 +++
.../create-dataset-1/create-dataset.07.ddl.sqlpp | 25 +++
.../create-dataset-2/create-dataset.01.ddl.sqlpp | 25 +++
.../create-dataset.02.update.sqlpp | 23 +++
.../create-dataset-2/create-dataset.03.ddl.sqlpp | 22 +++
.../ddl/create-dataset-1/create-dataset.03.adm | 2 +
.../ddl/create-dataset-1/create-dataset.04.adm | 2 +
.../ddl/create-dataset-1/create-dataset.05.adm | 2 +
.../ddl/create-dataset-1/create-dataset.06.adm | 2 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 12
.../util/ColumnSecondaryIndexSchemaUtil.java | 12
.../lang/common/statement/InternalDetailsDecl.java | 17 -
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj| 76 +
.../metadata/declared/MetadataManagerUtil.java | 31 +
.../metadata/declared/MetadataProvider.java| 5 ++
.../apache/asterix/metadata/entities/Dataset.java | 3 +
.../metadata/entities/InternalDatasetDetails.java | 34 +-
.../DatasetTupleTranslator.java| 15 +
.../IndexTupleTranslator.java | 2 +
.../apache/asterix/metadata/utils/DatasetUtil.java | 4 ++
.../metadata/utils/SampleOperationsHelper.java | 2 +
.../utils/SecondaryIndexOperationsHelper.java | 4 +-
36 files changed, 539 insertions(+), 44 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
index b8eba74b00..d56963e596 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
@@ -146,6 +146,8 @@ public class BTreeSearchPOperator extends
IndexSearchPOperator {
IProjectionFiltrationInfo metaProjectionInfo =
unnestMapOp.getMetaProjectionInfo();
ARecordType datasetType = (ARecordType)
metadataProvider.findType(dataset);
ARecordType metaItemType = (ARecordType)
metadataProvider.findMetaType(dataset);
+datasetType = (ARecordType)
metadataProvider.findTypeForDatasetWithoutType(datasetType,
+metaItemType, dataset);
tupleProjectorFactory =
IndexUtil.createTupleProjectorFactory(formatInfo, projectionInfo,
metaProjectionInfo, datasetType, metaItemType,
dataset.getPrimaryKeys().size());
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules
[asterixdb] branch master updated: [ASTERIXDB-3160][CONF] Add option to indicate cloud deployment
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 627666d782 [ASTERIXDB-3160][CONF] Add option to indicate cloud
deployment
627666d782 is described below
commit 627666d78235df45813f00e40968049382b249c7
Author: Murtadha Hubail
AuthorDate: Fri Mar 31 17:38:57 2023 +0300
[ASTERIXDB-3160][CONF] Add option to indicate cloud deployment
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add a config to indicate if the cluster is being deployed in
a cloud environment. This config can be used to optimize certain
operations for cloud environments.
Change-Id: I38c2bd3691b5f17b9c557533478f790fbb8e3cbe
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17459
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../results/api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../api/cluster_state_1_full/cluster_state_1_full.1.regexadm| 1 +
.../api/cluster_state_1_less/cluster_state_1_less.1.regexadm| 1 +
.../apache/hyracks/control/common/controllers/ControllerConfig.java | 6 +-
4 files changed, 8 insertions(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 0d42b1b374..4e96365008 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -9,6 +9,7 @@
"active\.stop\.timeout" : 3600,
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
+"cloud.deployment" : false,
"compiler\.arrayindex" : true,
"compiler.batch.lookup" : true,
"compiler.cbo" : false,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 6a123e1adf..415e96de01 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -9,6 +9,7 @@
"active\.stop\.timeout" : 3600,
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
+"cloud.deployment" : false,
"compiler\.arrayindex" : true,
"compiler.batch.lookup" : true,
"compiler.cbo" : false,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index d72d8bc2b3..add09ca27c 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -9,6 +9,7 @@
"active\.stop\.timeout" : 3600,
"active\.suspend\.timeout" : 3600,
"azure.request.timeout" : 120,
+"cloud.deployment" : false,
"compiler\.arrayindex" : true,
"compiler.batch.lookup" : true,
"compiler.cbo" : false,
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
index 19eba57d88..de0c174535 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
@@ -50,7 +50,8 @@ public class ControllerConfig implements Serializable {
(Function) appConfig -> FileUtil
.joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "logs"),
"The directory where logs for this node are written"
[asterixdb] branch master updated: [ASTERIXDB-3155][SQL] Supporting escape backticks and singleQuote in SQL++
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new df16f3b867 [ASTERIXDB-3155][SQL] Supporting escape backticks and
singleQuote in SQL++
df16f3b867 is described below
commit df16f3b867ae2e3887475c38735146ac2eb1e43a
Author: Ritik Raj
AuthorDate: Mon Mar 27 19:56:56 2023 +0530
[ASTERIXDB-3155][SQL] Supporting escape backticks and singleQuote in SQL++
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
This patch contains change to allow escaping of backticks
so that field containing backticks don't throw
parsing error when they are used in the query.
Allows to
1. escape backticks using backticks(`) and reverse solidus (\)
2. escape singleQuotes(') using (')
Change-Id: I5d9069c6aaa1365545f7e0ca728be6ea2ca4641d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17450
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Peeyush Gupta
Reviewed-by: Murtadha Hubail
---
.../escaping/failure/escape.4.query.sqlpp | 24 ++
.../escaping/success/escape.1.query.sqlpp | 24 ++
.../escaping/success/escape.2.query.sqlpp | 24 ++
.../escaping/success/escape.3.query.sqlpp | 24 ++
.../escaping/success/escape.4.query.sqlpp | 25 ++
.../escaping/success/escape.5.query.sqlpp | 25 ++
.../escaping/success/escape.6.query.sqlpp | 26 +++
.../escaping/success/escape.7.query.sqlpp | 26 +++
.../results/select-star/escaping/escape.1.adm | 1 +
.../results/select-star/escaping/escape.2.adm | 1 +
.../results/select-star/escaping/escape.3.adm | 1 +
.../results/select-star/escaping/escape.4.adm | 1 +
.../results/select-star/escaping/escape.5.adm | 1 +
.../results/select-star/escaping/escape.6.adm | 1 +
.../results/select-star/escaping/escape.7.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 11 +++
.../asterix/lang/common/parser/ScopeChecker.java | 91 +-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj| 6 ++
18 files changed, 275 insertions(+), 38 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/escaping/failure/escape.4.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/escaping/failure/escape.4.query.sqlpp
new file mode 100644
index 00..8ed44c0f9d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/escaping/failure/escape.4.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+* Should Throw Error when backticks(`) is not escaped
+*/
+
+select t.`first `name` from [{"userID":"1", "first `name":"XYZ"}] t;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/escaping/success/escape.1.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/escaping/success/escape.1.query.sqlpp
new file mode 100644
index 00..d9933ce3a5
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/select-star/escaping/success/escape.1.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS
[asterixdb] branch master updated: [ASTERIXDB-3156][API] Allow specifying lossless-adm as format
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new ffdcf57835 [ASTERIXDB-3156][API] Allow specifying lossless-adm as
format
ffdcf57835 is described below
commit ffdcf57835a49b049fd65a840469468bc7274fa1
Author: Murtadha Hubail
AuthorDate: Tue Mar 28 00:35:42 2023 +0300
[ASTERIXDB-3156][API] Allow specifying lossless-adm as format
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Allow specifying `lossless-adm` as format in the
REST API.
- Add test case.
Change-Id: I24057c56bafbe021b5d4da93a82cdaa0213bccd5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17451
Integration-Tests: Jenkins
Reviewed-by: Ali Alsuliman
Tested-by: Jenkins
---
.../http/server/QueryServiceRequestParameters.java | 2 ++
.../request-param/request-param.003.query.sqlpp| 24 ++
.../api/request-param/request-param.003.adm| 2 ++
3 files changed, 28 insertions(+)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
index df068c0824..b6913e448b 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
@@ -626,6 +626,8 @@ public class QueryServiceRequestParameters {
return Pair.of(OutputFormat.CLEAN_JSON, Boolean.FALSE);
} else if (value.equals(HttpUtil.ContentType.ADM)) {
return Pair.of(OutputFormat.ADM, Boolean.FALSE);
+} else if (value.equals(Attribute.LOSSLESS_ADM.str())) {
+return Pair.of(OutputFormat.LOSSLESS_ADM_JSON, Boolean.FALSE);
} else {
throw new RuntimeDataException(ErrorCode.INVALID_REQ_PARAM_VAL,
parameterName, value);
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.003.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.003.query.sqlpp
new file mode 100644
index 00..52800c8708
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/api/request-param/request-param.003.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// requesttype=application/json
+// param format:string=lossless-adm
+
+from [1, 2] as v
+select v;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.003.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.003.adm
new file mode 100644
index 00..cb88308596
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/request-param/request-param.003.adm
@@ -0,0 +1,2 @@
+{ "v": 1 }
+{ "v": 2 }
\ No newline at end of file
[asterixdb] branch master updated: [ASTERIXDB-3153][OTH] Make the default storage format configurable
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5ba7d8152b [ASTERIXDB-3153][OTH] Make the default storage format
configurable
5ba7d8152b is described below
commit 5ba7d8152b6f157676bd03820eb917009cb155a0
Author: Wail Alkowaileet
AuthorDate: Sun Mar 26 10:30:07 2023 -0700
[ASTERIXDB-3153][OTH] Make the default storage format configurable
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
Currently, columnar datasets must be declared explicitly using
the WITH clause. We should extend AsterixDB's capability
to configure the default storage format (either row or column).
Change-Id: I173dd026528aa4d35dbdddcf1de4a55249c19caf
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17447
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Wail Alkowaileet
Reviewed-by: Murtadha Al Hubail
---
.../asterix/app/translator/QueryTranslator.java | 4 ++--
.../asterix/common/config/StorageProperties.java| 9 -
.../asterix/lang/common/statement/DatasetDecl.java | 21 ++---
.../asterix/metadata/dataset/DatasetFormatInfo.java | 6 +-
.../apache/asterix/metadata/entities/Dataset.java | 2 +-
.../DatasetTupleTranslator.java | 4 ++--
.../DatasetTupleTranslatorTest.java | 2 +-
.../IndexTupleTranslatorTest.java | 2 +-
.../apache/asterix/object/base/AdmObjectNode.java | 14 ++
9 files changed, 48 insertions(+), 16 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 15a8238a6b..3fa74230bc 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -748,8 +748,8 @@ public class QueryTranslator extends AbstractLangTranslator
implements IStatemen
boolean itemTypeAdded = false, metaItemTypeAdded = false;
StorageProperties storageProperties =
metadataProvider.getStorageProperties();
-DatasetFormatInfo datasetFormatInfo =
dd.getDatasetFormatInfo(storageProperties.getColumnMaxTupleCount(),
-storageProperties.getColumnFreeSpaceTolerance());
+DatasetFormatInfo datasetFormatInfo =
dd.getDatasetFormatInfo(storageProperties.getStorageFormat(),
+storageProperties.getColumnMaxTupleCount(),
storageProperties.getColumnFreeSpaceTolerance());
try {
// Check if the dataverse exists
Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx,
dataverseName);
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index 5b99fa0e61..073da971f7 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -63,7 +63,8 @@ public class StorageProperties extends AbstractProperties {
STORAGE_GLOBAL_CLEANUP(BOOLEAN, true),
STORAGE_GLOBAL_CLEANUP_TIMEOUT(POSITIVE_INTEGER, (int)
TimeUnit.MINUTES.toSeconds(10)),
STORAGE_COLUMN_MAX_TUPLE_COUNT(NONNEGATIVE_INTEGER, 15000),
-STORAGE_COLUMN_FREE_SPACE_TOLERANCE(DOUBLE, 0.15);
+STORAGE_COLUMN_FREE_SPACE_TOLERANCE(DOUBLE, 0.15),
+STORAGE_FORMAT(STRING, "row");
private final IOptionType interpreter;
private final Object defaultValue;
@@ -136,6 +137,8 @@ public class StorageProperties extends AbstractProperties {
case STORAGE_COLUMN_FREE_SPACE_TOLERANCE:
return "The percentage of the maximum tolerable empty
space for a physical mega leaf page (e.g.,"
+ " 0.15 means a physical page with 15% or less
empty space is tolerable)";
+case STORAGE_FORMAT:
+return "The default storage format (either row or column)";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -280,4 +283,8 @@ public class StorageProperties extends AbstractProperties {
public float getColumnFreeSpaceTolerance() {
return (float)
accessor.getDouble(Option.STORAGE_COLUMN_FREE_SPACE_TOLERANCE);
}
+
+public String getStorageFormat() {
+return accessor.getString(Option.STORAGE_FORMA
[asterixdb] branch master updated: [NO ISSUE][OTH] Additional external indexing clean up
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new d9764ab1dc [NO ISSUE][OTH] Additional external indexing clean up d9764ab1dc is described below commit d9764ab1dc0d3f02e29aef257f5506c7e0afb297 Author: Murtadha Hubail AuthorDate: Sat Mar 25 20:03:10 2023 +0300 [NO ISSUE][OTH] Additional external indexing clean up - user model changes: yes - storage format changes: no - interface changes: yes Details: - Remove REFRESH statement. - More external indexing runtime removal. Change-Id: Ide588c3933979edae763b810dfa6f8a34116945f Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17449 Integration-Tests: Jenkins Tested-by: Jenkins Reviewed-by: Murtadha Al Hubail Reviewed-by: Ali Alsuliman --- .../asterix/app/translator/QueryTranslator.java| 88 - .../adapter/factory/GenericAdapterFactory.java | 22 +--- .../external/api/IExternalDataSourceFactory.java | 9 -- .../asterix/external/api/IExternalIndexer.java | 56 .../external/api/IIndexibleExternalDataSource.java | 34 - .../external/api/IIndexingAdapterFactory.java | 27 .../asterix/external/api/IIndexingDatasource.java | 50 .../dataflow/IndexingDataFlowController.java | 47 --- .../external/input/HDFSDataSourceFactory.java | 69 +++--- .../record/reader/IndexingStreamRecordReader.java | 101 --- .../AbstractExternalInputStreamFactory.java| 5 - .../input/record/reader/hdfs/HDFSRecordReader.java | 53 +--- .../record/reader/rss/RSSRecordReaderFactory.java | 5 - .../reader/twitter/TwitterRecordReaderFactory.java | 5 - .../external/input/stream/HDFSInputStream.java | 64 +- .../stream/factory/LocalFSInputStreamFactory.java | 5 - .../factory/SocketServerInputStreamFactory.java| 5 - .../factory/TwitterFirehoseStreamFactory.java | 5 - .../provider/DataflowControllerProvider.java | 9 +- .../apache/asterix/lang/common/base/Statement.java | 1 - .../statement/RefreshExternalDatasetStatement.java | 64 -- .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj| 19 --- .../declared/BTreeResourceFactoryProvider.java | 6 +- .../metadata/declared/MetadataProvider.java| 27 +--- .../metadata/lock/ExternalDatasetsRegistry.java| 142 - .../apache/asterix/metadata/utils/DatasetUtil.java | 16 --- .../utils/ExternalDatasetAccessManager.java| 122 -- .../apache/asterix/metadata/utils/IndexUtil.java | 3 - .../utils/SecondaryIndexOperationsHelper.java | 6 - 29 files changed, 30 insertions(+), 1035 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 399b7b4e1a..15a8238a6b 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -94,7 +94,6 @@ import org.apache.asterix.common.utils.JobUtils.ProgressState; import org.apache.asterix.common.utils.StorageConstants; import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.external.dataset.adapter.AdapterIdentifier; -import org.apache.asterix.external.indexing.IndexingConstants; import org.apache.asterix.external.operators.FeedIntakeOperatorNodePushable; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.ExternalDataUtils; @@ -187,7 +186,6 @@ import org.apache.asterix.metadata.entities.Synonym; import org.apache.asterix.metadata.entities.ViewDetails; import org.apache.asterix.metadata.feeds.FeedMetadataUtil; import org.apache.asterix.metadata.functions.ExternalFunctionCompilerUtil; -import org.apache.asterix.metadata.lock.ExternalDatasetsRegistry; import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.metadata.utils.IndexUtil; import org.apache.asterix.metadata.utils.KeyFieldTypeUtil; @@ -499,9 +497,6 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen case COMPACT: handleCompactStatement(metadataProvider, stmt, hcc); break; -case EXTERNAL_DATASET_REFRESH: - handleExternalDatasetRefreshStatement(metadataProvider, stmt, hcc); -break; case FUNCTION_DECL: handleDeclareFunctionStatement(metadataProvider, stmt); break; @@ -1549,9 +1544,6
[asterixdb] branch master updated: [ASTERIXDB-3140][OTH] Remove references to Write statement
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9662b805dd [ASTERIXDB-3140][OTH] Remove references to Write statement
9662b805dd is described below
commit 9662b805dde89603ec96e887ba09dd50da0ad530
Author: Murtadha Hubail
AuthorDate: Wed Mar 15 20:17:17 2023 +0300
[ASTERIXDB-3140][OTH] Remove references to Write statement
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Remove all references to the deprecated Write statement.
Change-Id: I262f59439e483760f37c6c4300dc2f937cd969ed
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17429
Integration-Tests: Jenkins
Reviewed-by: Ian Maxon
Tested-by: Jenkins
---
.../asterix/app/translator/QueryTranslator.java| 20 ---
.../apache/asterix/lang/common/base/Statement.java | 1 -
.../lang/common/statement/WriteStatement.java | 66 --
.../lang/common/visitor/FormatPrintVisitor.java| 11
.../lang/common/visitor/QueryPrintVisitor.java | 11
.../base/AbstractQueryExpressionVisitor.java | 6 --
.../lang/common/visitor/base/ILangVisitor.java | 3 -
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj| 20 ---
8 files changed, 138 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 8e61c50396..b0e58852f7 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -23,7 +23,6 @@ import static
org.apache.asterix.common.utils.IdentifierUtil.dataset;
import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
import static
org.apache.asterix.lang.common.statement.CreateFullTextFilterStatement.FIELD_TYPE_STOPWORDS;
-import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.rmi.RemoteException;
@@ -157,7 +156,6 @@ import org.apache.asterix.lang.common.statement.TypeDecl;
import org.apache.asterix.lang.common.statement.TypeDropStatement;
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.statement.ViewDropStatement;
-import org.apache.asterix.lang.common.statement.WriteStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.VarIdentifier;
import org.apache.asterix.lang.common.util.FunctionUtil;
@@ -244,7 +242,6 @@ import
org.apache.hyracks.algebricks.core.algebra.base.Counter;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import org.apache.hyracks.algebricks.data.IAWriterFactory;
import org.apache.hyracks.api.client.IClusterInfoCollector;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -252,7 +249,6 @@ import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.api.io.FileSplit;
-import org.apache.hyracks.api.io.UnmanagedFileSplit;
import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
@@ -510,9 +506,6 @@ public class QueryTranslator extends AbstractLangTranslator
implements IStatemen
case EXTERNAL_DATASET_REFRESH:
handleExternalDatasetRefreshStatement(metadataProvider, stmt, hcc);
break;
-case WRITE:
-//Deprecated.
-break;
case FUNCTION_DECL:
handleDeclareFunctionStatement(metadataProvider, stmt);
break;
@@ -575,18 +568,6 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
varCounter);
}
-protected Pair handleWriteStatement(Statement
stmt)
-throws InstantiationException, IllegalAccessException,
ClassNotFoundException {
-WriteStatement ws = (WriteStatement) stmt;
-File f = new File(ws.getFileName());
-FileSplit outputFile = new
UnmanagedFileSplit(ws.getNcName().getValue(), f.getPath());
-IAWriterFactory writerFactory = null;
-if (ws.getWriterClassName() != null
[asterixdb] 01/02: Advance versions to 0.9.9-SNAPSHOT and 0.3.9-SNAPSHOT
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit 4f3e48f97177840dad1d07fe5e7d429a2a29d5de Author: Ian Maxon AuthorDate: Tue Mar 14 20:12:58 2023 -0700 Advance versions to 0.9.9-SNAPSHOT and 0.3.9-SNAPSHOT Change-Id: I7caff25b24346ca8eca5bca7e0fe8fb1da72f271 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17426 Integration-Tests: Jenkins Tested-by: Jenkins Reviewed-by: Ian Maxon --- asterixdb/asterix-active/pom.xml| 2 +- asterixdb/asterix-algebra/pom.xml | 2 +- asterixdb/asterix-app/pom.xml | 2 +- asterixdb/asterix-benchmark/pom.xml | 2 +- asterixdb/asterix-client-helper/pom.xml | 2 +- asterixdb/asterix-common/pom.xml| 2 +- asterixdb/asterix-coverage/pom.xml | 2 +- asterixdb/asterix-dashboard/pom.xml | 2 +- asterixdb/asterix-doc/pom.xml | 2 +- asterixdb/asterix-docker/pom.xml| 2 +- asterixdb/asterix-examples/pom.xml | 2 +- asterixdb/asterix-external-data/pom.xml | 2 +- asterixdb/asterix-fuzzyjoin/pom.xml | 2 +- asterixdb/asterix-geo/pom.xml | 2 +- asterixdb/asterix-lang-common/pom.xml | 2 +- asterixdb/asterix-lang-sqlpp/pom.xml| 2 +- asterixdb/asterix-license/pom.xml | 2 +- .../asterix-grammar-extension-maven-plugin/pom.xml | 2 +- .../asterix-test-datagenerator-maven-plugin/pom.xml | 2 +- .../asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml | 2 +- asterixdb/asterix-maven-plugins/pom.xml | 2 +- .../record-manager-generator-maven-plugin/pom.xml | 2 +- asterixdb/asterix-metadata/pom.xml | 2 +- asterixdb/asterix-om/pom.xml| 2 +- asterixdb/asterix-replication/pom.xml | 2 +- asterixdb/asterix-runtime/pom.xml | 2 +- asterixdb/asterix-server/pom.xml| 2 +- asterixdb/asterix-spidersilk/pom.xml| 2 +- asterixdb/asterix-test-framework/pom.xml| 2 +- asterixdb/asterix-tools/pom.xml | 2 +- asterixdb/asterix-transactions/pom.xml | 2 +- asterixdb/pom.xml | 6 +++--- hyracks-fullstack/algebricks/algebricks-common/pom.xml | 2 +- hyracks-fullstack/algebricks/algebricks-compiler/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-core/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-data/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-runtime/pom.xml | 2 +- hyracks-fullstack/algebricks/algebricks-tests/pom.xml | 2 +- hyracks-fullstack/algebricks/pom.xml| 2 +- hyracks-fullstack/hyracks-fullstack-license/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-api/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-client/pom.xml| 2 +- hyracks-fullstack/hyracks/hyracks-comm/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-control-cc/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-control-common/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-control-nc/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-nc-service/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-control/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-data/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-dataflow-common/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-dist/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-documentation/pom.xml | 2 +- .../hyracks/hyracks-examples/btree-example/btreeclient/pom.xml | 2 +- .../hyracks/hyracks-examples/btree-example/btreehelper/pom.xml | 2 +- .../hyracks/hyracks-examples/btree-example/btreeserver/pom.xml | 2 +- hyracks-fullstack
[asterixdb] branch master updated (37e3f5cc0a -> 0428c81fc3)
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git from 37e3f5cc0a [ASTERIXDB-3137][STO] Introduce LSM write operations for columnar format add 67915ce761 Release Apache AsterixDB 0.9.8 and Hyracks 0.3.8 new 4f3e48f971 Advance versions to 0.9.9-SNAPSHOT and 0.3.9-SNAPSHOT new 0428c81fc3 Merge branch 'release-0.9.8' The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: asterixdb/asterix-active/pom.xml| 2 +- asterixdb/asterix-algebra/pom.xml | 2 +- asterixdb/asterix-app/pom.xml | 2 +- asterixdb/asterix-benchmark/pom.xml | 2 +- asterixdb/asterix-client-helper/pom.xml | 2 +- asterixdb/asterix-column/pom.xml| 2 +- asterixdb/asterix-common/pom.xml| 2 +- asterixdb/asterix-coverage/pom.xml | 2 +- asterixdb/asterix-dashboard/pom.xml | 2 +- asterixdb/asterix-doc/pom.xml | 2 +- asterixdb/asterix-examples/pom.xml | 2 +- asterixdb/asterix-external-data/pom.xml | 2 +- asterixdb/asterix-fuzzyjoin/pom.xml | 2 +- asterixdb/asterix-geo/pom.xml | 2 +- asterixdb/asterix-lang-common/pom.xml | 2 +- asterixdb/asterix-lang-sqlpp/pom.xml| 2 +- asterixdb/asterix-license/pom.xml | 2 +- .../asterix-grammar-extension-maven-plugin/pom.xml | 2 +- .../asterix-test-datagenerator-maven-plugin/pom.xml | 2 +- .../asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml | 2 +- asterixdb/asterix-maven-plugins/pom.xml | 2 +- .../record-manager-generator-maven-plugin/pom.xml | 2 +- asterixdb/asterix-metadata/pom.xml | 2 +- asterixdb/asterix-om/pom.xml| 2 +- asterixdb/asterix-podman/pom.xml| 2 +- asterixdb/asterix-replication/pom.xml | 2 +- asterixdb/asterix-runtime/pom.xml | 2 +- asterixdb/asterix-server/pom.xml| 2 +- asterixdb/asterix-spidersilk/pom.xml| 2 +- asterixdb/asterix-test-framework/pom.xml| 2 +- asterixdb/asterix-tools/pom.xml | 2 +- asterixdb/asterix-transactions/pom.xml | 2 +- asterixdb/pom.xml | 6 +++--- hyracks-fullstack/algebricks/algebricks-common/pom.xml | 2 +- hyracks-fullstack/algebricks/algebricks-compiler/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-core/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-data/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-runtime/pom.xml | 2 +- hyracks-fullstack/algebricks/algebricks-tests/pom.xml | 2 +- hyracks-fullstack/algebricks/pom.xml| 2 +- hyracks-fullstack/hyracks-fullstack-license/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-api/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-client/pom.xml| 2 +- hyracks-fullstack/hyracks/hyracks-comm/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-control-cc/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-control-common/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-control-nc/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-nc-service/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-control/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-data/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-dataflow-common/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-dist/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-documentation/pom.xml | 2 +-
[asterixdb] 02/02: Merge branch 'release-0.9.8'
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit 0428c81fc30dc4c9350017a376bf91997024a18b Merge: 37e3f5cc0a 4f3e48f971 Author: Ian Maxon AuthorDate: Wed Mar 15 08:57:31 2023 -0700 Merge branch 'release-0.9.8' Change-Id: I511044b28d6a0f2f2d2639ea1432281f4b2a5c21 asterixdb/asterix-active/pom.xml| 2 +- asterixdb/asterix-algebra/pom.xml | 2 +- asterixdb/asterix-app/pom.xml | 2 +- asterixdb/asterix-benchmark/pom.xml | 2 +- asterixdb/asterix-client-helper/pom.xml | 2 +- asterixdb/asterix-column/pom.xml| 2 +- asterixdb/asterix-common/pom.xml| 2 +- asterixdb/asterix-coverage/pom.xml | 2 +- asterixdb/asterix-dashboard/pom.xml | 2 +- asterixdb/asterix-doc/pom.xml | 2 +- asterixdb/asterix-examples/pom.xml | 2 +- asterixdb/asterix-external-data/pom.xml | 2 +- asterixdb/asterix-fuzzyjoin/pom.xml | 2 +- asterixdb/asterix-geo/pom.xml | 2 +- asterixdb/asterix-lang-common/pom.xml | 2 +- asterixdb/asterix-lang-sqlpp/pom.xml| 2 +- asterixdb/asterix-license/pom.xml | 2 +- .../asterix-grammar-extension-maven-plugin/pom.xml | 2 +- .../asterix-test-datagenerator-maven-plugin/pom.xml | 2 +- .../asterix-maven-plugins/lexer-generator-maven-plugin/pom.xml | 2 +- asterixdb/asterix-maven-plugins/pom.xml | 2 +- .../record-manager-generator-maven-plugin/pom.xml | 2 +- asterixdb/asterix-metadata/pom.xml | 2 +- asterixdb/asterix-om/pom.xml| 2 +- asterixdb/asterix-podman/pom.xml| 2 +- asterixdb/asterix-replication/pom.xml | 2 +- asterixdb/asterix-runtime/pom.xml | 2 +- asterixdb/asterix-server/pom.xml| 2 +- asterixdb/asterix-spidersilk/pom.xml| 2 +- asterixdb/asterix-test-framework/pom.xml| 2 +- asterixdb/asterix-tools/pom.xml | 2 +- asterixdb/asterix-transactions/pom.xml | 2 +- asterixdb/pom.xml | 6 +++--- hyracks-fullstack/algebricks/algebricks-common/pom.xml | 2 +- hyracks-fullstack/algebricks/algebricks-compiler/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-core/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-data/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml| 2 +- hyracks-fullstack/algebricks/algebricks-runtime/pom.xml | 2 +- hyracks-fullstack/algebricks/algebricks-tests/pom.xml | 2 +- hyracks-fullstack/algebricks/pom.xml| 2 +- hyracks-fullstack/hyracks-fullstack-license/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-api/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-client/pom.xml| 2 +- hyracks-fullstack/hyracks/hyracks-comm/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-control-cc/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-control-common/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-control-nc/pom.xml | 2 +- .../hyracks/hyracks-control/hyracks-nc-service/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-control/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-data/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-dataflow-common/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-dist/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-documentation/pom.xml | 2 +- .../hyracks/hyracks-examples/btree-example/btreeclient/pom.xml | 2 +- .../hyracks/hyracks-examples/btree-example/btreehelper/pom.xml | 2 +- .../hyracks/hyracks-examples/btree-example/btreeserver/pom.xml | 2 +- hyracks-fullstack/hyracks/hyracks-examples/btree-example/pom.xml| 2 +- .../hyracks/hyracks
[asterixdb] 05/08: [ASTERIXDB-3092][COMP] Consider only field-access functions in LoadRecordFieldsRule
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit cac1e4ab32b24ae35c657c72d9979f41029c45d7
Author: Wail Alkowaileet
AuthorDate: Fri Dec 9 12:03:40 2022 -0800
[ASTERIXDB-3092][COMP] Consider only field-access functions in
LoadRecordFieldsRule
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
The compiler rule LoadRecordFieldsRule fails when it encounters
a non-field-access function. This patch fixes this issue by ensuring
that only field accesses are considered for this rule.
Change-Id: I88f72fd51716dd8152e709c841489e87af0a5137
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17298
Tested-by: Jenkins
Reviewed-by: Wail Alkowaileet
Reviewed-by: Ali Alsuliman
---
.../optimizer/rules/LoadRecordFieldsRule.java | 14 +-
.../queries_sqlpp/objects/ObjectsQueries.xml | 5
.../load-record-fields.1.ddl.sqlpp | 30 ++
.../load-record-fields.2.update.sqlpp | 25 ++
.../load-record-fields.3.query.sqlpp | 30 ++
.../load-record-fields.4.query.sqlpp | 30 ++
.../load-record-fields/load-record-fields.3.adm| 2 ++
.../load-record-fields/load-record-fields.4.plan | 26 +++
.../asterix/lang/common/util/FunctionUtil.java | 13 ++
9 files changed, 168 insertions(+), 7 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
index b9d512bb8e..42cce5233e 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
@@ -27,6 +27,7 @@ import java.util.List;
import org.apache.asterix.algebra.base.OperatorAnnotation;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.constants.AsterixConstantValue;
@@ -64,7 +65,7 @@ import
org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
-private ExtractFieldLoadExpressionVisitor exprVisitor = new
ExtractFieldLoadExpressionVisitor();
+private final ExtractFieldLoadExpressionVisitor exprVisitor = new
ExtractFieldLoadExpressionVisitor();
@Override
public boolean rewritePre(Mutable opRef,
IOptimizationContext context) {
@@ -98,13 +99,13 @@ public class LoadRecordFieldsRule implements
IAlgebraicRewriteRule {
// checking if we can annotate a Selection as using just one field
// access
SelectOperator sigma = (SelectOperator) op1;
-LinkedList vars = new
LinkedList();
+List vars = new ArrayList<>();
VariableUtilities.getUsedVariables(sigma, vars);
if (vars.size() == 1) {
// we can annotate Selection
AssignOperator assign1 = (AssignOperator)
op1.getInputs().get(0).getValue();
-AbstractLogicalExpression expr1 = (AbstractLogicalExpression)
getFirstExpr(assign1);
-if (expr1.getExpressionTag() ==
LogicalExpressionTag.FUNCTION_CALL) {
+ILogicalExpression expr1 = getFirstExpr(assign1);
+if (FunctionUtil.isFieldAccessFunction(expr1)) {
AbstractFunctionCallExpression f =
(AbstractFunctionCallExpression) expr1;
// f should be a call to a field/data access kind of
// function
@@ -141,7 +142,7 @@ public class LoadRecordFieldsRule implements
IAlgebraicRewriteRule {
}
// create an assign
LogicalVariable v = context.newVar();
-AssignOperator a2 = new AssignOperator(v, new
MutableObject(f));
+AssignOperator a2 = new AssignOperator(v, new
MutableObject<>(f));
a2.setSourceLocation(expr.getSourceLocation());
pushFieldAssign(a2, topOp, context);
context.computeAndSetTypeEnvironmentForOperator(a2);
@@ -151,7 +152,7 @@ public class LoadRecordFieldsRule implements
IAlgebraicRewriteRule {
LogicalVariable var = ref.getVariableReference();
List keys =
context.findPrimaryKey(var);
if (
[asterixdb] 01/08: [NO ISSUE] Process shadowed projects w/ license plugin
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d7e54cf14441d895ef4fd26e228d9857a2439d2e
Author: Michael Blow
AuthorDate: Sat Nov 19 18:59:28 2022 -0500
[NO ISSUE] Process shadowed projects w/ license plugin
Change-Id: Id21dcd3dc61bd4766e71c5d509e7ef6802e423be
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17290
Reviewed-by: Michael Blow
Reviewed-by: Murtadha Hubail
Tested-by: Michael Blow
Integration-Tests: Jenkins
---
.../main/licenses/templates/asterix-license.ftl| 3 +
.../src/main/licenses/templates/asterix-notice.ftl | 3 +
.../src/main/licenses/templates/hyracks-notice.ftl | 3 +
.../maven/license/DownloadLicensesMojo.java| 2 +-
.../hyracks/maven/license/GenerateFileMojo.java| 120 ++---
.../apache/hyracks/maven/license/LicenseMojo.java | 47 +---
.../hyracks/maven/license/project/Project.java | 12 ++-
7 files changed, 159 insertions(+), 31 deletions(-)
diff --git a/asterixdb/src/main/licenses/templates/asterix-license.ftl
b/asterixdb/src/main/licenses/templates/asterix-license.ftl
index 381d400a90..434ce447f6 100644
--- a/asterixdb/src/main/licenses/templates/asterix-license.ftl
+++ b/asterixdb/src/main/licenses/templates/asterix-license.ftl
@@ -44,6 +44,9 @@ ${license.content}
<#assign isare = "are"/>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+
* ${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git a/asterixdb/src/main/licenses/templates/asterix-notice.ftl
b/asterixdb/src/main/licenses/templates/asterix-notice.ftl
index c825397c1a..5090cb0868 100644
--- a/asterixdb/src/main/licenses/templates/asterix-notice.ftl
+++ b/asterixdb/src/main/licenses/templates/asterix-notice.ftl
@@ -34,6 +34,9 @@ AsterixDB utilizes many libraries, which come with the
following applicable NOTI
<#assign projects = e.getValue()/>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+
${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git
a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
index 8a2eb05d29..0cf9378659 100644
---
a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
+++
b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
@@ -30,6 +30,9 @@ Hyracks and Algebricks utilize many libraries, which come
with the following app
<#assign noticeText = e.getKey()/>
<#assign projects = e.getValue()/>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+
${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git
a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
index 293c08c6cf..c8980cd3da 100644
---
a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
+++
b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
@@ -63,7 +63,7 @@ public class DownloadLicensesMojo extends LicenseMojo {
String fileName = entry.getLicense().getContentFile(false);
doDownload(timeoutMillis, i, url, fileName);
});
-} catch (ProjectBuildingException e) {
+} catch (ProjectBuildingException | IOException e) {
throw new MojoExecutionException("Unexpected exception: " + e, e);
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index e69887c289..b3a0bb299c 100644
---
a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++
b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@ -39,6 +39,7 @@ import java.u
[asterixdb] 06/08: [NO ISSUE][OTH] Log node and node address on negative heartbeat ack
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2538c159d3e1c67b1c3cab15e626d2d4fcd7b79d
Author: Murtadha Hubail
AuthorDate: Wed Jan 4 20:14:08 2023 +0300
[NO ISSUE][OTH] Log node and node address on negative heartbeat ack
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When sending a negative heartbeat ack to an unregistered node,
log the node id as well as its address.
Change-Id: I001b98ef1faa953d92332d36c2d4985b34ed57dd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17305
Reviewed-by: Ali Alsuliman
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java | 5 +
1 file changed, 5 insertions(+)
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java
index b772ef93da..6fa6224f3d 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java
@@ -28,9 +28,12 @@ import org.apache.hyracks.control.cc.cluster.INodeManager;
import org.apache.hyracks.control.common.heartbeat.HeartbeatData;
import org.apache.hyracks.control.common.ipc.NodeControllerRemoteProxy;
import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class NodeHeartbeatWork extends AbstractHeartbeatWork {
+private static final Logger LOGGER = LogManager.getLogger();
private final InetSocketAddress ncAddress;
public NodeHeartbeatWork(ClusterControllerService ccs, String nodeId,
HeartbeatData hbData,
@@ -47,6 +50,8 @@ public class NodeHeartbeatWork extends AbstractHeartbeatWork {
ncState.getNodeController().heartbeatAck(ccs.getCcId(), null);
} else {
// unregistered nc- let him know
+LOGGER.info("received a heartbeat from unregistered node {};
sending negative ack to node address {}",
+nodeId, ncAddress);
NodeControllerRemoteProxy nc =
new NodeControllerRemoteProxy(ccs.getCcId(),
ccs.getClusterIPC().getReconnectingHandle(ncAddress));
nc.heartbeatAck(ccs.getCcId(),
HyracksDataException.create(ErrorCode.NO_SUCH_NODE, nodeId));
[asterixdb] 02/08: Merge branch 'gerrit/mad-hatter'
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 023e28c15d6d856419d7c595105818a76d3e3511
Merge: 4af9e73bb6 d7e54cf144
Author: Michael Blow
AuthorDate: Sun Nov 20 21:41:37 2022 -0500
Merge branch 'gerrit/mad-hatter'
Change-Id: I300a5e2910bde9e7bcd5f187efca26e4c7689ffb
.../main/licenses/templates/asterix-license.ftl| 3 +
.../src/main/licenses/templates/asterix-notice.ftl | 3 +
.../src/main/licenses/templates/hyracks-notice.ftl | 3 +
.../maven/license/DownloadLicensesMojo.java| 2 +-
.../hyracks/maven/license/GenerateFileMojo.java| 120 ++---
.../apache/hyracks/maven/license/LicenseMojo.java | 47 +---
.../hyracks/maven/license/project/Project.java | 12 ++-
7 files changed, 159 insertions(+), 31 deletions(-)
diff --cc
hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index 6bc92b8539,b3a0bb299c..6d30a691fb
---
a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++
b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@@ -335,19 -357,17 +357,19 @@@ public class GenerateFileMojo extends L
}
private void resolveNoticeFiles() throws MojoExecutionException,
IOException {
-// TODO(mblow): this will match *any* NOTICE[.txt] file located
within the artifact- this seems way too liberal
+// TODO(mblow): this will match *any* NOTICE[.(txt|md)] file located
within the artifact-
+// this seems way too liberal
resolveArtifactFiles("NOTICE", IGNORE_MISSING_EMBEDDED_NOTICE,
ALTERNATE_NOTICE_FILE,
-entry -> entry.getName().matches("(.*/|^)" + "NOTICE" +
"(.txt)?"), Project::setNoticeText,
+entry -> entry.getName().matches("(.*/|^)" + "NOTICE" +
"(.(txt|md))?"), Project::setNoticeText,
- text -> stripFoundationAssertionFromNotices ?
FOUNDATION_PATTERN.matcher(text).replaceAll("") : text);
+ getNoticeFileContentTransformer(), !validateShadowLicenses);
}
private void resolveLicenseFiles() throws MojoExecutionException,
IOException {
-// TODO(mblow): this will match *any* LICENSE[.txt] file located
within the artifact- this seems way too liberal
+// TODO(mblow): this will match *any* LICENSE[.(txt|md)] file located
within the artifact-
+// this seems way too liberal
resolveArtifactFiles("LICENSE", IGNORE_MISSING_EMBEDDED_LICENSE,
ALTERNATE_LICENSE_FILE,
-entry -> entry.getName().matches("(.*/|^)" + "LICENSE" +
"(.txt)?"), Project::setLicenseText,
+entry -> entry.getName().matches("(.*/|^)" + "LICENSE" +
"(.(txt|md))?"), Project::setLicenseText,
- UnaryOperator.identity());
+ UnaryOperator.identity(), !validateShadowLicenses);
}
private void resolveArtifactFiles(final String name, final ProjectFlag
ignoreFlag,
[asterixdb] 03/08: Merge branch 'gerrit/cheshire-cat'
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit faae0fb4a91754aea4c1cc3fe0304ea8158d68d4
Merge: b69c90dc17 023e28c15d
Author: Michael Blow
AuthorDate: Sun Nov 20 23:11:52 2022 -0500
Merge branch 'gerrit/cheshire-cat'
Change-Id: Id677655399af06ca4f1703a64482daf281a8c482
.../main/licenses/templates/asterix-license.ftl| 3 +
.../src/main/licenses/templates/asterix-notice.ftl | 3 +
.../src/main/licenses/templates/hyracks-notice.ftl | 3 +
.../maven/license/DownloadLicensesMojo.java| 2 +-
.../hyracks/maven/license/GenerateFileMojo.java| 117 ++---
.../apache/hyracks/maven/license/LicenseMojo.java | 47 ++---
.../hyracks/maven/license/project/Project.java | 12 ++-
7 files changed, 157 insertions(+), 30 deletions(-)
diff --cc
hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index 347c19ad3c,6d30a691fb..8edc1b212f
---
a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++
b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@@ -348,45 -365,17 +370,44 @@@ public class GenerateFileMojo extends L
}
private void resolveLicenseFiles() throws MojoExecutionException,
IOException {
-// TODO(mblow): this will match *any* LICENSE[.(txt|md)] file located
within the artifact-
-// this seems way too liberal
-resolveArtifactFiles("LICENSE", IGNORE_MISSING_EMBEDDED_LICENSE,
ALTERNATE_LICENSE_FILE,
-entry -> entry.getName().matches("(.*/|^)" + "LICENSE" +
"(.(txt|md))?"), Project::setLicenseText,
-UnaryOperator.identity(), !validateShadowLicenses);
+resolveArtifactFiles(LICENSE);
}
-private void resolveArtifactFiles(final String name, final ProjectFlag
ignoreFlag,
-final ProjectFlag alternateFilenameFlag, final
Predicate filter,
-final BiConsumer consumer, final
UnaryOperator contentTransformer,
-boolean skipShadowed) throws MojoExecutionException, IOException {
+private void resolveArtifactFiles(final EmbeddedArtifact artifact) throws
MojoExecutionException, IOException {
+final String name;
+final ProjectFlag ignoreFlag;
+final ProjectFlag alternateFilenameFlag;
+final ProjectFlag onMultipleFlag;
+final Predicate filter;
+final BiConsumer consumer;
+final UnaryOperator contentTransformer;
+
+switch (artifact) {
+case NOTICE:
+name = "NOTICE";
+ignoreFlag = IGNORE_MISSING_EMBEDDED_NOTICE;
+alternateFilenameFlag = ALTERNATE_NOTICE_FILE;
+onMultipleFlag = ON_MULTIPLE_EMBEDDED_NOTICE;
+// TODO(mblow): this will match *any* NOTICE[.(txt|md)] file
located within the artifact-
+// this seems way too liberal
+filter = entry -> entry.getName().matches("(.*/|^)" +
"NOTICE" + "(.(txt|md))?");
+consumer = Project::setNoticeText;
- contentTransformer = UnaryOperator.identity();
++contentTransformer = getNoticeFileContentTransformer();
+break;
+case LICENSE:
+name = "LICENSE";
+ignoreFlag = IGNORE_MISSING_EMBEDDED_LICENSE;
+alternateFilenameFlag = ALTERNATE_LICENSE_FILE;
+onMultipleFlag = ON_MULTIPLE_EMBEDDED_LICENSE;
+// TODO(mblow): this will match *any* LICENSE[.(txt|md)] file
located within the artifact-
+// this seems way too liberal
+filter = entry -> entry.getName().matches("(.*/|^)" +
"LICENSE" + "(.(txt|md))?");
+consumer = Project::setLicenseText;
- contentTransformer = stripFoundationAssertionFromNotices
- ? text ->
FOUNDATION_PATTERN.matcher(text).replaceAll("") : UnaryOperator.identity();
++contentTransformer = UnaryOperator.identity();
+break;
+default:
+throw new IllegalStateException("NYI: " + artifact);
+}
for (Project p : getProjects()) {
File artifactFile = new File(p.getArtifactPath());
if (!artifactFile.exists()) {
@@@ -394,6 -383,9 +415,9 @@@
} else if (!artifactFile.getName().endsWith(".jar")) {
getLog().info(&qu
[asterixdb] 04/08: [NO ISSUE] Exclude licenses that are exclusively shadowed
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 667a908755391fcf07957ca2c01d9f42cb89b291
Author: Michael Blow
AuthorDate: Fri Dec 9 15:56:28 2022 -0500
[NO ISSUE] Exclude licenses that are exclusively shadowed
Change-Id: I19ebe2977e85293fd7c2e67768a218062fe9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17302
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Michael Blow
---
asterixdb/src/main/licenses/templates/asterix-license.ftl | 8
.../apache/hyracks/maven/license/project/LicensedProjects.java| 6 ++
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/asterixdb/src/main/licenses/templates/asterix-license.ftl
b/asterixdb/src/main/licenses/templates/asterix-license.ftl
index 434ce447f6..f9c969e27c 100644
--- a/asterixdb/src/main/licenses/templates/asterix-license.ftl
+++ b/asterixdb/src/main/licenses/templates/asterix-license.ftl
@@ -34,8 +34,11 @@ ${license.content}
<#list licenseMap as e>
<#assign licenseUrl = e.getKey()/>
<#assign entry = e.getValue()/>
- <#assign projects = entry.projects/>
+ <#assign projects = entry.nonShadowedProjects/>
<#assign license = entry.getLicense()/>
+ <#if projects?size == 0>
+ <#continue/>
+
<#if projects?size == 1>
Component:
<#assign isare = "is"/>
@@ -44,9 +47,6 @@ ${license.content}
<#assign isare = "are"/>
<#list projects as p>
- <#if p.shadowed>
- <#continue/>
-
* ${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git
a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java
b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java
index f6f9f32444..31b4a86061 100644
---
a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java
+++
b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.maven.license.project;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.function.Predicate;
import org.apache.hyracks.maven.license.LicenseSpec;
@@ -51,6 +52,11 @@ public class LicensedProjects {
return projects;
}
+public SortedSet getNonShadowedProjects() {
+return projects.stream().filter(Predicate.not(Project::isShadowed))
+.collect(() -> new TreeSet<>(Project.PROJECT_COMPARATOR),
TreeSet::add, TreeSet::addAll);
+}
+
public void addProject(Project project) {
projects.add(project);
}
[asterixdb] 08/08: Merge "Merge branch 'neo'"
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit 6ff8eec85d2b433f654a1d8a84707d398dedced1 Merge: cbdf6fdb48 ab2fbf432a Author: Apache AsterixDB Gerrit AuthorDate: Thu Jan 12 15:33:29 2023 + Merge "Merge branch 'neo'" .../optimizer/rules/LoadRecordFieldsRule.java | 14 +-- .../queries_sqlpp/objects/ObjectsQueries.xml | 5 + .../load-record-fields.1.ddl.sqlpp | 30 + .../load-record-fields.2.update.sqlpp | 25 + .../load-record-fields.3.query.sqlpp | 30 + .../load-record-fields.4.query.sqlpp | 30 + .../load-record-fields/load-record-fields.3.adm| 2 + .../load-record-fields/load-record-fields.4.plan | 26 + .../asterix/lang/common/util/FunctionUtil.java | 13 +++ .../main/licenses/templates/asterix-license.ftl| 5 +- .../src/main/licenses/templates/asterix-notice.ftl | 3 + .../src/main/licenses/templates/hyracks-notice.ftl | 3 + .../hyracks/control/cc/work/NodeHeartbeatWork.java | 5 + .../maven/license/DownloadLicensesMojo.java| 2 +- .../hyracks/maven/license/GenerateFileMojo.java| 124 ++--- .../apache/hyracks/maven/license/LicenseMojo.java | 47 +--- .../maven/license/project/LicensedProjects.java| 6 + .../hyracks/maven/license/project/Project.java | 12 +- 18 files changed, 344 insertions(+), 38 deletions(-)
[asterixdb] 07/08: Merge branch 'neo'
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ab2fbf432a4a59b6171fb0d5e02cd59746d082fd
Merge: 9f04aefb57 2538c159d3
Author: Murtadha Hubail
AuthorDate: Tue Jan 10 21:09:06 2023 +0300
Merge branch 'neo'
Change-Id: I1334d543bc86fffa849dd38e60fdf98c706444ac
.../optimizer/rules/LoadRecordFieldsRule.java | 14 +--
.../queries_sqlpp/objects/ObjectsQueries.xml | 5 +
.../load-record-fields.1.ddl.sqlpp | 30 +
.../load-record-fields.2.update.sqlpp | 25 +
.../load-record-fields.3.query.sqlpp | 30 +
.../load-record-fields.4.query.sqlpp | 30 +
.../load-record-fields/load-record-fields.3.adm| 2 +
.../load-record-fields/load-record-fields.4.plan | 26 +
.../asterix/lang/common/util/FunctionUtil.java | 13 +++
.../main/licenses/templates/asterix-license.ftl| 5 +-
.../src/main/licenses/templates/asterix-notice.ftl | 3 +
.../src/main/licenses/templates/hyracks-notice.ftl | 3 +
.../hyracks/control/cc/work/NodeHeartbeatWork.java | 5 +
.../maven/license/DownloadLicensesMojo.java| 2 +-
.../hyracks/maven/license/GenerateFileMojo.java| 124 ++---
.../apache/hyracks/maven/license/LicenseMojo.java | 47 +---
.../maven/license/project/LicensedProjects.java| 6 +
.../hyracks/maven/license/project/Project.java | 12 +-
18 files changed, 344 insertions(+), 38 deletions(-)
diff --cc
hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index 347c19ad3c,8edc1b212f..2c7a23b0f5
---
a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++
b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@@ -110,6 -118,9 +118,11 @@@ public class GenerateFileMojo extends L
@Parameter
private boolean stripFoundationAssertionFromNotices = false;
++@Parameter
++private boolean includeShadowedDependencies = true;
+ @Parameter
+ private boolean validateShadowLicenses = false;
+
private SortedMap> noticeMap;
@java.lang.Override
@@@ -472,4 -496,71 +498,76 @@@
}
return matches;
}
+
+ private UnaryOperator getNoticeFileContentTransformer() {
+ UnaryOperator transformer;
+ if (stripFoundationAssertionFromNotices) {
+ transformer = text ->
FOUNDATION_PATTERN.matcher(text).replaceAll("");
+ } else {
+ transformer = UnaryOperator.identity();
+ }
+ return transformer;
+ }
+
+ @java.lang.Override
+ protected void gatherProjectDependencies(MavenProject project,
+ Map>>
dependencyLicenseMap,
+ Map dependencyGavMap) throws
ProjectBuildingException, MojoExecutionException {
+ super.gatherProjectDependencies(project, dependencyLicenseMap,
dependencyGavMap);
+ gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap);
+ }
+
+ @java.lang.Override
+ protected void processExtraDependencies(Map>> dependencyLicenseMap,
+ Map dependencyGavMap) throws
ProjectBuildingException, MojoExecutionException {
+ super.processExtraDependencies(dependencyLicenseMap,
dependencyGavMap);
+ gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap);
+ }
+
+ private void gatherShadowedDependencies(Map>> dependencyLicenseMap,
+ Map dependencyGavMap) throws
MojoExecutionException, ProjectBuildingException {
+ Set projects = new
TreeSet<>(Comparator.comparing(MavenProject::getId));
++if (!includeShadowedDependencies) {
++getLog().info("Not gathering shadowed dependencies as
'includeShadowedDependencies' is set to "
+++ includeShadowedDependencies);
++return;
++}
+ projects.addAll(dependencyLicenseMap.keySet());
+ for (MavenProject p : projects) {
+ boolean finished = false;
+ File artifactFile = p.getArtifact().getFile();
+ if (!artifactFile.exists()) {
+ throw new MojoExecutionException("Artifact file " +
artifactFile + " does not exist!");
+ } else if (!artifactFile.getName().endsWith(".jar")) {
+ getLog().info("Skipping unknown artifact file type: " +
artifactFile);
+ finished = true;
+ }
+ if (!finished) {
+ try (JarFile jarFile = new JarFile(artifactFile)) {
+ SortedMap matches =
gat
[asterixdb] branch master updated (cbdf6fdb48 -> 6ff8eec85d)
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git from cbdf6fdb48 [ASTERIXDB-3096][COMP] Array-index recognition bug w/ multiple subplans new d7e54cf144 [NO ISSUE] Process shadowed projects w/ license plugin new 023e28c15d Merge branch 'gerrit/mad-hatter' new faae0fb4a9 Merge branch 'gerrit/cheshire-cat' new 667a908755 [NO ISSUE] Exclude licenses that are exclusively shadowed new cac1e4ab32 [ASTERIXDB-3092][COMP] Consider only field-access functions in LoadRecordFieldsRule new 2538c159d3 [NO ISSUE][OTH] Log node and node address on negative heartbeat ack new ab2fbf432a Merge branch 'neo' new 6ff8eec85d Merge "Merge branch 'neo'" The 8 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../optimizer/rules/LoadRecordFieldsRule.java | 14 +-- .../queries_sqlpp/objects/ObjectsQueries.xml | 5 + .../load-record-fields.1.ddl.sqlpp | 30 + .../load-record-fields.2.update.sqlpp | 25 + .../load-record-fields.3.query.sqlpp | 30 + .../load-record-fields.4.query.sqlpp | 30 + .../load-record-fields/load-record-fields.3.adm| 2 + .../load-record-fields/load-record-fields.4.plan | 26 + .../asterix/lang/common/util/FunctionUtil.java | 13 +++ .../main/licenses/templates/asterix-license.ftl| 5 +- .../src/main/licenses/templates/asterix-notice.ftl | 3 + .../src/main/licenses/templates/hyracks-notice.ftl | 3 + .../hyracks/control/cc/work/NodeHeartbeatWork.java | 5 + .../maven/license/DownloadLicensesMojo.java| 2 +- .../hyracks/maven/license/GenerateFileMojo.java| 124 ++--- .../apache/hyracks/maven/license/LicenseMojo.java | 47 +--- .../maven/license/project/LicensedProjects.java| 6 + .../hyracks/maven/license/project/Project.java | 12 +- 18 files changed, 344 insertions(+), 38 deletions(-) create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.1.ddl.sqlpp create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.2.update.sqlpp create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.3.query.sqlpp create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.4.query.sqlpp create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.3.adm create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.4.plan
[asterixdb] branch master updated: [NO ISSUE][CONF] Make partitions related config visible
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new f65920c [NO ISSUE][CONF] Make partitions related config visible
f65920c is described below
commit f65920c86679783f074c5371c8a42bf85a9262fe
Author: Murtadha Hubail
AuthorDate: Mon Nov 22 18:18:36 2021 +0300
[NO ISSUE][CONF] Make partitions related config visible
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: Ia1c6a3a0a6e4ac94351c9ef314482e2f224276c9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14163
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Michael Blow
---
.../src/main/java/org/apache/asterix/common/config/NodeProperties.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java
index a03530d..afb103d 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java
@@ -96,7 +96,7 @@ public class NodeProperties extends AbstractProperties {
@Override
public boolean hidden() {
-return this == INITIAL_RUN || this == STARTING_PARTITION_ID ||
this == ACTIVE_PARTITIONS;
+return this == INITIAL_RUN;
}
}
[asterixdb] branch master updated: [NO ISSUE][CLUS] Interrupt global recovery on node failure
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new bd7208d [NO ISSUE][CLUS] Interrupt global recovery on node failure
bd7208d is described below
commit bd7208d093c95a1f4ff906ebae7086cc58e8
Author: Murtadha Hubail
AuthorDate: Wed Nov 10 22:29:34 2021 +0300
[NO ISSUE][CLUS] Interrupt global recovery on node failure
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When a node fails while global recovery is on-going, interrupt
recovery to avoid unnecessary waiting.
Change-Id: I58852e046ff4021f4c5d115f5c3488b249fc61a2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14025
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../asterix/hyracks/bootstrap/GlobalRecoveryManager.java| 13 -
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
index 9438b16..e6ef8df 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.asterix.app.message.StorageCleanupRequestMessage;
@@ -64,6 +65,7 @@ public class GlobalRecoveryManager implements
IGlobalRecoveryManager {
protected final IHyracksClientConnection hcc;
protected volatile boolean recoveryCompleted;
protected volatile boolean recovering;
+protected Future recoveryFuture;
public GlobalRecoveryManager(ICCServiceContext serviceCtx,
IHyracksClientConnection hcc,
IStorageComponentProvider componentProvider) {
@@ -98,7 +100,7 @@ public class GlobalRecoveryManager implements
IGlobalRecoveryManager {
* Perform recovery on a different thread to avoid
deadlocks in
* {@link
org.apache.asterix.common.cluster.IClusterStateManager}
*/
-serviceCtx.getControllerService().getExecutor().submit(()
-> {
+recoveryFuture =
serviceCtx.getControllerService().getExecutor().submit(() -> {
try {
recover(appCtx);
} catch (Throwable e) {
@@ -127,6 +129,9 @@ public class GlobalRecoveryManager implements
IGlobalRecoveryManager {
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
recoveryCompleted = true;
recovering = false;
+synchronized (this) {
+recoveryFuture = null;
+}
LOGGER.info("Global Recovery Completed. Refreshing cluster state...");
appCtx.getClusterStateManager().refreshState();
}
@@ -166,6 +171,12 @@ public class GlobalRecoveryManager implements
IGlobalRecoveryManager {
@Override
public void notifyStateChange(ClusterState newState) {
+synchronized (this) {
+if (recovering && newState == ClusterState.UNUSABLE &&
recoveryFuture != null) {
+// interrupt the recovery attempt since cluster became
unusable during global recovery
+recoveryFuture.cancel(true);
+}
+}
if (newState != ClusterState.ACTIVE && newState !=
ClusterState.RECOVERING) {
recoveryCompleted = false;
}
[asterixdb] branch master updated: [NO ISSUE][OTH] Replace usage of FileUtils.listFiles
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new f630626 [NO ISSUE][OTH] Replace usage of FileUtils.listFiles
f630626 is described below
commit f6306265ee7edb3b2cb4317216f9dcc9d6b4a7b1
Author: Murtadha Hubail
AuthorDate: Wed Nov 3 13:04:16 2021 +0300
[NO ISSUE][OTH] Replace usage of FileUtils.listFiles
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Avoid depending on FileUtils.listFiles in production code
by replacing it with our own implementation.
Change-Id: I1ecb6143d5840fc12b35a641e1bc73c40bba3676
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13923
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../PersistentLocalResourceRepository.java | 47 --
.../java/org/apache/hyracks/api/util/IoUtil.java | 37 +
2 files changed, 44 insertions(+), 40 deletions(-)
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 02f5772..6577367 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -60,7 +60,6 @@ import org.apache.asterix.common.storage.ResourceStorageStats;
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -96,41 +95,10 @@ public class PersistentLocalResourceRepository implements
ILocalResourceReposito
private static final FilenameFilter MASK_FILES_FILTER =
(dir, name) -> name.startsWith(StorageConstants.MASK_FILE_PREFIX);
private static final int MAX_CACHED_RESOURCES = 1000;
-private static final IOFileFilter METADATA_FILES_FILTER = new
IOFileFilter() {
-@Override
-public boolean accept(File file) {
-return file.getName().equals(StorageConstants.METADATA_FILE_NAME);
-}
-
-@Override
-public boolean accept(File dir, String name) {
-return false;
-}
-};
-
-private static final IOFileFilter METADATA_MASK_FILES_FILTER = new
IOFileFilter() {
-@Override
-public boolean accept(File file) {
-return file.getName().equals(METADATA_FILE_MASK_NAME);
-}
-
-@Override
-public boolean accept(File dir, String name) {
-return false;
-}
-};
-
-private static final IOFileFilter ALL_DIR_FILTER = new IOFileFilter() {
-@Override
-public boolean accept(File file) {
-return true;
-}
-
-@Override
-public boolean accept(File dir, String name) {
-return true;
-}
-};
+private static final FilenameFilter METADATA_FILES_FILTER =
+(dir, name) -> name.equals(StorageConstants.METADATA_FILE_NAME);
+private static final FilenameFilter METADATA_MASK_FILES_FILTER =
+(dir, name) -> name.equals(METADATA_FILE_MASK_NAME);
// Finals
private final IIOManager ioManager;
@@ -270,7 +238,7 @@ public class PersistentLocalResourceRepository implements
ILocalResourceReposito
if (!Files.exists(root) || !Files.isDirectory(root)) {
continue;
}
-final Collection files = FileUtils.listFiles(root.toFile(),
METADATA_FILES_FILTER, ALL_DIR_FILTER);
+final Collection files = IoUtil.getMatchingFiles(root,
METADATA_FILES_FILTER);
try {
for (File file : files) {
final LocalResource localResource =
readLocalResource(file);
@@ -301,7 +269,7 @@ public class PersistentLocalResourceRepository implements
ILocalResourceReposito
public synchronized void deleteInvalidIndexes(Predicate
filter) throws HyracksDataException {
for (Path root : storageRoots) {
-final Collection files = FileUtils.listFiles(root.toFile(),
METADATA_FILES_FILTER, ALL_DIR_FILTER);
+final Collection files = IoUtil.getMatchingFiles(root,
METADATA_FILES_FILTER);
try {
for (
[asterixdb] branch master updated: [NO ISSUE][OTH] Delete unused class IndexFileNameUtil
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new b33d053 [NO ISSUE][OTH] Delete unused class IndexFileNameUtil
b33d053 is described below
commit b33d0537d74b4ea3798085205c3024bf4c893059
Author: Murtadha Hubail
AuthorDate: Wed Oct 13 02:51:35 2021 +0300
[NO ISSUE][OTH] Delete unused class IndexFileNameUtil
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: Idfd96475de64008d6e687abaf86f541746742690
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13665
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../storage/am/common/util/IndexFileNameUtil.java | 44 --
1 file changed, 44 deletions(-)
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java
deleted file mode 100644
index 6f2850d..000
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.util;
-
-import java.io.File;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.FileSplit;
-import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
-
-public class IndexFileNameUtil {
-
-public static final String IO_DEVICE_NAME_PREFIX = "device_id_";
-
-@Deprecated
-public static String prepareFileName(String path, int ioDeviceId) {
-return path + File.separator + IO_DEVICE_NAME_PREFIX + ioDeviceId;
-}
-
-public static FileReference getIndexAbsoluteFileRef(IFileSplitProvider
fileSplitProvider, int partition,
-IIOManager ioManager) throws HyracksDataException {
-FileSplit split = fileSplitProvider.getFileSplits()[partition];
-return split.getFileReference(ioManager);
-}
-}
[asterixdb] branch master updated: [NO ISSUE][HYR][NET] += trust store to sec cfg, store serialization
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new bf30272 [NO ISSUE][HYR][NET] += trust store to sec cfg, store
serialization
bf30272 is described below
commit bf3027299fe18bddc412c162150394a569ee8cbe
Author: Michael Blow
AuthorDate: Tue Sep 28 21:47:29 2021 -0400
[NO ISSUE][HYR][NET] += trust store to sec cfg, store serialization
Change-Id: I358eb5b9b0f0f40b1588c12ed473e4e920e8fbbe
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13423
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Michael Blow
Reviewed-by: Ian Maxon
---
.../external/library/ExternalLibraryManager.java | 4 +-
.../api/network/INetworkSecurityConfig.java| 9 ++-
.../ipc/security/NetworkSecurityConfig.java| 70 --
.../ipc/security/NetworkSecurityManager.java | 5 +-
4 files changed, 79 insertions(+), 9 deletions(-)
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
index e54b729..f71150a 100755
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
@@ -639,7 +639,9 @@ public final class ExternalLibraryManager implements
ILibraryManager, ILifeCycle
final INetworkSecurityConfig configuration =
networkSecurityManager.getConfiguration();
KeyStore trustStore =
KeyStore.getInstance(KeyStore.getDefaultType());
try (FileInputStream trustStoreFile = new
FileInputStream(configuration.getTrustStoreFile())) {
-trustStore.load(trustStoreFile,
configuration.getKeyStorePassword().toCharArray());
+String ksPassword = configuration.getKeyStorePassword();
+trustStore.load(trustStoreFile,
+ksPassword == null || ksPassword.isEmpty() ? null
: ksPassword.toCharArray());
}
SSLContext sslcontext =
SSLContexts.custom().loadTrustMaterial(trustStore, null).build();
SSLConnectionSocketFactory sslsf = new
SSLConnectionSocketFactory(sslcontext,
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/network/INetworkSecurityConfig.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/network/INetworkSecurityConfig.java
index 32f784b..b483158 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/network/INetworkSecurityConfig.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/network/INetworkSecurityConfig.java
@@ -53,7 +53,14 @@ public interface INetworkSecurityConfig extends Serializable
{
String getKeyStorePassword();
/**
- * Gets a trust store file to be used for validating certificates of
secured connections.
+ * Gets the trust store to be used for validating certificates of secured
connections
+ *
+ * @return the trust store to be used
+ */
+KeyStore getTrustStore();
+
+/**
+ * Gets a trust store file to be used if {@link
INetworkSecurityConfig#getTrustStore()} returns null.
*
* @return the trust store file
*/
diff --git
a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/security/NetworkSecurityConfig.java
b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/security/NetworkSecurityConfig.java
index 25ea787..2170c15 100644
---
a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/security/NetworkSecurityConfig.java
+++
b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/security/NetworkSecurityConfig.java
@@ -19,55 +19,113 @@
package org.apache.hyracks.ipc.security;
import java.io.File;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
import org.apache.hyracks.api.network.INetworkSecurityConfig;
public class NetworkSecurityConfig implements INetworkSecurityConfig {
-private static final long serialVersionUID = -1914030130038989199L;
+private static final long serialVersionUID = 2L;
+private static final char[] INTEGRITY_PASSWORD =
NetworkSecurityConfig.class.getName().toCharArray();
private final boolean sslEnabled;
private final File keyStoreFile
[asterixdb] branch master updated: [NO ISSUE][CLUS] Add API to check for failed nodes
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 195ce0f [NO ISSUE][CLUS] Add API to check for failed nodes
195ce0f is described below
commit 195ce0f2f2f0004c1e863339c0d39e038db23fa4
Author: Murtadha Hubail
AuthorDate: Fri Sep 17 21:20:36 2021 +0300
[NO ISSUE][CLUS] Add API to check for failed nodes
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add API to check if some nodes failed in ClusterStateManager.
- Add sync progress to PartitionReplica toString.
Change-Id: I7a5eb5b070f0a8a7a2f1a952fd7db35df244246e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13283
Reviewed-by: Ali Alsuliman
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../org/apache/asterix/common/cluster/IClusterStateManager.java | 8
.../java/org/apache/asterix/replication/api/PartitionReplica.java | 1 +
.../org/apache/asterix/runtime/utils/ClusterStateManager.java | 5 +
3 files changed, 14 insertions(+)
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
index 41f786a..76802d9 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
@@ -272,4 +272,12 @@ public interface IClusterStateManager {
* @return the cluster partitions map
*/
Map getClusterPartitions();
+
+/**
+ * Returns true if any of the nodes in {@code nodesId} is currently
inactive
+ *
+ * @param nodeIds
+ * @return true if any of the nodes is currently inactive, otherwise false
+ */
+boolean nodesFailed(Set nodeIds);
}
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
index c49bb7b..e1f99f4 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
@@ -160,6 +160,7 @@ public class PartitionReplica implements IPartitionReplica {
ObjectNode json = OBJECT_MAPPER.createObjectNode();
json.put("id", id.toString());
json.put("status", status.name());
+json.put("syncProgress", syncProgress);
return json;
}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index c5398e1..6de1dda 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -478,6 +478,11 @@ public class ClusterStateManager implements
IClusterStateManager {
return Collections.unmodifiableMap(clusterPartitions);
}
+@Override
+public synchronized boolean nodesFailed(Set nodeIds) {
+return nodeIds.stream().anyMatch(failedNodes::contains);
+}
+
private void updateClusterCounters(String nodeId, NcLocalCounters
localCounters) {
final IResourceIdManager resourceIdManager =
appCtx.getResourceIdManager();
resourceIdManager.report(nodeId, localCounters.getMaxResourceId());
[asterixdb] branch master updated: [NO ISSUE][REP] Ensure files delete order from replicas
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new d8a1a93 [NO ISSUE][REP] Ensure files delete order from replicas
d8a1a93 is described below
commit d8a1a93f99ce7c789dcc78afa1df3398d833cb15
Author: Murtadha Hubail
AuthorDate: Tue Sep 14 21:21:17 2021 +0300
[NO ISSUE][REP] Ensure files delete order from replicas
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When deleting files from a replica, delete the resource
files last to ensure no orphaned component files if
the replica fails after deleting the resource file.
- When replicating a resource file, ensure the index
directory is empty.
Change-Id: I7dbb9960cb644c063de49dd0920a1709964de2f8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13224
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java | 1 -
.../org/apache/asterix/replication/messaging/ReplicateFileTask.java | 5 +
.../apache/asterix/replication/sync/ReplicaFilesSynchronizer.java| 5 +
3 files changed, 10 insertions(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
index a926c66..dd7a663 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
@@ -46,7 +46,6 @@ public class LocalStorageCleanupTask implements
INCLifecycleTask {
deleteInvalidMetadataIndexes(localResourceRepository);
final Set nodePartitions =
appContext.getReplicaManager().getPartitions();
localResourceRepository.deleteCorruptedResources();
-//TODO optimize this to cleanup all active partitions at once
for (Integer partition : nodePartitions) {
localResourceRepository.cleanup(partition);
}
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
index 7f26b96..bf1613c 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
@@ -39,6 +39,7 @@ import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.replication.api.IReplicaTask;
import org.apache.asterix.replication.api.IReplicationWorker;
import org.apache.asterix.replication.management.NetworkingUtil;
+import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -72,6 +73,10 @@ public class ReplicateFileTask implements IReplicaTask {
// resolve path
final FileReference localPath = ioManager.resolve(file);
final Path resourceDir =
Files.createDirectories(localPath.getFile().getParentFile().toPath());
+if (indexMetadata) {
+// ensure clean index directory
+FileUtils.cleanDirectory(resourceDir.toFile());
+}
// create mask
final Path maskPath = Paths.get(resourceDir.toString(),
StorageConstants.MASK_FILE_PREFIX +
localPath.getFile().getName());
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java
index 735318d..477559c 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java
@@ -21,6 +21,7 @@ package org.apache.asterix.replication.sync;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -127,6 +128,10 @@ public class ReplicaFilesSynchronizer {
private void deleteInvalidFiles(List files) {
final FileSynchronizer sync = new FileSynchronizer(appCtx, replica);
+// sort files to ensure index metadata files starting
[asterixdb] branch master updated: [NO ISSUE][REP] Add API to perform non-delta recovery for a replica
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 8f278c0 [NO ISSUE][REP] Add API to perform non-delta recovery for a
replica
8f278c0 is described below
commit 8f278c042d92135f737cd7b26bd56a3479e11106
Author: Murtadha Hubail
AuthorDate: Tue Sep 14 02:12:15 2021 +0300
[NO ISSUE][REP] Add API to perform non-delta recovery for a replica
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add an option to perform non-delta recovery for a replica.
Change-Id: Ib1837e8f1aefdd9e085ccfd62f1c6e6d4eb969e8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13223
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../asterix/replication/api/PartitionReplica.java | 6 +-
.../replication/messaging/DeletePartitionTask.java | 75 ++
.../messaging/PartitionResourcesListResponse.java | 2 +-
.../replication/messaging/ReplicationProtocol.java | 5 +-
.../replication/sync/ReplicaFilesSynchronizer.java | 14 +++-
.../replication/sync/ReplicaSynchronizer.java | 8 +--
.../PersistentLocalResourceRepository.java | 12
7 files changed, 112 insertions(+), 10 deletions(-)
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
index e265d03..3b10700 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
@@ -79,17 +79,17 @@ public class PartitionReplica implements IPartitionReplica {
}
public synchronized void sync() {
-sync(true);
+sync(true, true);
}
-public synchronized void sync(boolean register) {
+public synchronized void sync(boolean register, boolean deltaRecovery) {
if (status == IN_SYNC || status == CATCHING_UP) {
return;
}
setStatus(CATCHING_UP);
appCtx.getThreadExecutor().execute(() -> {
try {
-new ReplicaSynchronizer(appCtx, this).sync(register);
+new ReplicaSynchronizer(appCtx, this).sync(register,
deltaRecovery);
setStatus(IN_SYNC);
} catch (Exception e) {
LOGGER.error(() -> "Failed to sync replica " + this, e);
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/DeletePartitionTask.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/DeletePartitionTask.java
new file mode 100644
index 000..90139df
--- /dev/null
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/DeletePartitionTask.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.replication.messaging;
+
+import java.io.DataInput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.exceptions.ReplicationException;
+import org.apache.asterix.replication.api.IReplicaTask;
+import org.apache.asterix.replication.api.IReplicationWorker;
+import
org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class DeletePartitionTask implements IReplicaTask {
+
+private static final Logger LOGGER = LogManager.getLogger();
+private final int partitionId;
+
+public DeletePartitionTask(int partitionId) {
+this.partitionId = partitionId;
[asterixdb] branch master updated: [NO ISSUE][ACTIVE] Account for force stop while suspending
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5e11053 [NO ISSUE][ACTIVE] Account for force stop while suspending
new bc8aaf6 Merge branch 'cheshire-cat' into master
5e11053 is described below
commit 5e110538870c9385f316e0395b80a76f0e6d8e7c
Author: Murtadha Hubail
AuthorDate: Fri Sep 10 01:36:34 2021 +0300
[NO ISSUE][ACTIVE] Account for force stop while suspending
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When a failure happens while trying to suspend ingestion,
we will force stop the active job. If the job completes
ungracefully, we set the listener state to TEMPORARILY_FAILED.
However, since force to stop only waits for STOPPED state,
the thread waiting for ingestion to be suspended will wait
forever. This change accounts for such case and makes
the force stop waits for TEMPORARILY_FAILED too.
Change-Id: Ib33f191be2b84d97a08e3bc6d607b0edbf35bed1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13144
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
(cherry picked from commit eed8714ae56bd61656750bf543181e7dd68c26c1)
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13203
---
.../org/apache/asterix/app/active/ActiveEntityEventsListener.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 0242ecd..ddd3d64 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -195,7 +195,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
@SuppressWarnings("unchecked")
protected void finish(ActiveEvent event) throws HyracksDataException {
if (LOGGER.isEnabled(level)) {
-LOGGER.log(level, "the job " + jobId + " finished");
+LOGGER.log(level, "the job {} finished", jobId);
}
JobId lastJobId = jobId;
if (numRegistered != numDeRegistered) {
@@ -208,7 +208,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
JobStatus jobStatus = status.getLeft();
List exceptions = status.getRight();
if (LOGGER.isEnabled(level)) {
-LOGGER.log(level, "The job finished with status: " + jobStatus);
+LOGGER.log(level, "The job finished with status: {}", jobStatus);
}
if (!jobSuccessfullyTerminated(jobStatus)) {
jobFailure = exceptions.isEmpty() ? new
RuntimeDataException(ErrorCode.UNREPORTED_TASK_FAILURE_EXCEPTION)
@@ -440,8 +440,9 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
private void cancelJob(Throwable th) {
cancelJobSafely(metadataProvider, th);
+// we can come here due to a failure while in suspending state
final WaitForStateSubscriber cancelSubscriber =
-new WaitForStateSubscriber(this,
EnumSet.of(ActivityState.STOPPED));
+new WaitForStateSubscriber(this,
EnumSet.of(ActivityState.STOPPED, ActivityState.TEMPORARILY_FAILED));
final Span span = Span.start(2, TimeUnit.MINUTES);
InvokeUtil.doUninterruptibly(() -> {
if (!cancelSubscriber.sync(span)) {
@@ -491,6 +492,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
forceStop(subscriber, ie);
Thread.currentThread().interrupt();
} catch (Throwable e) {
+LOGGER.error("forcing active job stop due to", e);
forceStop(subscriber, e);
} finally {
Thread.currentThread().setName(nameBefore);
[asterixdb] branch master updated: [NO ISSUE][STO] Ensure resources file operations are synchronized
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new d6adadc [NO ISSUE][STO] Ensure resources file operations are
synchronized
d6adadc is described below
commit d6adadc8fd4fb5c04f41097aca8a433528a560d6
Author: Murtadha Hubail
AuthorDate: Sun Sep 12 01:35:58 2021 +0300
[NO ISSUE][STO] Ensure resources file operations are synchronized
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- To avoid an operation reading a partially written resource file,
ensure all such operations are synchronized.
- Limit partition resources search to the partition's root directory.
Change-Id: I95f8565780675798393d7d43c0051c04e2b0a98c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13164
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../org/apache/asterix/app/nc/RecoveryManager.java | 5 +-
.../replication/logging/RemoteLogsNotifier.java| 4 +-
.../replication/messaging/ReplicateFileTask.java | 5 +-
.../PersistentLocalResourceRepository.java | 80 ++
4 files changed, 60 insertions(+), 34 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 4f5a9f8..6736642 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -291,7 +291,7 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
final IIndexCheckpointManagerProvider indexCheckpointManagerProvider =
((INcApplicationContext)
(serviceCtx.getApplicationContext())).getIndexCheckpointManagerProvider();
-Map resourcesMap =
localResourceRepository.loadAndGetAllResources();
+Map resourcesMap =
localResourceRepository.getResources(r -> true, partitions);
final Map resourceId2MaxLSNMap = new HashMap<>();
TxnEntityId tempKeyTxnEntityId = new TxnEntityId(-1, -1, -1, null, -1,
false);
@@ -503,7 +503,8 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
final List partitionResources =
localResourceRepository.getResources(resource -> {
DatasetLocalResource dsResource = (DatasetLocalResource)
resource.getResource();
return dsResource.getPartition() == partition;
-
}).values().stream().map(DatasetResourceReference::of).collect(Collectors.toList());
+},
Collections.singleton(partition)).values().stream().map(DatasetResourceReference::of)
+.collect(Collectors.toList());
for (DatasetResourceReference indexRef : partitionResources) {
try {
final IIndexCheckpointManager idxCheckpointMgr =
idxCheckpointMgrProvider.get(indexRef);
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/logging/RemoteLogsNotifier.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/logging/RemoteLogsNotifier.java
index 57463cb..004b640 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/logging/RemoteLogsNotifier.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/logging/RemoteLogsNotifier.java
@@ -19,6 +19,7 @@
package org.apache.asterix.replication.logging;
import java.io.IOException;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -91,7 +92,8 @@ class RemoteLogsNotifier implements Runnable {
return dls.getDatasetId() == datasetId && dls.getPartition() ==
resourcePartition
&& !masterPartitions.contains(dls.getPartition());
};
-final Map resources =
localResourceRep.getResources(replicaIndexesPredicate);
+final Map resources =
+localResourceRep.getResources(replicaIndexesPredicate,
Collections.singleton(resourcePartition));
final List replicaIndexesRef =
resources.values().stream().map(DatasetResourceReference::of).collect(Collectors.toList());
for (DatasetResourceReference replicaIndexRef : replicaIndexesRef) {
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
index 3ee3094..7f26b96 100644
---
a/asterixdb/asterix-replicat
[asterixdb] branch master updated: [NO ISSUE][ACTIVE] Account for force stop while suspending
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new eed8714 [NO ISSUE][ACTIVE] Account for force stop while suspending
eed8714 is described below
commit eed8714ae56bd61656750bf543181e7dd68c26c1
Author: Murtadha Hubail
AuthorDate: Fri Sep 10 01:36:34 2021 +0300
[NO ISSUE][ACTIVE] Account for force stop while suspending
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When a failure happens while trying to suspend ingestion,
we will force stop the active job. If the job completes
ungracefully, we set the listener state to TEMPORARILY_FAILED.
However, since force to stop only waits for STOPPED state,
the thread waiting for ingestion to be suspended will wait
forever. This change accounts for such case and makes
the force stop waits for TEMPORARILY_FAILED too.
Change-Id: Ib33f191be2b84d97a08e3bc6d607b0edbf35bed1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13144
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../org/apache/asterix/app/active/ActiveEntityEventsListener.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 0242ecd..ddd3d64 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -195,7 +195,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
@SuppressWarnings("unchecked")
protected void finish(ActiveEvent event) throws HyracksDataException {
if (LOGGER.isEnabled(level)) {
-LOGGER.log(level, "the job " + jobId + " finished");
+LOGGER.log(level, "the job {} finished", jobId);
}
JobId lastJobId = jobId;
if (numRegistered != numDeRegistered) {
@@ -208,7 +208,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
JobStatus jobStatus = status.getLeft();
List exceptions = status.getRight();
if (LOGGER.isEnabled(level)) {
-LOGGER.log(level, "The job finished with status: " + jobStatus);
+LOGGER.log(level, "The job finished with status: {}", jobStatus);
}
if (!jobSuccessfullyTerminated(jobStatus)) {
jobFailure = exceptions.isEmpty() ? new
RuntimeDataException(ErrorCode.UNREPORTED_TASK_FAILURE_EXCEPTION)
@@ -440,8 +440,9 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
private void cancelJob(Throwable th) {
cancelJobSafely(metadataProvider, th);
+// we can come here due to a failure while in suspending state
final WaitForStateSubscriber cancelSubscriber =
-new WaitForStateSubscriber(this,
EnumSet.of(ActivityState.STOPPED));
+new WaitForStateSubscriber(this,
EnumSet.of(ActivityState.STOPPED, ActivityState.TEMPORARILY_FAILED));
final Span span = Span.start(2, TimeUnit.MINUTES);
InvokeUtil.doUninterruptibly(() -> {
if (!cancelSubscriber.sync(span)) {
@@ -491,6 +492,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
forceStop(subscriber, ie);
Thread.currentThread().interrupt();
} catch (Throwable e) {
+LOGGER.error("forcing active job stop due to", e);
forceStop(subscriber, e);
} finally {
Thread.currentThread().setName(nameBefore);
[asterixdb] 02/03: Merge branch 'gerrit/mad-hatter' into 'gerrit/cheshire-cat'
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 20314d118059da634847e1ddb1b1d744e552e713
Merge: 2521ade cc6143b
Author: Ali Alsuliman
AuthorDate: Wed Sep 8 00:19:26 2021 +0300
Merge branch 'gerrit/mad-hatter' into 'gerrit/cheshire-cat'
Change-Id: I3e700b07781bec8fc5b9eabf15a1249ce2be0272
.../substr-ASTERIXDB-2949.0.query.sqlpp| 25
.../substr-ASTERIXDB-2949.0.adm| 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5
.../data/std/primitive/UTF8StringPointable.java| 3 +-
.../data/std/util/AbstractVarLenObjectBuilder.java | 5 ++--
.../std/primitive/UTF8StringPointableTest.java | 33 ++
6 files changed, 69 insertions(+), 3 deletions(-)
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index f0b20bf,600dde8..1e142a2
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@@ -9926,8 -9301,13 +9926,13 @@@
-
-substring-after-1
++
++substr-ASTERIXDB-2949
+
+
+
-
-substring-after-2
+
+regexp_position_with_flag/offset0/regex_position0_with_flag
diff --cc
hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java
index 828de18,eff71de..49f6221
---
a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java
+++
b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java
@@@ -463,12 -368,14 +463,13 @@@ public final class UTF8StringPointable
return false;
}
- builder.reset(out, Math.min(utfLen - byteIdx, (int) (codePointLength
* 1.0 * byteIdx / codePointIdx)));
-// for byteIdx = 0, this estimate assumes that every char size = 1
byte
-int estimateOutBytes = byteIdx == 0 ? charLength : (int) (charLength
* 1.0 * byteIdx / chIdx);
++int estimateOutBytes = byteIdx == 0 ? codePointLength : (int)
(codePointLength * 1.0 * byteIdx / codePointIdx);
+ builder.reset(out, Math.min(utfLen - byteIdx, estimateOutBytes));
-chIdx = 0;
-while (byteIdx < utfLen && chIdx < charLength) {
-builder.appendChar(src.charAt(src.getMetaDataLength() + byteIdx));
-chIdx++;
-byteIdx += src.charSize(src.getMetaDataLength() + byteIdx);
+codePointIdx = 0;
+while (byteIdx < utfLen && codePointIdx < codePointLength) {
+builder.appendCodePoint(src.codePointAt(src.getMetaDataLength() +
byteIdx));
+codePointIdx++;
+byteIdx += src.codePointSize(src.getMetaDataLength() + byteIdx);
}
builder.finish();
return true;
diff --cc
hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/test/java/org/apache/hyracks/data/std/primitive/UTF8StringPointableTest.java
index dcf9a10,ed439cb..f088c7e
---
a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/test/java/org/apache/hyracks/data/std/primitive/UTF8StringPointableTest.java
+++
b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/test/java/org/apache/hyracks/data/std/primitive/UTF8StringPointableTest.java
@@@ -306,60 -255,39 +306,93 @@@ public class UTF8StringPointableTest
result.set(storage.getByteArray(), 0, storage.getLength());
expected = generateUTF8Pointable("is is it.i am;here. ");
assertEquals(0, expected.compareTo(result));
+
+// Test Emoji trim
+input = STRING_POINTABLE_EMOJI_FAMILY_OF_4;
+pattern = "";
+patternPointable = generateUTF8Pointable(pattern);
+codePointSet.clear();
+patternPointable.getCodePoints(codePointSet);
+
+// Trim left
+storage.reset();
+input.trim(builder, storage, true, false, codePointSet);
+result.set(storage.getByteArray(), 0, storage.getLength());
+expected = generateUTF8Pointable("\u200D" + "");
+assertEquals(0, expected.compareTo(result));
+
+// Trim right
+storage.reset();
+input.trim(builder, storage, false, true, codePointSet);
+result.set(storage.getByteArray(), 0, storage.getLength());
+expected = generateUTF8Pointable("" + "\u200D");
+assertEquals(0, expected.compareTo(result));
+
+// Trim left and right
+storage.reset();
+input.trim(builder, storage, true, true, codePointSet);
+result.set(sto
[asterixdb] branch master updated (c84b073 -> d523ccc6)
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from c84b073 [ASTERIXDB-2933][COMP][EXT] Pushdowns Part3: Enable pushdown
new cc6143b [ASTERIXDB-2949][RUN][FUN] SUBSTR function produces malformed
string
new 20314d1 Merge branch 'gerrit/mad-hatter' into 'gerrit/cheshire-cat'
new d523ccc6 Merge branch 'cheshire-cat' into master
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../substr-ASTERIXDB-2949.0.query.sqlpp} | 13 ++---
.../substr-ASTERIXDB-2949.0.adm| 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5
.../data/std/primitive/UTF8StringPointable.java| 3 +-
.../data/std/util/AbstractVarLenObjectBuilder.java | 5 ++--
.../std/primitive/UTF8StringPointableTest.java | 33 ++
6 files changed, 46 insertions(+), 14 deletions(-)
copy
asterixdb/asterix-app/src/test/resources/{optimizerts/queries_sqlpp/unnest-to-join_01.sqlpp
=>
runtimets/queries_sqlpp/string/substr-ASTERIXDB-2949/substr-ASTERIXDB-2949.0.query.sqlpp}
(74%)
create mode 100644
asterixdb/asterix-app/src/test/resources/runtimets/results/string/substr-ASTERIXDB-2949/substr-ASTERIXDB-2949.0.adm
[asterixdb] 03/03: Merge branch 'cheshire-cat' into master
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git commit d523ccc6e7a0998d6452bf1650d9d9bf31f463a3 Merge: c84b073 20314d1 Author: Murtadha Hubail AuthorDate: Wed Sep 8 19:59:11 2021 +0300 Merge branch 'cheshire-cat' into master Change-Id: I762bdfd55c4aa54ea507facd8489f78ce3d73785 .../substr-ASTERIXDB-2949.0.query.sqlpp| 25 .../substr-ASTERIXDB-2949.0.adm| 1 + .../test/resources/runtimets/testsuite_sqlpp.xml | 5 .../data/std/primitive/UTF8StringPointable.java| 3 +- .../data/std/util/AbstractVarLenObjectBuilder.java | 5 ++-- .../std/primitive/UTF8StringPointableTest.java | 33 ++ 6 files changed, 69 insertions(+), 3 deletions(-)
[asterixdb] 01/03: [ASTERIXDB-2949][RUN][FUN] SUBSTR function produces malformed string
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit cc6143b4ef5bb3f505478ada2bd95350a0758f6a
Author: Ali Alsuliman
AuthorDate: Tue Aug 17 18:00:11 2021 +0300
[ASTERIXDB-2949][RUN][FUN] SUBSTR function produces malformed string
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix UTF8StringBuilder grow logic
UTF8StringBuilder initially takes an estimated length of the
string to be written and reserves space at the beginning
of the buffer to later store the length of the data written.
When the actual data written happens to be greater than the
estimated length requiring more space to store the length,
the string content needs to be shifted.
This patch is to fix the starting offset of the data to be shifted.
Also, the estimated length calculation of the substring method of
the UTF8StringPointable is modified to account for
SUBSTR(input_string, 0, num_chars_to_substring) with start offset = 0.
Change-Id: If36253ff884a9c19eaa130c4e5e926f2dd9eea1d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12864
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Ali Alsuliman
Reviewed-by: Ian Maxon
---
.../substr-ASTERIXDB-2949.0.query.sqlpp| 25
.../substr-ASTERIXDB-2949.0.adm| 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5
.../data/std/primitive/UTF8StringPointable.java| 4 ++-
.../data/std/util/AbstractVarLenObjectBuilder.java | 5 ++--
.../std/primitive/UTF8StringPointableTest.java | 33 ++
6 files changed, 70 insertions(+), 3 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/string/substr-ASTERIXDB-2949/substr-ASTERIXDB-2949.0.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/string/substr-ASTERIXDB-2949/substr-ASTERIXDB-2949.0.query.sqlpp
new file mode 100644
index 000..22105a4
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/string/substr-ASTERIXDB-2949/substr-ASTERIXDB-2949.0.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description: Test the issue described in ASTERIXDB-2949
+ * Success: Yes
+ */
+
+
+SELECT
SUBSTR("•\tABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\n•\tabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\tABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
0, 1000) AS s;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/string/substr-ASTERIXDB-2949/substr-ASTERIXDB-2949.0.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/string/substr-ASTERIXDB-2949/substr-ASTERIXDB-2949.0.adm
new file mode 100644
index 000..a36b551
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/string/substr-ASTERIXDB-2949/substr-ASTERIXDB-2949.0.adm
@@ -0,0 +1 @@
+{ "s":
"•\tABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\n•\tabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\tABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
}
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 8d06f71..600dde8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -9296,6 +9296,11 @@
+
+substr-ASTERIXDB-2949
+
+
+
substring-after-1
diff --git
a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/UTF8StringPointable.java
b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/st
[asterixdb] branch master updated: [NO ISSUE][REP] Add replica sync progress
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new a1de795 [NO ISSUE][REP] Add replica sync progress
a1de795 is described below
commit a1de795e82273b128b706794988fb7dd09a0267d
Author: Murtadha Hubail
AuthorDate: Thu Sep 2 13:35:49 2021 +0300
[NO ISSUE][REP] Add replica sync progress
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add replica sync progress based on the replica missing
files.
- Add replica last progress timestamp that can be used
to determine replica progress inactivity.
Change-Id: Iab2cd7e745c4150e2d0aef3af864ec0f66dd96e7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13063
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../common/replication/IPartitionReplica.java | 14 +
.../asterix/replication/api/PartitionReplica.java | 36 ++
.../replication/sync/ReplicaFilesSynchronizer.java | 7 -
3 files changed, 56 insertions(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IPartitionReplica.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IPartitionReplica.java
index 761b2c6..f311655 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IPartitionReplica.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IPartitionReplica.java
@@ -51,4 +51,18 @@ public interface IPartitionReplica {
* @param failure
*/
void notifyFailure(Exception failure);
+
+/**
+ * Gets the current sync progress
+ *
+ * @return the current sync progress
+ */
+double getSyncProgress();
+
+/**
+ * Gets the last progress time of this replica based on System.nanoTime
+ *
+ * @return the last progress time
+ */
+long getLastProgressTime();
}
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
index 282d475..e265d03 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
@@ -52,6 +52,8 @@ public class PartitionReplica implements IPartitionReplica {
private static final int INITIAL_BUFFER_SIZE =
StorageUtil.getIntSizeInBytes(4, StorageUtil.StorageUnit.KILOBYTE);
private final INcApplicationContext appCtx;
private final ReplicaIdentifier id;
+private double syncProgress = -1;
+private long lastProgressTime = -1;
private ByteBuffer reusbaleBuf;
private PartitionReplicaStatus status = DISCONNECTED;
private ISocketChannel sc;
@@ -133,6 +135,16 @@ public class PartitionReplica implements IPartitionReplica
{
return reusbaleBuf;
}
+public synchronized void setSyncProgress(double syncProgress) {
+this.syncProgress = syncProgress;
+lastProgressTime = System.nanoTime();
+}
+
+@Override
+public synchronized double getSyncProgress() {
+return syncProgress;
+}
+
private JsonNode asJson() {
ObjectNode json = OBJECT_MAPPER.createObjectNode();
json.put("id", id.toString());
@@ -153,6 +165,19 @@ public class PartitionReplica implements IPartitionReplica
{
}
@Override
+public synchronized long getLastProgressTime() {
+switch (status) {
+case IN_SYNC:
+return System.nanoTime();
+case CATCHING_UP:
+return lastProgressTime;
+case DISCONNECTED:
+return -1;
+}
+return -1;
+}
+
+@Override
public int hashCode() {
return id.hashCode();
}
@@ -172,6 +197,17 @@ public class PartitionReplica implements IPartitionReplica
{
}
LOGGER.info(() -> "Replica " + this + " status changing: " +
this.status + " -> " + status);
this.status = status;
+switch (status) {
+case IN_SYNC:
+syncProgress = 1;
+break;
+case CATCHING_UP:
+lastProgressTime = System.nanoTime();
+break;
+case DISCONNECTED:
+syncProgress = -1;
+break;
+}
}
private void sendGoodBye() {
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchroni
[asterixdb] branch master updated: [NO ISSUE][OTH] Add pending replicate ops in io stats
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 072ee50 [NO ISSUE][OTH] Add pending replicate ops in io stats
072ee50 is described below
commit 072ee5035413aedbc252fa63322dad38e30427df
Author: Murtadha Hubail
AuthorDate: Fri Aug 27 00:43:47 2021 +0300
[NO ISSUE][OTH] Add pending replicate ops in io stats
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Include pending replicate ops in the IO stats.
Change-Id: I56213b6d2027d98c2bf973fac5db5316d6e7fd8e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13003
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
Tested-by: Jenkins
Integration-Tests: Jenkins
---
.../java/org/apache/asterix/common/context/DatasetInfo.java | 11 +++
.../asterix/common/context/DatasetLifecycleManager.java | 1 +
.../org/apache/asterix/common/storage/StorageIOStats.java | 9 +
3 files changed, 21 insertions(+)
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
index 72dacc4..7483a17 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
@@ -45,6 +45,7 @@ public class DatasetInfo extends Info implements
Comparable {
private int numActiveIOOps;
private int pendingFlushes;
private int pendingMerges;
+private int pendingReplications;
private long lastAccess;
private boolean isExternal;
private boolean isRegistered;
@@ -82,6 +83,9 @@ public class DatasetInfo extends Info implements
Comparable {
case MERGE:
pendingMerges++;
break;
+case REPLICATE:
+pendingReplications++;
+break;
default:
break;
}
@@ -96,6 +100,9 @@ public class DatasetInfo extends Info implements
Comparable {
case MERGE:
pendingMerges--;
break;
+case REPLICATE:
+pendingReplications--;
+break;
default:
break;
}
@@ -251,4 +258,8 @@ public class DatasetInfo extends Info implements
Comparable {
public synchronized int getPendingMerges() {
return pendingMerges;
}
+
+public synchronized int getPendingReplications() {
+return pendingReplications;
+}
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index bd24696..c431dca 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -562,6 +562,7 @@ public class DatasetLifecycleManager implements
IDatasetLifecycleManager, ILifeC
for (DatasetResource dsr : datasets.values()) {
stats.addPendingFlushes(dsr.getDatasetInfo().getPendingFlushes());
stats.addPendingMerges(dsr.getDatasetInfo().getPendingMerges());
+
stats.addPendingReplications(dsr.getDatasetInfo().getPendingReplications());
}
return stats;
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java
index 0b44f76..da4578f 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/StorageIOStats.java
@@ -22,6 +22,7 @@ public class StorageIOStats {
private int pendingFlushes;
private int pendingMerges;
+private int pendingReplications;
public void addPendingFlushes(int pending) {
pendingFlushes += pending;
@@ -31,6 +32,10 @@ public class StorageIOStats {
pendingMerges += pending;
}
+public void addPendingReplications(int pending) {
+pendingReplications += pending;
+}
+
public int getPendingFlushes() {
return pendingFlushes;
}
@@ -38,4 +43,8 @@ public class StorageIOStats {
public int getPendingMerges() {
return pendingMerges;
}
+
+public int getPendingReplications() {
+return pendingReplications;
+}
}
[asterixdb] branch master updated: [NO ISSUE][REP] Do not fail create/drop resource on replica failure
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 07d202d [NO ISSUE][REP] Do not fail create/drop resource on replica
failure
07d202d is described below
commit 07d202d17ab0131703cc6731506d9880d180e65b
Author: Murtadha Hubail
AuthorDate: Thu Aug 26 02:27:42 2021 +0300
[NO ISSUE][REP] Do not fail create/drop resource on replica failure
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When creating/dropping a resource, do not fail the operation
on the master node if it fails to send the opreation to the
replica. When the replica node is recovered, it will be
re-synced and will pick up all the changes.
Change-Id: I8313569cfda4b7e4fbeb23d4fdb998b3805367e1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12985
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../management/resource/PersistentLocalResourceRepository.java | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 2494893..3b192aa 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -212,7 +212,11 @@ public class PersistentLocalResourceRepository implements
ILocalResourceReposito
resourceCache.put(resource.getPath(), resource);
//if replication enabled, send resource metadata info to remote nodes
if (isReplicationEnabled) {
-createReplicationJob(ReplicationOperation.REPLICATE, resourceFile);
+try {
+createReplicationJob(ReplicationOperation.REPLICATE,
resourceFile);
+} catch (Exception e) {
+LOGGER.error("failed to send resource file {} to replicas",
resourceFile);
+}
}
}
@@ -233,8 +237,10 @@ public class PersistentLocalResourceRepository implements
ILocalResourceReposito
FileReference resourceFile = getLocalResourceFileByName(ioManager,
relativePath);
try {
if (resourceFile.getFile().exists()) {
-if (isReplicationEnabled) {
+try {
createReplicationJob(ReplicationOperation.DELETE,
resourceFile);
+} catch (Exception e) {
+LOGGER.error("failed to delete resource file {} from
replicas", resourceFile);
}
final LocalResource localResource =
readLocalResource(resourceFile.getFile());
IoUtil.delete(resourceFile);
[asterixdb] branch master updated: [NO ISSUE][REP] Extensible Recovery Cleanup
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new e7f6f72 [NO ISSUE][REP] Extensible Recovery Cleanup
e7f6f72 is described below
commit e7f6f72a480ed09d879e540a325825a61d55fae0
Author: Murtadha Hubail
AuthorDate: Thu Aug 26 02:15:07 2021 +0300
[NO ISSUE][REP] Extensible Recovery Cleanup
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Extensible recovery cleanup after replaying replica
partitions log.
Change-Id: I57aa7e17026148d2e8d3307a2b82ef4dbc93b910
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12984
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../src/main/java/org/apache/asterix/app/nc/RecoveryManager.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index ffde7d0..4f5a9f8 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -161,14 +161,12 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
public void startLocalRecovery(Set partitions) throws
IOException, ACIDException {
state = SystemState.RECOVERING;
LOGGER.info("starting recovery for partitions {}", partitions);
-
long readableSmallestLSN = logMgr.getReadableSmallestLSN();
Checkpoint checkpointObject = checkpointManager.getLatest();
long lowWaterMarkLSN = checkpointObject.getMinMCTFirstLsn();
if (lowWaterMarkLSN < readableSmallestLSN) {
lowWaterMarkLSN = readableSmallestLSN;
}
-
//delete any recovery files from previous failed recovery attempts
deleteRecoveryTemporaryFiles();
@@ -538,6 +536,7 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
if (flush) {
appCtx.getDatasetLifecycleManager().flushAllDatasets();
}
+cleanUp(partitions);
} catch (IOException | ACIDException e) {
throw HyracksDataException.create(e);
} finally {
@@ -571,6 +570,10 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
FileUtils.deleteQuietly(recoveryFolderPath.toFile());
}
+protected void cleanUp(Set partitions) throws
HyracksDataException {
+// the cleanup is currently done by
PersistentLocalResourceRepository#clean
+}
+
private String getRecoveryDirPath() {
String logDir = logMgr.getLogManagerProperties().getLogDir();
if (!logDir.endsWith(File.separator)) {
[asterixdb] branch master updated: [NO ISSUE][REP] Persist master last valid seq on index checkpoint
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new bf22dbc [NO ISSUE][REP] Persist master last valid seq on index
checkpoint
bf22dbc is described below
commit bf22dbcb42fbb252dd2506b41ade61af1b3fa63c
Author: Murtadha Hubail
AuthorDate: Wed Aug 25 03:20:05 2021 +0300
[NO ISSUE][REP] Persist master last valid seq on index checkpoint
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- When a partition owner replicates a component to a replica,
maintain the last received component sequence from master.
This will be used to ensure that any component generated on master,
but the master fails before replicating it, will not be used when
the master is re-synced (recovered) from a promoted replica.
Change-Id: I102947712daa07c83b32103b3c58fad46de2dc6d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12966
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../asterix/app/nc/IndexCheckpointManager.java | 30 +++--
.../org/apache/asterix/app/nc/ReplicaManager.java | 18 +++
.../common/storage/IIndexCheckpointManager.java| 20 +++-
.../asterix/common/storage/IReplicaManager.java| 8 ++
.../asterix/common/storage/IndexCheckpoint.java| 24 +++-
.../asterix/common/storage/ResourceReference.java | 4 +
.../asterix/common/utils/StorageConstants.java | 1 +
.../messaging/CheckpointPartitionIndexesTask.java | 20 +++-
.../messaging/MarkComponentValidTask.java | 15 ++-
.../messaging/PartitionResourcesListResponse.java | 43 +--
.../messaging/PartitionResourcesListTask.java | 11 +-
.../replication/messaging/ReplicateFileTask.java | 15 ++-
.../asterix/replication/sync/FileSynchronizer.java | 4 +-
.../replication/sync/IndexSynchronizer.java| 6 +-
.../replication/sync/ReplicaFilesSynchronizer.java | 123 ++---
.../replication/sync/ReplicaSynchronizer.java | 4 +-
.../PersistentLocalResourceRepository.java | 20 +++-
17 files changed, 308 insertions(+), 58 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
index 4acc6d3..2d40ec8 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/IndexCheckpointManager.java
@@ -56,7 +56,7 @@ public class IndexCheckpointManager implements
IIndexCheckpointManager {
}
@Override
-public synchronized void init(long validComponentSequence, long lsn, long
validComponentId)
+public synchronized void init(long validComponentSequence, long lsn, long
validComponentId, String masterNodeId)
throws HyracksDataException {
List checkpoints;
try {
@@ -68,34 +68,42 @@ public class IndexCheckpointManager implements
IIndexCheckpointManager {
LOGGER.warn(() -> "Checkpoints found on initializing: " +
indexPath);
delete();
}
-IndexCheckpoint firstCheckpoint =
IndexCheckpoint.first(validComponentSequence, lsn, validComponentId);
+IndexCheckpoint firstCheckpoint =
+IndexCheckpoint.first(validComponentSequence, lsn,
validComponentId, masterNodeId);
persist(firstCheckpoint);
}
@Override
-public synchronized void replicated(long componentSequence, long
masterLsn, long componentId)
+public synchronized void replicated(long componentSequence, long
masterLsn, long componentId, String masterNodeId)
throws HyracksDataException {
final Long localLsn =
getLatest().getMasterNodeFlushMap().get(masterLsn);
if (localLsn == null) {
-throw new IllegalStateException("Component flushed before lsn
mapping was received");
+throw new IllegalStateException("Component replicated before lsn
mapping was received");
}
-flushed(componentSequence, localLsn, componentId);
+flushed(componentSequence, localLsn, componentId, masterNodeId);
}
@Override
-public synchronized void flushed(long componentSequence, long lsn, long
componentId) throws HyracksDataException {
+public synchronized void flushed(long componentSequence, long lsn, long
componentId, String masterNodeId)
+throws HyracksDataException {
final IndexCheckpoint latest = getLatest();
-IndexCheckpoint nextCheckpoint = IndexCheckpoint.next(latest, lsn,
componentSequence, componentId);
+
[asterixdb] branch master updated: [NO ISSUE][STO] Improve logging
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 0821193 [NO ISSUE][STO] Improve logging
0821193 is described below
commit 0821193be6c84b1ab80e85b75780f6cbcfb2f1e6
Author: Ali Alsuliman
AuthorDate: Sun Aug 22 03:37:30 2021 +0300
[NO ISSUE][STO] Improve logging
- user model changes: no
- storage format changes: no
- interface changes: no
Change-Id: I57c05bc089527e745cf1851be0814242ba89ac55
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12923
Reviewed-by: Murtadha Hubail
Tested-by: Jenkins
Integration-Tests: Jenkins
---
.../asterix/common/context/DatasetLifecycleManager.java | 17 -
.../common/context/PrimaryIndexOperationTracker.java| 4 ++--
2 files changed, 14 insertions(+), 7 deletions(-)
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index 2fe100e..bd24696 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -170,9 +170,15 @@ public class DatasetLifecycleManager implements
IDatasetLifecycleManager, ILifeC
closeIndex(iInfo);
dsInfo.removeIndex(resourceID);
synchronized (dsInfo) {
-if (dsInfo.getReferenceCount() == 0 && dsInfo.isOpen() &&
dsInfo.getIndexes().isEmpty()
-&& !dsInfo.isExternal()) {
+int referenceCount = dsInfo.getReferenceCount();
+boolean open = dsInfo.isOpen();
+boolean empty = dsInfo.getIndexes().isEmpty();
+if (referenceCount == 0 && open && empty && !dsInfo.isExternal()) {
+LOGGER.debug("removing dataset {} from cache",
dsInfo.getDatasetID());
removeDatasetFromCache(dsInfo.getDatasetID());
+} else {
+LOGGER.debug("keeping dataset {} in cache, ref count {}, open
{}, indexes count: {}",
+dsInfo.getDatasetID(), referenceCount, open,
dsInfo.getIndexes().size());
}
}
}
@@ -414,9 +420,10 @@ public class DatasetLifecycleManager implements
IDatasetLifecycleManager, ILifeC
logManager.log(waitLog);
for (PrimaryIndexOperationTracker primaryOpTracker :
dsr.getOpTrackers()) {
// flush each partition one by one
-if (primaryOpTracker.getNumActiveOperations() > 0) {
-throw new IllegalStateException(
-"flushDatasetOpenIndexes is called on a dataset with
currently active operations");
+int numActiveOperations =
primaryOpTracker.getNumActiveOperations();
+if (numActiveOperations > 0) {
+throw new IllegalStateException("flushDatasetOpenIndexes is
called on dataset " + dsInfo.getDatasetID()
++ " with currently " + "active operations, count=" +
numActiveOperations);
}
primaryOpTracker.setFlushOnExit(true);
primaryOpTracker.flushIfNeeded();
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
index ec05702..fb001a0 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
@@ -145,8 +145,8 @@ public class PrimaryIndexOperationTracker extends
BaseOperationTracker implement
}
}
if (primaryLsmIndex == null) {
-throw new IllegalStateException(
-"Primary index not found in dataset " +
dsInfo.getDatasetID() + " and partition " + partition);
+throw new IllegalStateException("Primary index not found in
dataset " + dsInfo.getDatasetID()
++ " and partition " + partition + " open indexes " +
indexes);
}
for (ILSMIndex lsmIndex : indexes) {
ILSMOperationTracker opTracker =
lsmIndex.getOperationTracker();
[asterixdb] branch master updated: [NO ISSUE][REP] Pass NC active partitions from CC
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new c9a48b9 [NO ISSUE][REP] Pass NC active partitions from CC
c9a48b9 is described below
commit c9a48b94e22384baf5305c55e8f23af7a26bf88b
Author: Murtadha Hubail
AuthorDate: Thu Aug 19 21:40:29 2021 +0300
[NO ISSUE][REP] Pass NC active partitions from CC
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Pass NC active partitions from CC and set them
during node bootstrap.
- Move local storage clean up to LocalStorageCleanupTask
to clean up after all node partitions have been set.
- Maintain last LSN in master flush LSN to account for
cases where the first LSN received from master is the
same as the low water mark LSN.
- Reduce replica failure logging at error level.
Change-Id: I3782bb2be61f8a57ac45dd6dd6ae0942e83ddc40
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12903
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
Tested-by: Murtadha Hubail
---
.../org/apache/asterix/app/nc/ReplicaManager.java | 7 ++
.../app/nc/task/LocalStorageCleanupTask.java | 13 +++
.../asterix/app/nc/task/UpdateNodeStatusTask.java | 26 +++---
.../app/replication/NcLifecycleCoordinator.java| 14 +---
.../message/RegistrationTasksResponseMessage.java | 2 +-
.../asterix/hyracks/bootstrap/NCApplication.java | 11 -
.../context/PrimaryIndexOperationTracker.java | 2 +-
.../asterix/common/storage/IReplicaManager.java| 7 ++
.../asterix/common/storage/IndexCheckpoint.java| 2 +-
.../asterix/replication/api/PartitionReplica.java | 6 -
.../replication/logging/RemoteLogsNotifier.java| 8 ---
.../management/IndexReplicationManager.java| 3 ++-
.../management/LogReplicationManager.java | 8 +--
.../replication/sync/ReplicaSynchronizer.java | 6 +++--
14 files changed, 86 insertions(+), 29 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 1d901bd..66a69ef 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -112,10 +112,17 @@ public class ReplicaManager implements IReplicaManager {
}
@Override
+public synchronized void setActivePartitions(Set
activePartitions) {
+partitions.clear();
+partitions.addAll(activePartitions);
+}
+
+@Override
public synchronized void promote(int partition) throws
HyracksDataException {
if (partitions.contains(partition)) {
return;
}
+LOGGER.warn("promoting partition {}", partition);
final PersistentLocalResourceRepository localResourceRepository =
(PersistentLocalResourceRepository)
appCtx.getLocalResourceRepository();
localResourceRepository.cleanup(partition);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
index 5471fb8..a926c66 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.nc.task;
+import java.util.Set;
+
import org.apache.asterix.common.api.INCLifecycleTask;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.dataflow.DatasetLocalResource;
@@ -41,6 +43,17 @@ public class LocalStorageCleanupTask implements
INCLifecycleTask {
INcApplicationContext appContext = (INcApplicationContext)
cs.getApplicationContext();
PersistentLocalResourceRepository localResourceRepository =
(PersistentLocalResourceRepository)
appContext.getLocalResourceRepository();
+deleteInvalidMetadataIndexes(localResourceRepository);
+final Set nodePartitions =
appContext.getReplicaManager().getPartitions();
+localResourceRepository.deleteCorruptedResources();
+//TODO optimize this to cleanup all active partitions at once
+for (Integer partition : nodePartitions) {
+localResourceRepository.cleanup(partition);
+}
+}
+
+private void
deleteInvalidMetadataIndexes(PersistentLocalResourceRepository
localResourceRepository)
+
[asterixdb] branch master updated: [ASTERIXDB-2952][COMP] Support CROSS JOIN
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new d3eccd4 [ASTERIXDB-2952][COMP] Support CROSS JOIN
d3eccd4 is described below
commit d3eccd4f3b3e95a71d6e8d1f1f9da12c50b248cd
Author: Dmitry Lychagin
AuthorDate: Thu Aug 19 12:18:17 2021 -0700
[ASTERIXDB-2952][COMP] Support CROSS JOIN
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add support for CROSS JOIN
- Add testcases and update documentation
Change-Id: Ib9f7350576592ca2858e46878e0b428855e3f2d2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12904
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Dmitry Lychagin
Reviewed-by: Ali Alsuliman
---
.../join/cross-join-01/cross-join-01.1.ddl.sqlpp | 46 ++
.../cross-join-01/cross-join-01.2.update.sqlpp | 22 +++
.../join/cross-join-01/cross-join-01.3.query.sqlpp | 27 +
.../join/cross-join-01/cross-join-01.4.query.sqlpp | 29 ++
.../join/cross-join-01/cross-join-01.5.query.sqlpp | 28 +
.../cross-join-02-negative.1.query.sqlpp | 26
.../results/join/cross-join-01/cross-join-01.3.adm | 1 +
.../results/join/cross-join-01/cross-join-01.4.adm | 25
.../results/join/cross-join-01/cross-join-01.5.adm | 4 ++
.../test/resources/runtimets/testsuite_sqlpp.xml | 11 ++
asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf | 3 +-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj| 37 +++--
12 files changed, 254 insertions(+), 5 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/cross-join-01/cross-join-01.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/cross-join-01/cross-join-01.1.ddl.sqlpp
new file mode 100644
index 000..dd75e79
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/cross-join-01/cross-join-01.1.ddl.sqlpp
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+
+create type test.Schema as
+ closed {
+ unique1 : bigint,
+ unique2 : bigint,
+ two : bigint,
+ four : bigint,
+ ten : bigint,
+ twenty : bigint,
+ onePercent : bigint,
+ tenPercent : bigint,
+ twentyPercent : bigint,
+ fiftyPercent : bigint,
+ unique3 : bigint,
+ evenOnePercent : bigint,
+ oddOnePercent : bigint,
+ stringu1 : string,
+ stringu2 : string,
+ string4 : string
+};
+
+create dataset onektup(Schema) primary key unique2;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/cross-join-01/cross-join-01.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/cross-join-01/cross-join-01.2.update.sqlpp
new file mode 100644
index 000..e5d85fe
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/join/cross-join-01/cross-join-01.2.update.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+load dataset onektup using lo
[asterixdb] branch master updated: [ASTERIXDB-2951][COMP] Support IS DISTINCT FROM operator
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new 4d5476e [ASTERIXDB-2951][COMP] Support IS DISTINCT FROM operator 4d5476e is described below commit 4d5476e6672e6bb749cc2a0de69f4887ff5dce9b Author: Dmitry Lychagin AuthorDate: Wed Aug 18 19:43:22 2021 -0700 [ASTERIXDB-2951][COMP] Support IS DISTINCT FROM operator - user model changes: no - storage format changes: no - interface changes: no Details: - Add support for IS [NOT] DISTINCT FROM comparison operator - Add testcases and update documentation Change-Id: Ifad404fa7c613771f60bd12f3aa2bd5fea77ba34 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12884 Integration-Tests: Jenkins Tested-by: Jenkins Reviewed-by: Dmitry Lychagin Reviewed-by: Ali Alsuliman --- .../is_distinct_01/is_distinct_01.1.query.sqlpp| 39 + .../is_distinct_01/is_distinct_01.2.query.sqlpp| 42 + .../comparison/is_distinct_01/is_distinct_01.1.adm | 81 ++ .../comparison/is_distinct_01/is_distinct_01.2.adm | 1 + .../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++ .../asterix-doc/src/main/markdown/sqlpp/2_expr.md | 4 +- .../asterix/lang/common/struct/OperatorType.java | 4 +- .../visitor/OperatorExpressionVisitor.java | 99 ++ .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj| 13 ++- 9 files changed, 268 insertions(+), 20 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/is_distinct_01/is_distinct_01.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/is_distinct_01/is_distinct_01.1.query.sqlpp new file mode 100644 index 000..c0ed568 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/is_distinct_01/is_distinct_01.1.query.sqlpp @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +from + [ int64(2), float(2.0), double(2.5), "str1", "str2", true, false, null, missing ] arg1, + [ int32(2), double(2.0), float(2.5), "str1", "str2", true, false, null, missing ] arg2 +let + is_distinct = arg1 is distinct from arg2, + is_not_distinct = arg1 is not distinct from arg2 +select + case +when arg1 is missing then "MISSING" +when arg1 is null then "NULL" +else arg1 + end as arg1, + case +when arg2 is missing then "MISSING" +when arg2 is null then "NULL" +else arg2 + end as arg2, + is_distinct as `!!==`, + is_not_distinct as `==` +order by is_distinct, arg1, arg2 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/is_distinct_01/is_distinct_01.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/is_distinct_01/is_distinct_01.2.query.sqlpp new file mode 100644 index 000..fcc1dc2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/is_distinct_01/is_distinct_01.2.query.sqlpp @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +let + x = 2, y = 3 +se
[asterixdb] branch master updated: [NO ISSUE][REP] Add Replicas Getter
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5444615 [NO ISSUE][REP] Add Replicas Getter
5444615 is described below
commit 5444615fcbb7ccf937e8ee38a2b9cf82a4c56fd9
Author: Murtadha Hubail
AuthorDate: Mon Aug 16 19:41:42 2021 +0300
[NO ISSUE][REP] Add Replicas Getter
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add getter in ReplicaManager for the current replicas.
- Add getter for the replica identifier.
Change-Id: I9394739ec7d591fbdf56127e9493c13005c6e38e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12863
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../src/main/java/org/apache/asterix/app/nc/ReplicaManager.java| 6 ++
.../java/org/apache/asterix/common/storage/IReplicaManager.java| 7 +++
.../java/org/apache/asterix/common/storage/ReplicaIdentifier.java | 4
3 files changed, 17 insertions(+)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 954ff8d..1d901bd 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.app.nc;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -141,6 +142,11 @@ public class ReplicaManager implements IReplicaManager {
return replicaSyncLock;
}
+@Override
+public synchronized List getReplicas() {
+return new ArrayList<>(replicas.values());
+}
+
public void closePartitionResources(int partition) throws
HyracksDataException {
final IDatasetLifecycleManager datasetLifecycleManager =
appCtx.getDatasetLifecycleManager();
//TODO(mhubail) we can flush only datasets of the requested partition
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
index b081c65..9630303 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
@@ -86,4 +86,11 @@ public interface IReplicaManager {
* @return The partition replica if found. Otherwise, null.
*/
IPartitionReplica getReplica(ReplicaIdentifier id);
+
+/**
+ * Gets the list of replicas
+ *
+ * @return the list of replicas
+ */
+List getReplicas();
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ReplicaIdentifier.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ReplicaIdentifier.java
index e14d005..1d02fee 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ReplicaIdentifier.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ReplicaIdentifier.java
@@ -75,6 +75,10 @@ public class ReplicaIdentifier {
return id.hashCode();
}
+public String getId() {
+return id;
+}
+
@Override
public String toString() {
return id;
[asterixdb] branch master updated: [NO ISSUE][REP] Extensible Replication Strategy Factory
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 3e4bf05 [NO ISSUE][REP] Extensible Replication Strategy Factory
3e4bf05 is described below
commit 3e4bf05fee9bdb879f0256647e532e2499d91c67
Author: Murtadha Hubail
AuthorDate: Sat Aug 14 21:01:57 2021 +0300
[NO ISSUE][REP] Extensible Replication Strategy Factory
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Allow specifying replication strategy factory at
NC application level.
- Add replica getter to replica manager.
Change-Id: I7c71ae2d19c81050c4e338efac155ae39f4b202e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12825
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 6 +++--
.../org/apache/asterix/app/nc/ReplicaManager.java | 5
.../asterix/hyracks/bootstrap/NCApplication.java | 8 +-
.../asterix/common/api/INcApplicationContext.java | 4 ++-
.../replication/IReplicationStrategyFactory.java | 30 ++
.../replication/ReplicationStrategyFactory.java| 9 +++
.../asterix/common/storage/IReplicaManager.java| 7 +
.../replication/management/ReplicationManager.java | 7 ++---
8 files changed, 63 insertions(+), 13 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index eb8a92f..f532352 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -53,6 +53,7 @@ import
org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.replication.IReplicationChannel;
import org.apache.asterix.common.replication.IReplicationManager;
+import org.apache.asterix.common.replication.IReplicationStrategyFactory;
import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
import org.apache.asterix.common.storage.IReplicaManager;
import org.apache.asterix.common.transactions.IRecoveryManager;
@@ -176,7 +177,8 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
@Override
public void initialize(IRecoveryManagerFactory recoveryManagerFactory,
IReceptionistFactory receptionistFactory,
-IConfigValidatorFactory configValidatorFactory, boolean
initialRun) throws IOException {
+IConfigValidatorFactory configValidatorFactory,
IReplicationStrategyFactory replicationStrategyFactory,
+boolean initialRun) throws IOException {
ioManager = getServiceContext().getIoManager();
int ioQueueLen =
getServiceContext().getAppConfig().getInt(NCConfig.Option.IO_QUEUE_SIZE);
threadExecutor =
@@ -231,7 +233,7 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
receptionist = receptionistFactory.create();
if (replicationProperties.isReplicationEnabled()) {
-replicationManager = new ReplicationManager(this,
replicationProperties);
+replicationManager = new ReplicationManager(this,
replicationStrategyFactory, replicationProperties);
//pass replication manager to replication required object
//LogManager to replicate logs
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 655f9da..954ff8d 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -101,6 +101,11 @@ public class ReplicaManager implements IReplicaManager {
}
@Override
+public synchronized IPartitionReplica getReplica(ReplicaIdentifier id) {
+return replicas.get(id);
+}
+
+@Override
public synchronized Set getPartitions() {
return Collections.unmodifiableSet(partitions);
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index b29843c..18e8e65 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -69,6 +69,8 @@ import
[asterixdb] branch master updated: [NO ISSUE][STO] Clean up invalid resources on bootstrap
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new fa2e379 [NO ISSUE][STO] Clean up invalid resources on bootstrap
fa2e379 is described below
commit fa2e379621640222f58902d6815c79307463d543
Author: Murtadha Hubail
AuthorDate: Wed Aug 11 02:31:13 2021 +0300
[NO ISSUE][STO] Clean up invalid resources on bootstrap
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add new node startup tasks to delete invalid resources.
- Ensure index max disk LSN is initialized during local
recovery even if the index is already registered.
Change-Id: Ia13182e96ec65951e837f71ffc4db3e92a43a7b0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12766
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../org/apache/asterix/app/nc/RecoveryManager.java | 28 ---
.../org/apache/asterix/app/nc/ReplicaManager.java | 6 +--
.../app/nc/task/LocalStorageCleanupTask.java | 55 ++
.../app/replication/NcLifecycleCoordinator.java| 5 +-
.../PersistentLocalResourceRepository.java | 18 +++
5 files changed, 102 insertions(+), 10 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 0359cf1..ffde7d0 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -160,7 +160,7 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
@Override
public void startLocalRecovery(Set partitions) throws
IOException, ACIDException {
state = SystemState.RECOVERING;
-LOGGER.info("starting recovery ...");
+LOGGER.info("starting recovery for partitions {}", partitions);
long readableSmallestLSN = logMgr.getReadableSmallestLSN();
Checkpoint checkpointObject = checkpointManager.getLatest();
@@ -363,10 +363,8 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
datasetLifecycleManager.register(localResource.getPath(), index);
datasetLifecycleManager.open(localResource.getPath());
try {
-final DatasetResourceReference
resourceReference =
-
DatasetResourceReference.of(localResource);
-maxDiskLastLsn =
-
indexCheckpointManagerProvider.get(resourceReference).getLowWatermark();
+maxDiskLastLsn =
getResourceLowWaterMark(localResource, datasetLifecycleManager,
+indexCheckpointManagerProvider);
} catch (HyracksDataException e) {
datasetLifecycleManager.close(localResource.getPath());
throw e;
@@ -374,7 +372,12 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
//#. set resourceId and maxDiskLastLSN to the
map
resourceId2MaxLSNMap.put(resourceId,
maxDiskLastLsn);
} else {
-maxDiskLastLsn =
resourceId2MaxLSNMap.get(resourceId);
+if
(!resourceId2MaxLSNMap.containsKey(resourceId)) {
+maxDiskLastLsn =
getResourceLowWaterMark(localResource, datasetLifecycleManager,
+indexCheckpointManagerProvider);
+} else {
+maxDiskLastLsn =
resourceId2MaxLSNMap.get(resourceId);
+}
}
// lsn @ maxDiskLastLsn is either a flush log or a
master replica log
if (lsn >= maxDiskLastLsn) {
@@ -858,6 +861,19 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
index.resetCurrentComponentIndex();
}
+private long getResourceLowWaterMark(LocalResource localResource,
IDatasetLifecycleManager datasetLifecycleManager,
+IIndexCheckpointManagerProvider indexCheckpointManagerProvider)
throws HyracksDataException {
+long maxDiskLastLsn;
+try {
+final DatasetResourceRefe
[asterixdb] branch master updated: [NO ISSUE][CLUS] Add getter to the cluster partitions map
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 91b555e [NO ISSUE][CLUS] Add getter to the cluster partitions map
91b555e is described below
commit 91b555eb42f3d8dc7879842ef470f8bd420f1188
Author: Murtadha Hubail
AuthorDate: Mon Aug 9 02:28:55 2021 +0300
[NO ISSUE][CLUS] Add getter to the cluster partitions map
- user model changes: no
- storage format changes: no
- interface changes: yes
Change-Id: Idd1872e3cbf3a95a6f4b51ed5d2135db041ebc31
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12723
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../org/apache/asterix/common/cluster/IClusterStateManager.java| 7 +++
.../java/org/apache/asterix/runtime/utils/ClusterStateManager.java | 5 +
2 files changed, 12 insertions(+)
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
index 2ee9435..41f786a 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
@@ -265,4 +265,11 @@ public interface IClusterStateManager {
* @param rebalanceRequired
*/
void setRebalanceRequired(boolean rebalanceRequired) throws
HyracksDataException;
+
+/**
+ * Gets a map of the cluster partitions
+ *
+ * @return the cluster partitions map
+ */
+Map getClusterPartitions();
}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index 7e50102..c5398e1 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -473,6 +473,11 @@ public class ClusterStateManager implements
IClusterStateManager {
refreshState();
}
+@Override
+public Map getClusterPartitions() {
+return Collections.unmodifiableMap(clusterPartitions);
+}
+
private void updateClusterCounters(String nodeId, NcLocalCounters
localCounters) {
final IResourceIdManager resourceIdManager =
appCtx.getResourceIdManager();
resourceIdManager.report(nodeId, localCounters.getMaxResourceId());
[asterixdb] branch master updated: [NO ISSUE][CLUS] Clone partition state
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5f11823 [NO ISSUE][CLUS] Clone partition state
5f11823 is described below
commit 5f118234d3253791ee3fa473979410c115290b0e
Author: Murtadha Hubail
AuthorDate: Sat Aug 7 02:58:42 2021 +0300
[NO ISSUE][CLUS] Clone partition state
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When cloning a cluster partition, ensure its current state is
cloned too.
Change-Id: Ic459de01b4bb2243625cd49c80181f25b63cce5f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12704
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../java/org/apache/asterix/common/cluster/ClusterPartition.java | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/ClusterPartition.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/ClusterPartition.java
index cc99421..504155e 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/ClusterPartition.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/ClusterPartition.java
@@ -71,7 +71,10 @@ public class ClusterPartition implements Cloneable {
@Override
public ClusterPartition clone() {
-return new ClusterPartition(partitionId, nodeId, ioDeviceNum);
+ClusterPartition clone = new ClusterPartition(partitionId, nodeId,
ioDeviceNum);
+clone.setPendingActivation(pendingActivation);
+clone.setActiveNodeId(activeNodeId);
+return clone;
}
@Override
[asterixdb] branch master updated: [NO ISSUE][REP] Fix logic of closing partition resources
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new c3bdeb0 [NO ISSUE][REP] Fix logic of closing partition resources
c3bdeb0 is described below
commit c3bdeb0ac6fddfa8a5e6fcb491d571cd2f0b3682
Author: Murtadha Hubail
AuthorDate: Wed Aug 4 19:54:56 2021 +0300
[NO ISSUE][REP] Fix logic of closing partition resources
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Fix logic of closing resources of a partition on node
partition release.
- Resolve files of negative partition ids to the first iodevice
in the node.
- Allow DatasetLocalResource partition to be set.
- Adapt test case.
Change-Id: I85cdd5cf9713880f248a535204b0738de723ab21
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12644
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../org/apache/asterix/app/nc/ReplicaManager.java | 5 ++-
.../asterix/hyracks/bootstrap/NCApplication.java | 6 ++-
.../release_partition/release_partition.1.sto.cmd | 2 +-
.../release_partition.2.pollget.http | 2 +-
.../release_partition.3.post.http | 2 +-
.../release_partition/release_partition.4.get.http | 2 +-
.../release_partition/release_partition.2.adm | 2 +-
.../common/api/IDatasetLifecycleManager.java | 13 ++
.../common/context/DatasetLifecycleManager.java| 46 ++
.../asterix/common/context/DatasetResource.java| 6 +++
.../common/dataflow/DatasetLocalResource.java | 6 ++-
.../PersistentLocalResourceRepository.java | 6 ++-
12 files changed, 88 insertions(+), 10 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 36e2d56..dd1b6e7 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -138,14 +138,15 @@ public class ReplicaManager implements IReplicaManager {
return replicaSyncLock;
}
-private void closePartitionResources(int partition) throws
HyracksDataException {
+public void closePartitionResources(int partition) throws
HyracksDataException {
final PersistentLocalResourceRepository resourceRepository =
(PersistentLocalResourceRepository)
appCtx.getLocalResourceRepository();
final Map partitionResources =
resourceRepository.getPartitionResources(partition);
final IDatasetLifecycleManager datasetLifecycleManager =
appCtx.getDatasetLifecycleManager();
for (LocalResource resource : partitionResources.values()) {
-datasetLifecycleManager.close(resource.getPath());
+datasetLifecycleManager.closeIfOpen(resource.getPath());
}
+datasetLifecycleManager.closePartition(partition);
}
private boolean isSelf(ReplicaIdentifier id) {
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 5b10512..b29843c 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -341,7 +341,11 @@ public class NCApplication extends BaseNCApplication {
@Override
public IFileDeviceResolver getFileDeviceResolver() {
return (relPath, devices) -> {
-int ioDeviceIndex =
Math.abs(StoragePathUtil.getPartitionNumFromRelativePath(relPath) %
devices.size());
+int partition =
StoragePathUtil.getPartitionNumFromRelativePath(relPath);
+if (partition < 0) {
+return devices.get(0);
+}
+int ioDeviceIndex = Math.abs(partition % devices.size());
return devices.get(ioDeviceIndex);
};
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd
index 7ddaa20..ee4684a 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/release_partition/release_partition.1.sto.cmd
@@ -16,4 +16,4 @@
* specific language gov
[asterixdb] branch master updated: [NO ISSUE][REP] Add node id to replica identifier
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 138d41d [NO ISSUE][REP] Add node id to replica identifier
138d41d is described below
commit 138d41d1fb1409c67724c47f38aa6776dc02282f
Author: Murtadha Hubail
AuthorDate: Thu Aug 5 22:29:31 2021 +0300
[NO ISSUE][REP] Add node id to replica identifier
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add node id to replica identifier.
- Adapt test cases.
Change-Id: I1cd37be58617f9b2c692f64659c5b9dcc5a8abda
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12683
Tested-by: Jenkins
Tested-by: Murtadha Hubail
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../apache/asterix/api/http/server/StorageApiServlet.java| 8 +---
.../main/java/org/apache/asterix/app/nc/ReplicaManager.java | 11 ++-
.../java/org/apache/asterix/test/common/TestExecutor.java| 5 +++--
.../results/replication/add_replica/add_replica.2.adm| 1 +
.../runtimets/results/replication/bulkload/bulkload.3.adm| 1 +
.../runtimets/results/replication/bulkload/bulkload.4.adm| 1 +
.../replication/flushed_component/flushed_component.2.adm| 1 +
.../mem_component_recovery/mem_component_recovery.3.adm | 1 +
.../mem_component_recovery/mem_component_recovery.4.adm | 1 +
.../replication/metadata_failover/metadata_failover.2.adm| 1 +
.../replication/release_partition/release_partition.2.adm| 1 +
.../org/apache/asterix/common/storage/ReplicaIdentifier.java | 12 +---
.../resync_failed_replica/resync_failed_replica.10.adm | 1 +
.../resync_failed_replica/resync_failed_replica.2.adm| 1 +
.../resync_failed_replica/resync_failed_replica.6.adm| 1 +
15 files changed, 30 insertions(+), 17 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
index d7d5994..26fee28 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
@@ -135,6 +135,7 @@ public class StorageApiServlet extends AbstractServlet {
final ObjectNode replicaJson =
OBJECT_MAPPER.createObjectNode();
replicaJson.put("location",
toHostPort(replica.getIdentifier().getLocation()));
replicaJson.put("status", replica.getStatus().toString());
+replicaJson.put("nodeId", replica.getIdentifier().getNodeId());
replicasArray.add(replicaJson);
}
partitionJson.set("replicas", replicasArray);
@@ -167,11 +168,12 @@ public class StorageApiServlet extends AbstractServlet {
final String partition = request.getParameter("partition");
final String host = request.getParameter("host");
final String port = request.getParameter("port");
-if (partition == null || host == null || port == null) {
+final String nodeId = request.getParameter("nodeId");
+if (partition == null || host == null || port == null || nodeId ==
null) {
return null;
}
-final InetSocketAddress replicaAddress = new InetSocketAddress(host,
Integer.valueOf(port));
-return ReplicaIdentifier.of(Integer.valueOf(partition),
replicaAddress);
+final InetSocketAddress replicaAddress = new InetSocketAddress(host,
Integer.parseInt(port));
+return ReplicaIdentifier.of(Integer.parseInt(partition), nodeId,
replicaAddress);
}
private void processPromote(IServletRequest request, IServletResponse
response) throws HyracksDataException {
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index ad70cf4..36e2d56 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -18,7 +18,6 @@
*/
package org.apache.asterix.app.nc;
-import java.net.InetSocketAddress;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -37,9 +36,7 @@ import
org.apache.asterix.common.transactions.IRecoveryManager;
import org.apache.asterix.replication.api.PartitionReplica;
import
org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
import org.apache.hyracks
[asterixdb] branch master updated: [NO ISSUE][CLUS] Add node active partitions config
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2f405b3 [NO ISSUE][CLUS] Add node active partitions config
2f405b3 is described below
commit 2f405b312c75f947cb947c3f915f10c2b7f812c0
Author: Murtadha Hubail
AuthorDate: Thu Aug 5 03:13:36 2021 +0300
[NO ISSUE][CLUS] Add node active partitions config
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add a new config (ACTIVE_PARTITIONS) that contains the current
list of active partitions on a node.
- By default, a node's active partitions list is the same as the
node's assigned partitions.
- Pass node active partitions to CC during bootstrap tasks.
- Adapt test cases.
Change-Id: Ia91e15897221f512aeeccbbe134f1d91db8aa629
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12663
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../apache/asterix/app/nc/NCAppRuntimeContext.java | 9 ++
.../app/replication/NcLifecycleCoordinator.java| 23 +++---
.../message/NCLifecycleTaskReportMessage.java | 11 ++-
.../message/RegistrationTasksRequestMessage.java | 15 ++---
.../message/RegistrationTasksResponseMessage.java | 5 ++-
.../asterix/hyracks/bootstrap/NCApplication.java | 3 +-
.../asterix/runtime/ClusterStateManagerTest.java | 21 -
.../common/cluster/IClusterStateManager.java | 4 ++-
.../asterix/common/config/MetadataProperties.java | 5 +++
.../asterix/common/config/NodeProperties.java | 5 +--
.../asterix/common/config/PropertiesAccessor.java | 13
.../asterix/runtime/utils/ClusterStateManager.java | 36 +-
12 files changed, 106 insertions(+), 44 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 9aa433f..eb8a92f 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -21,12 +21,10 @@ package org.apache.asterix.app.nc;
import java.io.IOException;
import java.rmi.RemoteException;
import java.rmi.server.UnicastRemoteObject;
-import java.util.Arrays;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
-import java.util.stream.Collectors;
import org.apache.asterix.active.ActiveManager;
import org.apache.asterix.common.api.IConfigValidator;
@@ -38,7 +36,6 @@ import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.api.IPropertiesFactory;
import org.apache.asterix.common.api.IReceptionist;
import org.apache.asterix.common.api.IReceptionistFactory;
-import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.config.ActiveProperties;
import org.apache.asterix.common.config.BuildProperties;
import org.apache.asterix.common.config.CompilerProperties;
@@ -225,10 +222,8 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
new DatasetLifecycleManager(storageProperties,
localResourceRepository, txnSubsystem.getLogManager(),
virtualBufferCache, indexCheckpointManagerProvider,
ioManager.getIODevices().size());
final String nodeId = getServiceContext().getNodeId();
-final ClusterPartition[] nodePartitions =
metadataProperties.getNodePartitions().get(nodeId);
-final Set nodePartitionsIds =
-
Arrays.stream(nodePartitions).map(ClusterPartition::getPartitionId).collect(Collectors.toSet());
-replicaManager = new ReplicaManager(this, nodePartitionsIds);
+final Set nodePartitions =
metadataProperties.getNodeActivePartitions(nodeId);
+replicaManager = new ReplicaManager(this, nodePartitions);
isShuttingdown = false;
activeManager = new ActiveManager(threadExecutor,
getServiceContext().getNodeId(),
activeProperties.getMemoryComponentGlobalBudget(),
compilerProperties.getFrameSize(),
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index c4e4f82..f164773 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
[asterixdb] branch master updated: [NO ISSUE][RT] Introduce query compilation lock
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new c5c6ec4 [NO ISSUE][RT] Introduce query compilation lock
c5c6ec4 is described below
commit c5c6ec478aa2a37134271240d1530c38513b2342
Author: Murtadha Hubail
AuthorDate: Wed Aug 4 02:48:34 2021 +0300
[NO ISSUE][RT] Introduce query compilation lock
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add query compilation lock that can be used to ensure
no queries are currently running in the cluster.
Change-Id: I663458df62040c0c01a13afde36b5cf765e57929
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12645
Integration-Tests: Jenkins
Tested-by: Murtadha Hubail
Reviewed-by: Murtadha Hubail
Reviewed-by: Ian Maxon
---
.../java/org/apache/asterix/app/translator/QueryTranslator.java | 7 +++
.../apache/asterix/common/dataflow/ICcApplicationContext.java| 9 +
.../org/apache/asterix/runtime/utils/CcApplicationContext.java | 7 +++
3 files changed, 23 insertions(+)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 267bd76..e79a4fd 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -42,6 +42,7 @@ import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import org.apache.asterix.active.ActivityState;
@@ -283,6 +284,7 @@ public class QueryTranslator extends AbstractLangTranslator
implements IStatemen
protected final IMetadataLockUtil lockUtil;
protected final IResponsePrinter responsePrinter;
protected final WarningCollector warningCollector;
+protected final ReentrantReadWriteLock compilationLock;
public QueryTranslator(ICcApplicationContext appCtx, List
statements, SessionOutput output,
ILangCompilationProvider compilationProvider, ExecutorService
executorService,
@@ -301,6 +303,7 @@ public class QueryTranslator extends AbstractLangTranslator
implements IStatemen
this.executorService = executorService;
this.responsePrinter = responsePrinter;
this.warningCollector = new WarningCollector();
+this.compilationLock = appCtx.getCompilationLock();
if
(appCtx.getServiceContext().getAppConfig().getBoolean(CCConfig.Option.ENFORCE_FRAME_WRITER_PROTOCOL))
{
this.jobFlags.add(JobFlag.ENFORCE_CONTRACT);
}
@@ -3392,12 +3395,14 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
final IMetadataLocker locker = new IMetadataLocker() {
@Override
public void lock() throws AlgebricksException {
+compilationLock.readLock().lock();
lockUtil.insertDeleteUpsertBegin(lockManager,
metadataProvider.getLocks(), dataverseName, datasetName);
}
@Override
public void unlock() {
metadataProvider.getLocks().unlock();
+compilationLock.readLock().unlock();
}
};
final IStatementCompiler compiler = () -> {
@@ -3987,6 +3992,7 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
final IMetadataLocker locker = new IMetadataLocker() {
@Override
public void lock() {
+compilationLock.readLock().lock();
}
@Override
@@ -3994,6 +4000,7 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
metadataProvider.getLocks().unlock();
// release external datasets' locks acquired during
compilation of the query
ExternalDatasetsRegistry.INSTANCE.releaseAcquiredLocks(metadataProvider);
+compilationLock.readLock().unlock();
}
};
final IStatementCompiler compiler = () -> {
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java
index ad90814..c22ac4c 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java
@@ -18,6
