[asterixdb] branch master updated: [ASTERIXDB-2959][IDX] Fix initial composite atomic-array index ingestion on creation
This is an automated email from the ASF dual-hosted git repository.
ggalvizo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2ee6d37 [ASTERIXDB-2959][IDX] Fix initial composite atomic-array
index ingestion on creation
2ee6d37 is described below
commit 2ee6d37456c6149cd088d236b12bcb6701d2004f
Author: ggalvizo
AuthorDate: Fri Sep 10 12:58:38 2021 -0700
[ASTERIXDB-2959][IDX] Fix initial composite atomic-array index ingestion on
creation
- user model changes: no
- storage format changes: no
- interface changes: no
Composite atomic-array indexes were not ingesting properly, due to the
wrong source column being UNNESTed in the loading job. This small change
addresses this.
Change-Id: I86af26a7649e0631a0555e1f98fb5fea8e63528e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13163
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Dmitry Lychagin
---
.../composite-atomic/composite-atomic.1.ddl.sqlpp | 25
.../composite-atomic.2.update.sqlpp| 27 ++
.../composite-atomic/composite-atomic.3.ddl.sqlpp | 26 +
.../composite-atomic.4.query.sqlpp | 25
.../open/composite-atomic/composite-atomic.1.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 9 ++--
.../asterix/metadata/utils/ArrayIndexUtil.java | 2 +-
7 files changed, 112 insertions(+), 3 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.ddl.sqlpp
new file mode 100644
index 000..6312915
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.1.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSETestDataverse IF EXISTS;
+CREATE DATAVERSE TestDataverse;
+USE TestDataverse;
+
+CREATE TYPE GenericType AS { _id: bigint };
+CREATE DATASETTestDataset (GenericType)
+PRIMARY KEY _id;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.2.update.sqlpp
new file mode 100644
index 000..e901b37
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/bulk-loading/on-index-creation/open/composite-atomic/composite-atomic.2.update.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE TestDataverse;
+
+INSERT INTO TestDataset [
+{ "_id": 100, "f": 1, "a": {"b": [{"c": {"d": 1}}]} },
+{ "_id": 101, "f": "a", "a": null },
+{ "_id": 102, "f": "a", "a": {"b": null} },
+{ "_id": 103, "f": "a", "a": {"b": [{"c": null}]} },
+{ "_id": 104, "f": "a", "a":
[asterixdb] branch master updated: [NO ISSUE][REP] Ensure files delete order from replicas
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new d8a1a93 [NO ISSUE][REP] Ensure files delete order from replicas
d8a1a93 is described below
commit d8a1a93f99ce7c789dcc78afa1df3398d833cb15
Author: Murtadha Hubail
AuthorDate: Tue Sep 14 21:21:17 2021 +0300
[NO ISSUE][REP] Ensure files delete order from replicas
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When deleting files from a replica, delete the resource
files last to ensure no orphaned component files if
the replica fails after deleting the resource file.
- When replicating a resource file, ensure the index
directory is empty.
Change-Id: I7dbb9960cb644c063de49dd0920a1709964de2f8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13224
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java | 1 -
.../org/apache/asterix/replication/messaging/ReplicateFileTask.java | 5 +
.../apache/asterix/replication/sync/ReplicaFilesSynchronizer.java| 5 +
3 files changed, 10 insertions(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
index a926c66..dd7a663 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
@@ -46,7 +46,6 @@ public class LocalStorageCleanupTask implements
INCLifecycleTask {
deleteInvalidMetadataIndexes(localResourceRepository);
final Set nodePartitions =
appContext.getReplicaManager().getPartitions();
localResourceRepository.deleteCorruptedResources();
-//TODO optimize this to cleanup all active partitions at once
for (Integer partition : nodePartitions) {
localResourceRepository.cleanup(partition);
}
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
index 7f26b96..bf1613c 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/ReplicateFileTask.java
@@ -39,6 +39,7 @@ import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.replication.api.IReplicaTask;
import org.apache.asterix.replication.api.IReplicationWorker;
import org.apache.asterix.replication.management.NetworkingUtil;
+import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -72,6 +73,10 @@ public class ReplicateFileTask implements IReplicaTask {
// resolve path
final FileReference localPath = ioManager.resolve(file);
final Path resourceDir =
Files.createDirectories(localPath.getFile().getParentFile().toPath());
+if (indexMetadata) {
+// ensure clean index directory
+FileUtils.cleanDirectory(resourceDir.toFile());
+}
// create mask
final Path maskPath = Paths.get(resourceDir.toString(),
StorageConstants.MASK_FILE_PREFIX +
localPath.getFile().getName());
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java
index 735318d..477559c 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaFilesSynchronizer.java
@@ -21,6 +21,7 @@ package org.apache.asterix.replication.sync;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -127,6 +128,10 @@ public class ReplicaFilesSynchronizer {
private void deleteInvalidFiles(List files) {
final FileSynchronizer sync = new FileSynchronizer(appCtx, replica);
+// sort files to ensure index metadata files starting with "."
[asterixdb] branch master updated: [NO ISSUE][REP] Add API to perform non-delta recovery for a replica
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 8f278c0 [NO ISSUE][REP] Add API to perform non-delta recovery for a
replica
8f278c0 is described below
commit 8f278c042d92135f737cd7b26bd56a3479e11106
Author: Murtadha Hubail
AuthorDate: Tue Sep 14 02:12:15 2021 +0300
[NO ISSUE][REP] Add API to perform non-delta recovery for a replica
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add an option to perform non-delta recovery for a replica.
Change-Id: Ib1837e8f1aefdd9e085ccfd62f1c6e6d4eb969e8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13223
Integration-Tests: Jenkins
Tested-by: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../asterix/replication/api/PartitionReplica.java | 6 +-
.../replication/messaging/DeletePartitionTask.java | 75 ++
.../messaging/PartitionResourcesListResponse.java | 2 +-
.../replication/messaging/ReplicationProtocol.java | 5 +-
.../replication/sync/ReplicaFilesSynchronizer.java | 14 +++-
.../replication/sync/ReplicaSynchronizer.java | 8 +--
.../PersistentLocalResourceRepository.java | 12
7 files changed, 112 insertions(+), 10 deletions(-)
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
index e265d03..3b10700 100644
---
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/api/PartitionReplica.java
@@ -79,17 +79,17 @@ public class PartitionReplica implements IPartitionReplica {
}
public synchronized void sync() {
-sync(true);
+sync(true, true);
}
-public synchronized void sync(boolean register) {
+public synchronized void sync(boolean register, boolean deltaRecovery) {
if (status == IN_SYNC || status == CATCHING_UP) {
return;
}
setStatus(CATCHING_UP);
appCtx.getThreadExecutor().execute(() -> {
try {
-new ReplicaSynchronizer(appCtx, this).sync(register);
+new ReplicaSynchronizer(appCtx, this).sync(register,
deltaRecovery);
setStatus(IN_SYNC);
} catch (Exception e) {
LOGGER.error(() -> "Failed to sync replica " + this, e);
diff --git
a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/DeletePartitionTask.java
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/DeletePartitionTask.java
new file mode 100644
index 000..90139df
--- /dev/null
+++
b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/DeletePartitionTask.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.replication.messaging;
+
+import java.io.DataInput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.exceptions.ReplicationException;
+import org.apache.asterix.replication.api.IReplicaTask;
+import org.apache.asterix.replication.api.IReplicationWorker;
+import
org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class DeletePartitionTask implements IReplicaTask {
+
+private static final Logger LOGGER = LogManager.getLogger();
+private final int partitionId;
+
+public DeletePartitionTask(int partitionId) {
+this.partitionId = partitionId;
+}
+
+@Override
+public
[asterixdb] branch master updated: [NO ISSUE][ACTIVE] Account for force stop while suspending
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5e11053 [NO ISSUE][ACTIVE] Account for force stop while suspending
new bc8aaf6 Merge branch 'cheshire-cat' into master
5e11053 is described below
commit 5e110538870c9385f316e0395b80a76f0e6d8e7c
Author: Murtadha Hubail
AuthorDate: Fri Sep 10 01:36:34 2021 +0300
[NO ISSUE][ACTIVE] Account for force stop while suspending
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When a failure happens while trying to suspend ingestion,
we will force stop the active job. If the job completes
ungracefully, we set the listener state to TEMPORARILY_FAILED.
However, since force to stop only waits for STOPPED state,
the thread waiting for ingestion to be suspended will wait
forever. This change accounts for such case and makes
the force stop waits for TEMPORARILY_FAILED too.
Change-Id: Ib33f191be2b84d97a08e3bc6d607b0edbf35bed1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13144
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
(cherry picked from commit eed8714ae56bd61656750bf543181e7dd68c26c1)
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13203
---
.../org/apache/asterix/app/active/ActiveEntityEventsListener.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 0242ecd..ddd3d64 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -195,7 +195,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
@SuppressWarnings("unchecked")
protected void finish(ActiveEvent event) throws HyracksDataException {
if (LOGGER.isEnabled(level)) {
-LOGGER.log(level, "the job " + jobId + " finished");
+LOGGER.log(level, "the job {} finished", jobId);
}
JobId lastJobId = jobId;
if (numRegistered != numDeRegistered) {
@@ -208,7 +208,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
JobStatus jobStatus = status.getLeft();
List exceptions = status.getRight();
if (LOGGER.isEnabled(level)) {
-LOGGER.log(level, "The job finished with status: " + jobStatus);
+LOGGER.log(level, "The job finished with status: {}", jobStatus);
}
if (!jobSuccessfullyTerminated(jobStatus)) {
jobFailure = exceptions.isEmpty() ? new
RuntimeDataException(ErrorCode.UNREPORTED_TASK_FAILURE_EXCEPTION)
@@ -440,8 +440,9 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
private void cancelJob(Throwable th) {
cancelJobSafely(metadataProvider, th);
+// we can come here due to a failure while in suspending state
final WaitForStateSubscriber cancelSubscriber =
-new WaitForStateSubscriber(this,
EnumSet.of(ActivityState.STOPPED));
+new WaitForStateSubscriber(this,
EnumSet.of(ActivityState.STOPPED, ActivityState.TEMPORARILY_FAILED));
final Span span = Span.start(2, TimeUnit.MINUTES);
InvokeUtil.doUninterruptibly(() -> {
if (!cancelSubscriber.sync(span)) {
@@ -491,6 +492,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
forceStop(subscriber, ie);
Thread.currentThread().interrupt();
} catch (Throwable e) {
+LOGGER.error("forcing active job stop due to", e);
forceStop(subscriber, e);
} finally {
Thread.currentThread().setName(nameBefore);
