[asterixdb] branch master updated: [ASTERIXDB-2958][IDX] Fix array index ingestion for datasets with meta fields
This is an automated email from the ASF dual-hosted git repository.
ggalvizo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 032d694 [ASTERIXDB-2958][IDX] Fix array index ingestion for datasets
with meta fields
032d694 is described below
commit 032d694548ad3972afdab5855a0422886d8704ec
Author: ggalvizo
AuthorDate: Thu Sep 9 19:16:37 2021 -0700
[ASTERIXDB-2958][IDX] Fix array index ingestion for datasets with meta
fields
- user model changes: no
- storage format changes: no
- interface changes: no
Loading job for array index creation has been fixed to project away the
meta field before performing the UNNEST on the record itself. This fixes
the issue where the index was not being populated when an array index
was created on a dataset with a meta field.
Change-Id: I19859eb8a3becf44438afb1ad66b1ae672d21ff3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13084
Reviewed-by: Glenn Galvizo
Reviewed-by: Dmitry Lychagin
Integration-Tests: Jenkins
Tested-by: Jenkins
---
asterixdb/asterix-app/data/yelp-checkin/feeds.csv | 3 ++
.../using-feed-new-index.1.ddl.sqlpp | 55
.../using-feed-new-index.2.update.sqlpp| 26 ++
.../using-feed-new-index.3.ddl.sqlpp | 23 +
.../using-feed-new-index.4.query.sqlpp | 26 ++
.../using-feed-old-index.1.ddl.sqlpp | 58 ++
.../using-feed-old-index.2.update.sqlpp| 26 ++
.../using-feed-old-index.3.query.sqlpp | 26 ++
.../open/using-feed/using-feed.1.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 10
.../SecondaryArrayIndexBTreeOperationsHelper.java | 47 ++
11 files changed, 291 insertions(+), 10 deletions(-)
diff --git a/asterixdb/asterix-app/data/yelp-checkin/feeds.csv
b/asterixdb/asterix-app/data/yelp-checkin/feeds.csv
new file mode 100644
index 000..b7bd79c
--- /dev/null
+++ b/asterixdb/asterix-app/data/yelp-checkin/feeds.csv
@@ -0,0 +1,3 @@
+1,"{ ""business_id"": ""--1UhMGODdWsrMastO9DZw"", ""dates"": [ ""2016-04-26
19:49:16"", ""2016-08-30 18:36:57"", ""2016-10-15 02:45:18"", ""2016-11-18
01:54:50"", ""2017-04-20 18:39:06"", ""2017-05-03 17:58:02"", ""2019-03-19
22:04:48"" ] }"
+2,"{ ""business_id"": ""--EF5N7P70J_UYBTPypYlA"", ""dates"": [ ""2018-05-25
19:52:07"", ""2018-09-18 16:09:44"", ""2019-10-18 21:29:09"" ] }"
+3,"{ ""business_id"": ""--Ni3oJ4VOqfOEu7Sj2Vzg"", ""dates"": [ ""2019-06-07
17:54:58"" ] }"
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/select-unnest-queries/open/using-feed-new-index/using-feed-new-index.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/select-unnest-queries/open/using-feed-new-index/using-feed-new-index.1.ddl.sqlpp
new file mode 100644
index 000..3180b30
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/select-unnest-queries/open/using-feed-new-index/using-feed-new-index.1.ddl.sqlpp
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: Verify that we are able to correctly perform the following
types of queries using an array->atomic index with a feed (+ meta field).
+ * 1) UNNEST query, performing a point search.
+ */
+
+DROP DATAVERSE TestYelp IF EXISTS;
+CREATE DATAVERSE TestYelp;
+USE TestYelp;
+
+CREATE TYPE CheckinType AS {
+business_id: string
+};
+
+CREATE TYPE MetaType AS {
+_id: string
+};
+
+CREATE DATASET YelpCheckin(CheckinType)
+WITH META(MetaType)
+PRIMARY KEY business_id;
+
+CREATE FEED YelpCheckinStream WITH {
+ "adapter-name" : "localfs",
+ "reader" : "localfs",
+ "parser" : "record-with-metadata",
+ "type-name" : "CheckinType",
+ "meta-type-name" : "MetaType",
+ "path" : "asterix_nc1://data/yelp-checkin/feeds.csv",
+ "format": "csv",
+ "delimiter": ",",
+ "record-format": "adm",
+ "record-index" :
[asterixdb] branch master updated: [NO ISSUE][ACTIVE] Account for force stop while suspending
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new eed8714 [NO ISSUE][ACTIVE] Account for force stop while suspending
eed8714 is described below
commit eed8714ae56bd61656750bf543181e7dd68c26c1
Author: Murtadha Hubail
AuthorDate: Fri Sep 10 01:36:34 2021 +0300
[NO ISSUE][ACTIVE] Account for force stop while suspending
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When a failure happens while trying to suspend ingestion,
we will force stop the active job. If the job completes
ungracefully, we set the listener state to TEMPORARILY_FAILED.
However, since force to stop only waits for STOPPED state,
the thread waiting for ingestion to be suspended will wait
forever. This change accounts for such case and makes
the force stop waits for TEMPORARILY_FAILED too.
Change-Id: Ib33f191be2b84d97a08e3bc6d607b0edbf35bed1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13144
Tested-by: Jenkins
Integration-Tests: Jenkins
Reviewed-by: Murtadha Hubail
Reviewed-by: Ali Alsuliman
---
.../org/apache/asterix/app/active/ActiveEntityEventsListener.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
index 0242ecd..ddd3d64 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java
@@ -195,7 +195,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
@SuppressWarnings("unchecked")
protected void finish(ActiveEvent event) throws HyracksDataException {
if (LOGGER.isEnabled(level)) {
-LOGGER.log(level, "the job " + jobId + " finished");
+LOGGER.log(level, "the job {} finished", jobId);
}
JobId lastJobId = jobId;
if (numRegistered != numDeRegistered) {
@@ -208,7 +208,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
JobStatus jobStatus = status.getLeft();
List exceptions = status.getRight();
if (LOGGER.isEnabled(level)) {
-LOGGER.log(level, "The job finished with status: " + jobStatus);
+LOGGER.log(level, "The job finished with status: {}", jobStatus);
}
if (!jobSuccessfullyTerminated(jobStatus)) {
jobFailure = exceptions.isEmpty() ? new
RuntimeDataException(ErrorCode.UNREPORTED_TASK_FAILURE_EXCEPTION)
@@ -440,8 +440,9 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
private void cancelJob(Throwable th) {
cancelJobSafely(metadataProvider, th);
+// we can come here due to a failure while in suspending state
final WaitForStateSubscriber cancelSubscriber =
-new WaitForStateSubscriber(this,
EnumSet.of(ActivityState.STOPPED));
+new WaitForStateSubscriber(this,
EnumSet.of(ActivityState.STOPPED, ActivityState.TEMPORARILY_FAILED));
final Span span = Span.start(2, TimeUnit.MINUTES);
InvokeUtil.doUninterruptibly(() -> {
if (!cancelSubscriber.sync(span)) {
@@ -491,6 +492,7 @@ public abstract class ActiveEntityEventsListener implements
IActiveEntityControl
forceStop(subscriber, ie);
Thread.currentThread().interrupt();
} catch (Throwable e) {
+LOGGER.error("forcing active job stop due to", e);
forceStop(subscriber, e);
} finally {
Thread.currentThread().setName(nameBefore);
[asterixdb] branch master updated: [NO ISSUE][HYR] Eliminate reflection when obtaining pid from RuntimeMXBean
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 8550e06 [NO ISSUE][HYR] Eliminate reflection when obtaining pid from
RuntimeMXBean
8550e06 is described below
commit 8550e06c533800ae8b4b98a3159e9c0dad3a3eac
Author: Michael Blow
AuthorDate: Thu Sep 9 23:21:10 2021 -0400
[NO ISSUE][HYR] Eliminate reflection when obtaining pid from RuntimeMXBean
Change-Id: I880f9e3bedafc2b9ebd2122fcc2f316bab696bf4
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13146
Reviewed-by: Michael Blow
Reviewed-by: Hussain Towaileb
Integration-Tests: Jenkins
Tested-by: Jenkins
---
.../main/java/org/apache/hyracks/util/PidHelper.java | 19 +--
1 file changed, 1 insertion(+), 18 deletions(-)
diff --git
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
index 46e77e3..3cba2d3 100644
---
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
+++
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java
@@ -20,9 +20,6 @@ package org.apache.hyracks.util;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -40,22 +37,8 @@ public class PidHelper {
}
public static long getPid(RuntimeMXBean runtimeMXBean) {
-// TODO: replace with direct invoke of getPid() once compatibility is
at JDK 10 or higher
try {
-Method getPidMethod = runtimeMXBean.getClass().getMethod("getPid");
-return (Long) getPidMethod.invoke(runtimeMXBean);
-} catch (NoSuchMethodException e) {
-LOGGER.debug("ignoring exception trying to find getPid() (expected
pre-JDK 10)", e);
-} catch (IllegalAccessException | InvocationTargetException e) {
-LOGGER.debug("ignoring exception trying to execute getPid()", e);
-}
-try {
-Field jvmField = runtimeMXBean.getClass().getDeclaredField("jvm");
-jvmField.setAccessible(true);
-Object vmManagement = jvmField.get(runtimeMXBean);
-Method getProcessIdMethod =
vmManagement.getClass().getDeclaredMethod("getProcessId");
-getProcessIdMethod.setAccessible(true);
-return (Integer) getProcessIdMethod.invoke(vmManagement);
+return runtimeMXBean.getPid();
} catch (Exception e) {
LOGGER.log(Level.INFO, "Unable to determine PID due to exception",
e);
return -1;
