[asterixdb] branch master updated: [ASTERIXDB-2958][IDX] Fix array index ingestion for datasets with meta fields
This is an automated email from the ASF dual-hosted git repository. ggalvizo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new 032d694 [ASTERIXDB-2958][IDX] Fix array index ingestion for datasets with meta fields 032d694 is described below commit 032d694548ad3972afdab5855a0422886d8704ec Author: ggalvizo AuthorDate: Thu Sep 9 19:16:37 2021 -0700 [ASTERIXDB-2958][IDX] Fix array index ingestion for datasets with meta fields - user model changes: no - storage format changes: no - interface changes: no Loading job for array index creation has been fixed to project away the meta field before performing the UNNEST on the record itself. This fixes the issue where the index was not being populated when an array index was created on a dataset with a meta field. Change-Id: I19859eb8a3becf44438afb1ad66b1ae672d21ff3 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13084 Reviewed-by: Glenn Galvizo Reviewed-by: Dmitry Lychagin Integration-Tests: Jenkins Tested-by: Jenkins --- asterixdb/asterix-app/data/yelp-checkin/feeds.csv | 3 ++ .../using-feed-new-index.1.ddl.sqlpp | 55 .../using-feed-new-index.2.update.sqlpp| 26 ++ .../using-feed-new-index.3.ddl.sqlpp | 23 + .../using-feed-new-index.4.query.sqlpp | 26 ++ .../using-feed-old-index.1.ddl.sqlpp | 58 ++ .../using-feed-old-index.2.update.sqlpp| 26 ++ .../using-feed-old-index.3.query.sqlpp | 26 ++ .../open/using-feed/using-feed.1.adm | 1 + .../test/resources/runtimets/testsuite_sqlpp.xml | 10 .../SecondaryArrayIndexBTreeOperationsHelper.java | 47 ++ 11 files changed, 291 insertions(+), 10 deletions(-) diff --git a/asterixdb/asterix-app/data/yelp-checkin/feeds.csv b/asterixdb/asterix-app/data/yelp-checkin/feeds.csv new file mode 100644 index 000..b7bd79c --- /dev/null +++ b/asterixdb/asterix-app/data/yelp-checkin/feeds.csv @@ -0,0 +1,3 @@ +1,"{ ""business_id"": ""--1UhMGODdWsrMastO9DZw"", ""dates"": [ ""2016-04-26 19:49:16"", ""2016-08-30 18:36:57"", ""2016-10-15 02:45:18"", ""2016-11-18 01:54:50"", ""2017-04-20 18:39:06"", ""2017-05-03 17:58:02"", ""2019-03-19 22:04:48"" ] }" +2,"{ ""business_id"": ""--EF5N7P70J_UYBTPypYlA"", ""dates"": [ ""2018-05-25 19:52:07"", ""2018-09-18 16:09:44"", ""2019-10-18 21:29:09"" ] }" +3,"{ ""business_id"": ""--Ni3oJ4VOqfOEu7Sj2Vzg"", ""dates"": [ ""2019-06-07 17:54:58"" ] }" \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/select-unnest-queries/open/using-feed-new-index/using-feed-new-index.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/select-unnest-queries/open/using-feed-new-index/using-feed-new-index.1.ddl.sqlpp new file mode 100644 index 000..3180b30 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/select-unnest-queries/open/using-feed-new-index/using-feed-new-index.1.ddl.sqlpp @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description: Verify that we are able to correctly perform the following types of queries using an array->atomic index with a feed (+ meta field). + * 1) UNNEST query, performing a point search. + */ + +DROP DATAVERSE TestYelp IF EXISTS; +CREATE DATAVERSE TestYelp; +USE TestYelp; + +CREATE TYPE CheckinType AS { +business_id: string +}; + +CREATE TYPE MetaType AS { +_id: string +}; + +CREATE DATASET YelpCheckin(CheckinType) +WITH META(MetaType) +PRIMARY KEY business_id; + +CREATE FEED YelpCheckinStream WITH { + "adapter-name" : "localfs", + "reader" : "localfs", + "parser" : "record-with-metadata", + "type-name" : "CheckinType", + "meta-type-name" : "MetaType", + "path" : "asterix_nc1://data/yelp-checkin/feeds.csv", + "format": "csv", + "delimiter": ",", + "record-format": "adm", + "record-index" :
[asterixdb] branch master updated: [NO ISSUE][ACTIVE] Account for force stop while suspending
This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new eed8714 [NO ISSUE][ACTIVE] Account for force stop while suspending eed8714 is described below commit eed8714ae56bd61656750bf543181e7dd68c26c1 Author: Murtadha Hubail AuthorDate: Fri Sep 10 01:36:34 2021 +0300 [NO ISSUE][ACTIVE] Account for force stop while suspending - user model changes: no - storage format changes: no - interface changes: no Details: - When a failure happens while trying to suspend ingestion, we will force stop the active job. If the job completes ungracefully, we set the listener state to TEMPORARILY_FAILED. However, since force to stop only waits for STOPPED state, the thread waiting for ingestion to be suspended will wait forever. This change accounts for such case and makes the force stop waits for TEMPORARILY_FAILED too. Change-Id: Ib33f191be2b84d97a08e3bc6d607b0edbf35bed1 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13144 Tested-by: Jenkins Integration-Tests: Jenkins Reviewed-by: Murtadha Hubail Reviewed-by: Ali Alsuliman --- .../org/apache/asterix/app/active/ActiveEntityEventsListener.java | 8 +--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java index 0242ecd..ddd3d64 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveEntityEventsListener.java @@ -195,7 +195,7 @@ public abstract class ActiveEntityEventsListener implements IActiveEntityControl @SuppressWarnings("unchecked") protected void finish(ActiveEvent event) throws HyracksDataException { if (LOGGER.isEnabled(level)) { -LOGGER.log(level, "the job " + jobId + " finished"); +LOGGER.log(level, "the job {} finished", jobId); } JobId lastJobId = jobId; if (numRegistered != numDeRegistered) { @@ -208,7 +208,7 @@ public abstract class ActiveEntityEventsListener implements IActiveEntityControl JobStatus jobStatus = status.getLeft(); List exceptions = status.getRight(); if (LOGGER.isEnabled(level)) { -LOGGER.log(level, "The job finished with status: " + jobStatus); +LOGGER.log(level, "The job finished with status: {}", jobStatus); } if (!jobSuccessfullyTerminated(jobStatus)) { jobFailure = exceptions.isEmpty() ? new RuntimeDataException(ErrorCode.UNREPORTED_TASK_FAILURE_EXCEPTION) @@ -440,8 +440,9 @@ public abstract class ActiveEntityEventsListener implements IActiveEntityControl private void cancelJob(Throwable th) { cancelJobSafely(metadataProvider, th); +// we can come here due to a failure while in suspending state final WaitForStateSubscriber cancelSubscriber = -new WaitForStateSubscriber(this, EnumSet.of(ActivityState.STOPPED)); +new WaitForStateSubscriber(this, EnumSet.of(ActivityState.STOPPED, ActivityState.TEMPORARILY_FAILED)); final Span span = Span.start(2, TimeUnit.MINUTES); InvokeUtil.doUninterruptibly(() -> { if (!cancelSubscriber.sync(span)) { @@ -491,6 +492,7 @@ public abstract class ActiveEntityEventsListener implements IActiveEntityControl forceStop(subscriber, ie); Thread.currentThread().interrupt(); } catch (Throwable e) { +LOGGER.error("forcing active job stop due to", e); forceStop(subscriber, e); } finally { Thread.currentThread().setName(nameBefore);
[asterixdb] branch master updated: [NO ISSUE][HYR] Eliminate reflection when obtaining pid from RuntimeMXBean
This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git The following commit(s) were added to refs/heads/master by this push: new 8550e06 [NO ISSUE][HYR] Eliminate reflection when obtaining pid from RuntimeMXBean 8550e06 is described below commit 8550e06c533800ae8b4b98a3159e9c0dad3a3eac Author: Michael Blow AuthorDate: Thu Sep 9 23:21:10 2021 -0400 [NO ISSUE][HYR] Eliminate reflection when obtaining pid from RuntimeMXBean Change-Id: I880f9e3bedafc2b9ebd2122fcc2f316bab696bf4 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13146 Reviewed-by: Michael Blow Reviewed-by: Hussain Towaileb Integration-Tests: Jenkins Tested-by: Jenkins --- .../main/java/org/apache/hyracks/util/PidHelper.java | 19 +-- 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java index 46e77e3..3cba2d3 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/PidHelper.java @@ -20,9 +20,6 @@ package org.apache.hyracks.util; import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -40,22 +37,8 @@ public class PidHelper { } public static long getPid(RuntimeMXBean runtimeMXBean) { -// TODO: replace with direct invoke of getPid() once compatibility is at JDK 10 or higher try { -Method getPidMethod = runtimeMXBean.getClass().getMethod("getPid"); -return (Long) getPidMethod.invoke(runtimeMXBean); -} catch (NoSuchMethodException e) { -LOGGER.debug("ignoring exception trying to find getPid() (expected pre-JDK 10)", e); -} catch (IllegalAccessException | InvocationTargetException e) { -LOGGER.debug("ignoring exception trying to execute getPid()", e); -} -try { -Field jvmField = runtimeMXBean.getClass().getDeclaredField("jvm"); -jvmField.setAccessible(true); -Object vmManagement = jvmField.get(runtimeMXBean); -Method getProcessIdMethod = vmManagement.getClass().getDeclaredMethod("getProcessId"); -getProcessIdMethod.setAccessible(true); -return (Integer) getProcessIdMethod.invoke(vmManagement); +return runtimeMXBean.getPid(); } catch (Exception e) { LOGGER.log(Level.INFO, "Unable to determine PID due to exception", e); return -1;