>From Michael Blow <[email protected]>: Michael Blow has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10564 )
Change subject: [NO ISSUE][*DB][EXT] FeedRecordDataFlowController stats += timestamp ...................................................................... [NO ISSUE][*DB][EXT] FeedRecordDataFlowController stats += timestamp also, += ThrowingConsumer.forEach() utility method Change-Id: I53ed2f4a9637de00b90a4d7c637653d440cd3740 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10564 Reviewed-by: Michael Blow <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Tested-by: Michael Blow <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> --- D asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.5.regexjson M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java D asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DataverseNameUtils.java M hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java 5 files changed, 33 insertions(+), 99 deletions(-) Approvals: Murtadha Hubail: Looks good to me, approved Michael Blow: Looks good to me, but someone else must approve; Verified Jenkins: Verified; Verified diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm deleted file mode 100644 index 0fcdf15..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.1.adm +++ /dev/null @@ -1,10 +0,0 @@ -{ - "experiments.TweetFeed(Feed)" : { - "Stats" : [ { - "adapter-stats" : { - "incoming-records-count" : 13, - "failed-at-parser-records-count" : 3 - } - } ] - } -} \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.5.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.5.regexjson new file mode 100644 index 0000000..54028c3 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/feed-stats/feed-stats.5.regexjson @@ -0,0 +1,13 @@ +{ + "experiments.TweetFeed(Feed)": { + "Stats": [ + { + "adapter-stats": { + "timestamp": "R{[0-9]+}", + "incoming-records-count": 13, + "failed-at-parser-records-count": 3 + } + } + ] + } +} diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java index 3fb7db7..56257a8 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java @@ -42,6 +42,7 @@ public static final String INCOMING_RECORDS_COUNT_FIELD_NAME = "incoming-records-count"; public static final String FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME = "failed-at-parser-records-count"; public static final String READER_STATS_FIELD_NAME = "reader-stats"; + public static final String TIMESTAMP_FIELD_NAME = "timestamp"; public enum State { CREATED, @@ -271,11 +272,12 @@ StringBuilder str = new StringBuilder(); str.append("{"); if (readerStats != null) { - str.append("\"").append(READER_STATS_FIELD_NAME).append("\":").append(readerStats).append(", "); + str.append("\"" + READER_STATS_FIELD_NAME + "\":").append(readerStats).append(","); } - str.append("\"").append(INCOMING_RECORDS_COUNT_FIELD_NAME).append("\": ").append(incomingRecordsCount) - .append(", \"").append(FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME).append("\": ") - .append(failedRecordsCount).append("}"); + str.append("\"" + TIMESTAMP_FIELD_NAME + "\":").append(System.currentTimeMillis()).append(","); + str.append("\"" + INCOMING_RECORDS_COUNT_FIELD_NAME + "\":").append(incomingRecordsCount) + .append(",\"" + FAILED_AT_PARSER_RECORDS_COUNT_FIELD_NAME + "\":").append(failedRecordsCount) + .append("}"); return str.toString(); } } diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DataverseNameUtils.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DataverseNameUtils.java deleted file mode 100644 index 1a3f26a..0000000 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DataverseNameUtils.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.lang.common.util; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.asterix.common.metadata.DataverseName; -import org.apache.asterix.lang.common.visitor.FormatPrintVisitor; - -public class DataverseNameUtils { - - static protected Set<Character> validIdentifierChars = new HashSet<>(); - static protected Set<Character> validIdentifierStartChars = new HashSet<>(); - - static { - for (char ch = 'a'; ch <= 'z'; ++ch) { - validIdentifierChars.add(ch); - validIdentifierStartChars.add(ch); - } - for (char ch = 'A'; ch <= 'Z'; ++ch) { - validIdentifierChars.add(ch); - validIdentifierStartChars.add(ch); - } - for (char ch = '0'; ch <= '9'; ++ch) { - validIdentifierChars.add(ch); - } - validIdentifierChars.add('_'); - validIdentifierChars.add('$'); - } - - protected static boolean needQuotes(String str) { - if (str.length() == 0) { - return false; - } - if (!validIdentifierStartChars.contains(str.charAt(0))) { - return true; - } - for (char ch : str.toCharArray()) { - if (!validIdentifierChars.contains(ch)) { - return true; - } - } - return false; - } - - protected static String normalize(String str) { - if (needQuotes(str)) { - return FormatPrintVisitor.revertStringToQuoted(str); - } - return str; - } - - public static String generateDataverseName(DataverseName dataverseName) { - List<String> dataverseNameParts = new ArrayList<>(); - StringBuilder sb = new StringBuilder(); - dataverseNameParts.clear(); - dataverseName.getParts(dataverseNameParts); - for (int i = 0, ln = dataverseNameParts.size(); i < ln; i++) { - if (i > 0) { - sb.append(DataverseName.CANONICAL_FORM_SEPARATOR_CHAR); - } - sb.append(normalize(dataverseNameParts.get(i))); - } - return sb.toString(); - } -} diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java index 4f73680..36b5cf0 100644 --- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingConsumer.java @@ -46,4 +46,18 @@ return null; }; } + + static <R> void forEach(Iterable<R> iterable, ThrowingConsumer<R> consumer) throws Exception { + try { + iterable.forEach(value -> { + try { + consumer.process(value); + } catch (Exception e) { + throw new UncheckedExecutionException(e); + } + }); + } catch (UncheckedExecutionException e) { + throw (Exception) e.getCause(); + } + } } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10564 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: cheshire-cat Gerrit-Change-Id: I53ed2f4a9637de00b90a4d7c637653d440cd3740 Gerrit-Change-Number: 10564 Gerrit-PatchSet: 7 Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-MessageType: merged
