[ASTERIXDB-1983] Feed pipeline refactoring for SQL++ - user model changes: no - storage format changes: no - interface changes: no
Current implementation of feed uses handcraft AQL queries for creating feed pipeline. This causes a lot of issues and does not support SQL++ very well. Also, there is an overhead for parsing the query everytime. In this patch, it's replaced with compiled statement in SQL++ which provides support for attaching UDF to feed as well. Details: 1. Remove SubscribeFeedStatement. 2. Remove SubscribeFeed related query compilation code, and reuse the upsert dataflow. 3. Added SQL++ User Defined Function support for feed, including adding multiple functions to one feed. 4. Related test cases added. 5. Change the default behavior of feed to be upsert instead of upsert. 'insert-feed' option is provided for experiment uses. 6. This patch also fixes several feed related bugs: [ASTERIXDB-2085] [ASTERIXDB-2124]. Change-Id: I0ae5a837613780a4d2c90c98139fdc6d5e040cc9 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2059 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: abdullah alamoudi <bamou...@gmail.com> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/7e76a079 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/7e76a079 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/7e76a079 Branch: refs/heads/master Commit: 7e76a0797388d5c8f92a96c7fa6b4cf30c0886f0 Parents: 799046d Author: Xikui Wang <xkk...@gmail.com> Authored: Fri Oct 20 13:12:30 2017 -0700 Committer: Xikui Wang <xkk...@gmail.com> Committed: Sat Oct 21 09:10:23 2017 -0700 ---------------------------------------------------------------------- .../asterix/translator/CompiledStatements.java | 82 -------- .../LangExpressionToPlanTranslator.java | 119 +----------- .../apache/asterix/api/common/APIFramework.java | 4 +- .../asterix/app/active/FeedEventsListener.java | 16 +- .../asterix/app/translator/QueryTranslator.java | 7 +- .../apache/asterix/utils/FeedOperations.java | 163 ++++++++++++---- .../cross-dv15/cross-dv15.3.query.aql | 3 +- .../connect-feed-with-aql-function.1.ddl.aql | 57 ------ .../connect-feed-with-aql-function.2.update.aql | 32 ---- .../connect-feed-with-aql-function.3.server.aql | 27 --- .../connect-feed-with-aql-function.4.sleep.aql | 27 --- .../connect-feed-with-aql-function.5.update.aql | 29 --- .../connect-feed-with-aql-function.6.query.aql | 31 --- .../connect-feed-with-aql-function.7.server.aql | 28 --- .../connect-feed-with-aql-function.8.ddl.aql | 28 --- .../connect-feed-with-function.1.ddl.aql | 57 ++++++ .../connect-feed-with-function.2.update.aql | 32 ++++ .../connect-feed-with-function.3.ddl.aql | 27 +++ .../feeds/insert-feed/insert-feed.1.ddl.aql | 45 +++++ .../feeds/insert-feed/insert-feed.2.update.aql | 24 +++ .../feeds/insert-feed/insert-feed.3.server.aql | 19 ++ .../feeds/insert-feed/insert-feed.4.sleep.aql | 19 ++ .../feeds/insert-feed/insert-feed.5.update.aql | 21 +++ .../feeds/insert-feed/insert-feed.6.query.aql | 24 +++ .../feeds/insert-feed/insert-feed.7.server.aql | 20 ++ .../feeds/insert-feed/insert-feed.8.ddl.aql | 20 ++ .../feeds/upsert-feed/upsert-feed.1.ddl.aql | 46 ----- .../feeds/upsert-feed/upsert-feed.2.update.aql | 31 --- .../feeds/upsert-feed/upsert-feed.3.server.aql | 26 --- .../feeds/upsert-feed/upsert-feed.4.sleep.aql | 26 --- .../feeds/upsert-feed/upsert-feed.5.update.aql | 28 --- .../feeds/upsert-feed/upsert-feed.6.query.aql | 31 --- .../feeds/upsert-feed/upsert-feed.7.server.aql | 27 --- .../feeds/upsert-feed/upsert-feed.8.ddl.aql | 27 --- .../single-line-definition.2.query.aql | 12 +- .../udf28/udf28.2.query.aql | 13 +- .../cross-dv15/cross-dv15.3.query.sqlpp | 3 +- .../connect-feed-with-function.1.ddl.sqlpp | 55 ++++++ .../connect-feed-with-function.2.update.sqlpp | 23 +++ .../connect-feed-with-function.3.server.sqlpp | 27 +++ .../connect-feed-with-function.4.sleep.sqlpp | 20 ++ .../connect-feed-with-function.5.update.sqlpp | 21 +++ .../connect-feed-with-function.6.query.sqlpp | 22 +++ .../connect-feed-with-function.7.server.sqlpp | 19 ++ .../connect-feed-with-function.8.ddl.sqlpp | 19 ++ .../feed-with-external-function.1.ddl.sqlpp | 53 ++++++ .../feed-with-external-function.2.lib.sqlpp | 19 ++ .../feed-with-external-function.3.update.sqlpp | 28 +++ ...eed-with-external-function.5.pollquery.sqlpp | 29 +++ .../feed-with-external-function.6.lib.sqlpp | 19 ++ .../feed-with-external-function.7.ddl.sqlpp | 19 ++ .../feeds/insert-feed/insert-feed.1.ddl.sqlpp | 46 +++++ .../insert-feed/insert-feed.2.update.sqlpp | 30 +++ .../insert-feed/insert-feed.3.server.sqlpp | 26 +++ .../feeds/insert-feed/insert-feed.4.sleep.sqlpp | 26 +++ .../insert-feed/insert-feed.5.update.sqlpp | 29 +++ .../feeds/insert-feed/insert-feed.6.query.sqlpp | 31 +++ .../insert-feed/insert-feed.7.server.sqlpp | 27 +++ .../feeds/insert-feed/insert-feed.8.ddl.sqlpp | 27 +++ .../feeds/upsert-feed/upsert-feed.1.ddl.sqlpp | 46 ----- .../upsert-feed/upsert-feed.2.update.sqlpp | 30 --- .../upsert-feed/upsert-feed.3.server.sqlpp | 26 --- .../feeds/upsert-feed/upsert-feed.4.sleep.sqlpp | 26 --- .../upsert-feed/upsert-feed.5.update.sqlpp | 29 --- .../feeds/upsert-feed/upsert-feed.6.query.sqlpp | 31 --- .../upsert-feed/upsert-feed.7.server.sqlpp | 27 --- .../feeds/upsert-feed/upsert-feed.8.ddl.sqlpp | 27 --- .../single-line-definition.2.query.sqlpp | 15 +- .../udf28/udf28.2.query.sqlpp | 17 +- .../cross-dataverse/cross-dv15/cross-dv15.1.adm | 6 +- .../connect-feed-with-aql-function.1.adm | 4 - .../connect-feed-with-function.1.adm | 4 + .../results/feeds/feeds_03/feeds_03.1.adm | 2 +- .../results/feeds/insert-feed/insert-feed.1.adm | 6 + .../results/feeds/upsert-feed/upsert-feed.1.adm | 6 - .../single-line-definition.1.adm | 2 +- .../user-defined-functions/udf28/udf28.1.adm | 2 +- .../cross-dataverse/cross-dv15/cross-dv15.3.ast | 8 - .../user-defined-functions/udf28/udf28.2.ast | 48 ++++- .../src/test/resources/runtimets/testsuite.xml | 5 +- .../resources/runtimets/testsuite_it_sqlpp.xml | 5 + .../resources/runtimets/testsuite_sqlpp.xml | 5 + .../asterix/common/exceptions/ErrorCode.java | 1 + .../main/resources/asx_errormsg/en.properties | 1 + .../external/util/ExternalDataConstants.java | 2 +- .../external/util/ExternalDataUtils.java | 4 +- .../asterix/external/util/FeedConstants.java | 7 +- asterixdb/asterix-lang-aql/pom.xml | 10 - .../asterix/lang/aql/parser/FunctionParser.java | 5 + .../aql/statement/SubscribeFeedStatement.java | 189 ------------------- .../asterix/lang/common/util/FunctionUtil.java | 3 +- .../lang/sqlpp/parser/FunctionParser.java | 5 + .../asterix/metadata/entities/Dataset.java | 32 +++- .../asterix/metadata/entities/Function.java | 1 + 94 files changed, 1229 insertions(+), 1254 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java index 403c26b..068aa29 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java @@ -320,56 +320,6 @@ public class CompiledStatements { } } - public static class CompiledConnectFeedStatement implements ICompiledDmlStatement { - private final String dataverseName; - private final String feedName; - private final String datasetName; - private final String policyName; - private final Query query; - private final int varCounter; - - public CompiledConnectFeedStatement(String dataverseName, String feedName, String datasetName, - String policyName, Query query, int varCounter) { - this.dataverseName = dataverseName; - this.feedName = feedName; - this.datasetName = datasetName; - this.policyName = policyName; - this.query = query; - this.varCounter = varCounter; - } - - @Override - public String getDataverseName() { - return dataverseName; - } - - public String getFeedName() { - return feedName; - } - - @Override - public String getDatasetName() { - return datasetName; - } - - public int getVarCounter() { - return varCounter; - } - - public Query getQuery() { - return query; - } - - @Override - public byte getKind() { - return Statement.Kind.CONNECT_FEED; - } - - public String getPolicyName() { - return policyName; - } - } - public static class CompiledSubscribeFeedStatement implements ICompiledDmlStatement { private FeedConnectionRequest request; @@ -404,38 +354,6 @@ public class CompiledStatements { } } - public static class CompiledDisconnectFeedStatement implements ICompiledDmlStatement { - private final String dataverseName; - private final String datasetName; - private final String feedName; - - public CompiledDisconnectFeedStatement(String dataverseName, String feedName, String datasetName) { - this.dataverseName = dataverseName; - this.feedName = feedName; - this.datasetName = datasetName; - } - - @Override - public String getDataverseName() { - return dataverseName; - } - - @Override - public String getDatasetName() { - return datasetName; - } - - public String getFeedName() { - return feedName; - } - - @Override - public byte getKind() { - return Statement.Kind.DISCONNECT_FEED; - } - - } - public static class CompiledDeleteStatement implements ICompiledDmlStatement { private final String dataverseName; private final String datasetName; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index a1c5cf4..66a1073 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -409,15 +409,6 @@ class LangExpressionToPlanTranslator leafOperator = translateDelete(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign); break; - case Statement.Kind.CONNECT_FEED: - leafOperator = translateConnectFeed(targetDatasource, varRef, varRefsForLoading, - additionalFilteringExpressions, assign); - break; - case Statement.Kind.SUBSCRIBE_FEED: - leafOperator = translateSubscribeFeed((CompiledSubscribeFeedStatement) stmt, targetDatasource, - unnestVar, topOp, exprs, resVar, varRefsForLoading, varRef, assign, - additionalFilteringField, additionalFilteringAssign, additionalFilteringExpressions); - break; default: throw new AlgebricksException("Unsupported statement kind " + stmt.getKind()); } @@ -429,18 +420,6 @@ class LangExpressionToPlanTranslator return plan; } - private ILogicalOperator translateConnectFeed(DatasetDataSource targetDatasource, - Mutable<ILogicalExpression> varRef, List<Mutable<ILogicalExpression>> varRefsForLoading, - List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign) { - InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, - varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false); - insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); - insertOp.getInputs().add(new MutableObject<>(assign)); - ILogicalOperator leafOperator = new DelegateOperator(new CommitOperator(true)); - leafOperator.getInputs().add(new MutableObject<>(insertOp)); - return leafOperator; - } - private ILogicalOperator translateDelete(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef, List<Mutable<ILogicalExpression>> varRefsForLoading, List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign) @@ -458,100 +437,6 @@ class LangExpressionToPlanTranslator return leafOperator; } - private ILogicalOperator translateSubscribeFeed(CompiledSubscribeFeedStatement sfs, - DatasetDataSource targetDatasource, LogicalVariable unnestVar, ILogicalOperator topOp, - ArrayList<Mutable<ILogicalExpression>> exprs, LogicalVariable resVar, - List<Mutable<ILogicalExpression>> varRefsForLoading, Mutable<ILogicalExpression> varRef, - ILogicalOperator assign, List<String> additionalFilteringField, AssignOperator additionalFilteringAssign, - List<Mutable<ILogicalExpression>> additionalFilteringExpressions) throws AlgebricksException { - // if the feed is a change feed (i.e, performs different operations), we need to project op variable - InsertDeleteUpsertOperator feedModificationOp; - AssignOperator metaAndKeysAssign; - List<LogicalVariable> metaAndKeysVars = null; - List<Mutable<ILogicalExpression>> metaAndKeysExprs = null; - List<Mutable<ILogicalExpression>> metaExpSingletonList = null; - Feed feed = metadataProvider.findFeed(sfs.getDataverseName(), sfs.getFeedName()); - boolean isChangeFeed = ExternalDataUtils.isChangeFeed(feed.getAdapterConfiguration()); - boolean isUpsertFeed = ExternalDataUtils.isUpsertFeed(feed.getAdapterConfiguration()); - - ProjectOperator project = (ProjectOperator) topOp; - if (targetDatasource.getDataset().hasMetaPart() || isChangeFeed) { - metaAndKeysVars = new ArrayList<>(); - metaAndKeysExprs = new ArrayList<>(); - if (targetDatasource.getDataset().hasMetaPart()) { - // add the meta function - IFunctionInfo finfoMeta = FunctionUtil.getFunctionInfo(BuiltinFunctions.META); - ScalarFunctionCallExpression metaFunction = new ScalarFunctionCallExpression(finfoMeta, - new MutableObject<>(new VariableReferenceExpression(unnestVar))); - // create assign for the meta part - LogicalVariable metaVar = context.newVar(); - metaExpSingletonList = new ArrayList<>(1); - metaExpSingletonList.add(new MutableObject<>(new VariableReferenceExpression(metaVar))); - metaAndKeysVars.add(metaVar); - metaAndKeysExprs.add(new MutableObject<>(metaFunction)); - project.getVariables().add(metaVar); - } - } - if (isChangeFeed) { - varRefsForLoading.clear(); - for (Mutable<ILogicalExpression> assignExpr : exprs) { - if (assignExpr.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { - AbstractFunctionCallExpression funcCall = (AbstractFunctionCallExpression) assignExpr.getValue(); - funcCall.substituteVar(resVar, unnestVar); - LogicalVariable pkVar = context.newVar(); - metaAndKeysVars.add(pkVar); - metaAndKeysExprs.add(new MutableObject<>(assignExpr.getValue())); - project.getVariables().add(pkVar); - varRefsForLoading.add(new MutableObject<>(new VariableReferenceExpression(pkVar))); - } - } - // A change feed, we don't need the assign to access PKs - feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, - metaExpSingletonList, InsertDeleteUpsertOperator.Kind.UPSERT, false); - // Create and add a new variable used for representing the original record - feedModificationOp.setPrevRecordVar(context.newVar()); - feedModificationOp.setPrevRecordType(targetDatasource.getItemType()); - if (targetDatasource.getDataset().hasMetaPart()) { - List<LogicalVariable> metaVars = new ArrayList<>(); - metaVars.add(context.newVar()); - feedModificationOp.setPrevAdditionalNonFilteringVars(metaVars); - List<Object> metaTypes = new ArrayList<>(); - metaTypes.add(targetDatasource.getMetaItemType()); - feedModificationOp.setPrevAdditionalNonFilteringTypes(metaTypes); - } - - if (additionalFilteringField != null) { - feedModificationOp.setPrevFilterVar(context.newVar()); - feedModificationOp.setPrevFilterType( - ((ARecordType) targetDatasource.getItemType()).getFieldType(additionalFilteringField.get(0))); - additionalFilteringAssign.getInputs().clear(); - additionalFilteringAssign.getInputs().add(assign.getInputs().get(0)); - feedModificationOp.getInputs().add(new MutableObject<>(additionalFilteringAssign)); - } else { - feedModificationOp.getInputs().add(assign.getInputs().get(0)); - } - } else { - final InsertDeleteUpsertOperator.Kind opKind = - isUpsertFeed ? InsertDeleteUpsertOperator.Kind.UPSERT : InsertDeleteUpsertOperator.Kind.INSERT; - feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading, - metaExpSingletonList, opKind, false); - if (isUpsertFeed) { - feedModificationOp.setPrevRecordVar(context.newVar()); - feedModificationOp.setPrevRecordType(targetDatasource.getItemType()); - } - feedModificationOp.getInputs().add(new MutableObject<>(assign)); - } - if (targetDatasource.getDataset().hasMetaPart() || isChangeFeed) { - metaAndKeysAssign = new AssignOperator(metaAndKeysVars, metaAndKeysExprs); - metaAndKeysAssign.getInputs().add(topOp.getInputs().get(0)); - topOp.getInputs().set(0, new MutableObject<>(metaAndKeysAssign)); - } - feedModificationOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); - ILogicalOperator leafOperator = new DelegateOperator(new CommitOperator(true)); - leafOperator.getInputs().add(new MutableObject<>(feedModificationOp)); - return leafOperator; - } - private ILogicalOperator translateUpsert(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef, List<Mutable<ILogicalExpression>> varRefsForLoading, List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign, @@ -570,7 +455,6 @@ class LangExpressionToPlanTranslator if (targetDatasource.getDataset().hasMetaPart()) { if (returnExpression != null) { throw new AlgebricksException("Returning not allowed on datasets with Meta records"); - } AssignOperator metaAndKeysAssign; List<LogicalVariable> metaAndKeysVars; @@ -859,7 +743,8 @@ class LangExpressionToPlanTranslator IFunctionInfo finfo = ExternalFunctionCompilerUtil .getExternalFunctionInfo(metadataProvider.getMetadataTxnContext(), function); f = new ScalarFunctionCallExpression(finfo, args); - } else if (function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_AQL)) { + } else if (function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_AQL) + || function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_SQLPP)) { IFunctionInfo finfo = FunctionUtil.getFunctionInfo(signature); f = new ScalarFunctionCallExpression(finfo, args); } else { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index 583302b..4b78b93 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -50,7 +50,6 @@ import org.apache.asterix.dataflow.data.common.PartialAggregationTypeComputer; import org.apache.asterix.external.feed.watch.FeedActivityDetails; import org.apache.asterix.formats.base.IDataFormat; import org.apache.asterix.jobgen.QueryLogicalExpressionJobGen; -import org.apache.asterix.lang.aql.statement.SubscribeFeedStatement; import org.apache.asterix.lang.common.base.IAstPrintVisitorFactory; import org.apache.asterix.lang.common.base.IQueryRewriter; import org.apache.asterix.lang.common.base.IReturningStatement; @@ -59,6 +58,7 @@ import org.apache.asterix.lang.common.base.Statement; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.statement.FunctionDecl; import org.apache.asterix.lang.common.statement.Query; +import org.apache.asterix.lang.common.statement.StartFeedStatement; import org.apache.asterix.lang.common.util.FunctionUtil; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.optimizer.base.FuzzyUtils; @@ -117,7 +117,7 @@ public class APIFramework { ImmutableSet.of(CompilerProperties.COMPILER_JOINMEMORY_KEY, CompilerProperties.COMPILER_GROUPMEMORY_KEY, CompilerProperties.COMPILER_SORTMEMORY_KEY, CompilerProperties.COMPILER_PARALLELISM_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, FuzzyUtils.SIM_FUNCTION_PROP_NAME, - FuzzyUtils.SIM_THRESHOLD_PROP_NAME, SubscribeFeedStatement.WAIT_FOR_COMPLETION, + FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME, FeedActivityDetails.COLLECT_LOCATIONS, "inline_with", "hash_merge", "output-record-type"); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/FeedEventsListener.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/FeedEventsListener.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/FeedEventsListener.java index 38e8a21..c0ce6ec 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/FeedEventsListener.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/FeedEventsListener.java @@ -67,8 +67,8 @@ public class FeedEventsListener extends ActiveEntityEventsListener { @Override public synchronized void remove(Dataset dataset) throws HyracksDataException { super.remove(dataset); - feedConnections.removeIf(o -> o.getDataverseName().equals(dataset.getDataverseName()) - && o.getDatasetName().equals(dataset.getDatasetName())); + feedConnections.removeIf(o -> o.getDataverseName().equals(dataset.getDataverseName()) && o.getDatasetName() + .equals(dataset.getDatasetName())); } public synchronized void addFeedConnection(FeedConnection feedConnection) { @@ -82,12 +82,8 @@ public class FeedEventsListener extends ActiveEntityEventsListener { @Override protected void doStart(MetadataProvider mdProvider) throws HyracksDataException { try { - ILangCompilationProvider compilationProvider = new AqlCompilationProvider(); - IStorageComponentProvider storageComponentProvider = new StorageComponentProvider(); - DefaultStatementExecutorFactory statementExecutorFactory = new DefaultStatementExecutorFactory(); - Pair<JobSpecification, AlgebricksAbsolutePartitionConstraint> jobInfo = FeedOperations.buildStartFeedJob( - ((QueryTranslator) statementExecutor).getSessionOutput(), mdProvider, feed, feedConnections, - compilationProvider, storageComponentProvider, statementExecutorFactory, hcc); + Pair<JobSpecification, AlgebricksAbsolutePartitionConstraint> jobInfo = + FeedOperations.buildStartFeedJob(mdProvider, feed, feedConnections, statementExecutor, hcc); JobSpecification feedJob = jobInfo.getLeft(); WaitForStateSubscriber eventSubscriber = new WaitForStateSubscriber(this, EnumSet.of(ActivityState.RUNNING, ActivityState.TEMPORARILY_FAILED, ActivityState.PERMANENTLY_FAILED)); @@ -119,8 +115,8 @@ public class FeedEventsListener extends ActiveEntityEventsListener { // Construct ActiveMessage for (int i = 0; i < getLocations().getLocations().length; i++) { String intakeLocation = getLocations().getLocations()[i]; - FeedOperations.SendStopMessageToNode(metadataProvider.getApplicationContext(), entityId, intakeLocation, - i); + FeedOperations + .SendStopMessageToNode(metadataProvider.getApplicationContext(), entityId, intakeLocation, i); } eventSubscriber.sync(); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 6b4483c..d3be23c 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -122,6 +122,7 @@ import org.apache.asterix.lang.common.statement.TypeDecl; import org.apache.asterix.lang.common.statement.TypeDropStatement; import org.apache.asterix.lang.common.statement.WriteStatement; import org.apache.asterix.lang.common.struct.Identifier; +import org.apache.asterix.lang.sqlpp.rewrites.SqlppRewriterFactory; import org.apache.asterix.metadata.IDatasetDetails; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; @@ -1669,7 +1670,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen throw new AlgebricksException("There is no dataverse with this name " + dataverse + "."); } Function function = new Function(dataverse, functionName, cfs.getaAterixFunction().getArity(), - cfs.getParamList(), Function.RETURNTYPE_VOID, cfs.getFunctionBody(), Function.LANGUAGE_AQL, + cfs.getParamList(), Function.RETURNTYPE_VOID, cfs.getFunctionBody(), + rewriterFactory instanceof SqlppRewriterFactory ? Function.LANGUAGE_SQLPP : Function.LANGUAGE_AQL, FunctionKind.SCALAR.toString()); MetadataManager.INSTANCE.addFunction(mdTxnCtx, function); @@ -2156,6 +2158,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen String policyName = cfs.getPolicy(); MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); + // TODO: Check whether we are connecting a change feed to a non-meta dataset // Check whether feed is alive ActiveNotificationHandler activeEventHandler = (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); @@ -2186,7 +2189,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen throw new AlgebricksException("Feed" + feedName + " is already connected dataset " + datasetName); } fc = new FeedConnection(dataverseName, feedName, datasetName, appliedFunctions, policyName, - outputType.toString()); + outputType.getTypeName()); MetadataManager.INSTANCE.addFeedConnection(metadataProvider.getMetadataTxnContext(), fc); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); if (listener != null) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java index cc95770..2b9386e 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java @@ -20,6 +20,7 @@ package org.apache.asterix.utils; import java.rmi.RemoteException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -33,32 +34,53 @@ import org.apache.asterix.active.EntityId; import org.apache.asterix.active.message.ActiveManagerMessage; import org.apache.asterix.active.message.ActiveManagerMessage.Kind; import org.apache.asterix.app.translator.DefaultStatementExecutorFactory; +import org.apache.asterix.app.translator.QueryTranslator; import org.apache.asterix.common.cluster.IClusterStateManager; -import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.dataflow.LSMTreeInsertDeleteOperatorDescriptor; import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.common.messaging.api.ICCMessageBroker; import org.apache.asterix.common.transactions.JobId; import org.apache.asterix.common.utils.StoragePathUtil; -import org.apache.asterix.compiler.provider.ILangCompilationProvider; +import org.apache.asterix.compiler.provider.SqlppCompilationProvider; import org.apache.asterix.external.api.IAdapterFactory; import org.apache.asterix.external.feed.management.FeedConnectionId; -import org.apache.asterix.external.feed.management.FeedConnectionRequest; import org.apache.asterix.external.feed.policy.FeedPolicyAccessor; import org.apache.asterix.external.feed.watch.FeedActivityDetails; import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor; import org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor; import org.apache.asterix.external.operators.FeedIntakeOperatorNodePushable; import org.apache.asterix.external.operators.FeedMetaOperatorDescriptor; +import org.apache.asterix.external.util.ExternalDataUtils; +import org.apache.asterix.external.util.FeedConstants; import org.apache.asterix.external.util.FeedUtils; import org.apache.asterix.external.util.FeedUtils.FeedRuntimeType; -import org.apache.asterix.lang.aql.statement.SubscribeFeedStatement; +import org.apache.asterix.file.StorageComponentProvider; +import org.apache.asterix.lang.common.base.Expression; import org.apache.asterix.lang.common.base.Statement; -import org.apache.asterix.lang.common.statement.DataverseDecl; +import org.apache.asterix.lang.common.clause.LetClause; +import org.apache.asterix.lang.common.expression.CallExpr; +import org.apache.asterix.lang.common.expression.LiteralExpr; +import org.apache.asterix.lang.common.expression.VariableExpr; +import org.apache.asterix.lang.common.literal.IntegerLiteral; +import org.apache.asterix.lang.common.literal.StringLiteral; +import org.apache.asterix.lang.common.statement.InsertStatement; +import org.apache.asterix.lang.common.statement.Query; +import org.apache.asterix.lang.common.statement.UpsertStatement; import org.apache.asterix.lang.common.struct.Identifier; +import org.apache.asterix.lang.common.struct.VarIdentifier; import org.apache.asterix.lang.common.util.FunctionUtil; +import org.apache.asterix.lang.sqlpp.clause.FromClause; +import org.apache.asterix.lang.sqlpp.clause.FromTerm; +import org.apache.asterix.lang.sqlpp.clause.SelectBlock; +import org.apache.asterix.lang.sqlpp.clause.SelectClause; +import org.apache.asterix.lang.sqlpp.clause.SelectElement; +import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation; +import org.apache.asterix.lang.sqlpp.expression.SelectExpression; +import org.apache.asterix.lang.sqlpp.struct.SetOperationInput; +import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Feed; import org.apache.asterix.metadata.entities.FeedConnection; @@ -109,6 +131,8 @@ import org.apache.hyracks.dataflow.std.misc.ReplicateOperatorDescriptor; */ public class FeedOperations { + public static final String FEED_DATAFLOW_INTERMEIDATE_VAL_PREFIX = "val"; + private FeedOperations() { } @@ -154,30 +178,76 @@ public class FeedOperations { return spec; } - private static JobSpecification getConnectionJob(SessionOutput sessionOutput, MetadataProvider metadataProvider, - FeedConnection feedConnection, String[] locations, ILangCompilationProvider compilationProvider, - IStorageComponentProvider storageComponentProvider, DefaultStatementExecutorFactory qtFactory, - IHyracksClientConnection hcc) throws AlgebricksException, RemoteException, ACIDException { - DataverseDecl dataverseDecl = new DataverseDecl(new Identifier(feedConnection.getDataverseName())); - FeedConnectionRequest fcr = - new FeedConnectionRequest(FeedRuntimeType.INTAKE, feedConnection.getAppliedFunctions(), - feedConnection.getDatasetName(), feedConnection.getPolicyName(), feedConnection.getFeedId()); - SubscribeFeedStatement subscribeStmt = new SubscribeFeedStatement(locations, fcr); - subscribeStmt.initialize(metadataProvider.getMetadataTxnContext()); - List<Statement> statements = new ArrayList<>(); - statements.add(dataverseDecl); - statements.add(subscribeStmt); - IStatementExecutor translator = qtFactory.create(metadataProvider.getApplicationContext(), statements, - sessionOutput, compilationProvider, storageComponentProvider); - // configure the metadata provider - metadataProvider.getConfig().put(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, "" + Boolean.TRUE); - metadataProvider.getConfig().put(FeedActivityDetails.FEED_POLICY_NAME, "" + subscribeStmt.getPolicy()); - metadataProvider.getConfig().put(FeedActivityDetails.COLLECT_LOCATIONS, - StringUtils.join(subscribeStmt.getLocations(), ',')); + private static List<Expression> addArgs(Object... args) { + List<Expression> argExprs = new ArrayList<>(); + for (Object arg : args) { + if (arg instanceof Integer) { + argExprs.add(new LiteralExpr(new IntegerLiteral((Integer) arg))); + } else if (arg instanceof String) { + argExprs.add(new LiteralExpr(new StringLiteral((String) arg))); + } else if (arg instanceof VariableExpr) { + argExprs.add((VariableExpr) arg); + } + } + return argExprs; + } - CompiledStatements.CompiledSubscribeFeedStatement csfs = new CompiledStatements.CompiledSubscribeFeedStatement( - subscribeStmt.getSubscriptionRequest(), subscribeStmt.getVarCounter()); - return translator.rewriteCompileQuery(hcc, metadataProvider, subscribeStmt.getQuery(), csfs); + private static Query makeConnectionQuery(FeedConnection feedConnection) { + // Construct from clause + VarIdentifier fromVarId = SqlppVariableUtil.toInternalVariableIdentifier(feedConnection.getFeedName()); + VariableExpr fromTermLeftExpr = new VariableExpr(fromVarId); + // TODO: remove target feedid from args list (xikui) + // TODO: Get rid of this INTAKE + List<Expression> exprList = + addArgs(feedConnection.getDataverseName(), feedConnection.getFeedId().getEntityName(), + feedConnection.getFeedId().getEntityName(), FeedRuntimeType.INTAKE.toString(), + feedConnection.getDatasetName(), feedConnection.getOutputType()); + CallExpr datasrouceCallFunction = new CallExpr(FeedConstants.FEED_COLLECT_FUN_SIGNATURE, exprList); + FromTerm fromterm = new FromTerm(datasrouceCallFunction, fromTermLeftExpr, null, null); + FromClause fromClause = new FromClause(Arrays.asList(fromterm)); + // TODO: This can be the place to add select predicate for ingestion + // Attaching functions + int varIdx = 1; + VariableExpr previousVarExpr = fromTermLeftExpr; + ArrayList<LetClause> letClauses = new ArrayList<>(); + for (FunctionSignature funcSig : feedConnection.getAppliedFunctions()) { + VarIdentifier intermediateVar = SqlppVariableUtil + .toInternalVariableIdentifier(FEED_DATAFLOW_INTERMEIDATE_VAL_PREFIX + String.valueOf(varIdx)); + VariableExpr intermediateVarExpr = new VariableExpr(intermediateVar); + CallExpr functionCallExpr = new CallExpr(funcSig, addArgs(previousVarExpr)); + previousVarExpr = intermediateVarExpr; + LetClause letClause = new LetClause(intermediateVarExpr, functionCallExpr); + letClauses.add(letClause); + varIdx++; + } + // Constructing select clause + SelectElement selectElement = new SelectElement(previousVarExpr); + SelectClause selectClause = new SelectClause(selectElement, null, false); + SelectBlock selectBlock = new SelectBlock(selectClause, fromClause, letClauses, null, null, null, null); + SelectSetOperation selectSetOperation = new SelectSetOperation(new SetOperationInput(selectBlock, null), null); + SelectExpression body = new SelectExpression(null, selectSetOperation, null, null, true); + Query query = new Query(false, true, body, 0); + return query; + } + + private static JobSpecification getConnectionJob(MetadataProvider metadataProvider, FeedConnection feedConn, + IStatementExecutor statementExecutor, IHyracksClientConnection hcc, Boolean insertFeed) + throws AlgebricksException, RemoteException, ACIDException { + metadataProvider.getConfig().put(FeedActivityDetails.FEED_POLICY_NAME, feedConn.getPolicyName()); + Query feedConnQuery = makeConnectionQuery(feedConn); + CompiledStatements.ICompiledDmlStatement clfrqs; + if (insertFeed) { + InsertStatement stmtUpsert = new InsertStatement(new Identifier(feedConn.getDataverseName()), + new Identifier(feedConn.getDatasetName()), feedConnQuery, -1, null, null); + clfrqs = new CompiledStatements.CompiledInsertStatement(feedConn.getDataverseName(), + feedConn.getDatasetName(), feedConnQuery, stmtUpsert.getVarCounter(), null, null); + } else { + UpsertStatement stmtUpsert = new UpsertStatement(new Identifier(feedConn.getDataverseName()), + new Identifier(feedConn.getDatasetName()), feedConnQuery, -1, null, null); + clfrqs = new CompiledStatements.CompiledUpsertStatement(feedConn.getDataverseName(), + feedConn.getDatasetName(), feedConnQuery, stmtUpsert.getVarCounter(), null, null); + } + return statementExecutor.rewriteCompileQuery(hcc, metadataProvider, feedConnQuery, clfrqs); } private static JobSpecification combineIntakeCollectJobs(MetadataProvider metadataProvider, Feed feed, @@ -220,9 +290,9 @@ public class FeedOperations { String datasetName = feedConnections.get(iter1).getDatasetName(); FeedConnectionId feedConnectionId = new FeedConnectionId(ingestionOp.getEntityId(), datasetName); - FeedPolicyEntity feedPolicyEntity = - FeedMetadataUtil.validateIfPolicyExists(curFeedConnection.getDataverseName(), - curFeedConnection.getPolicyName(), metadataProvider.getMetadataTxnContext()); + FeedPolicyEntity feedPolicyEntity = FeedMetadataUtil + .validateIfPolicyExists(curFeedConnection.getDataverseName(), curFeedConnection.getPolicyName(), + metadataProvider.getMetadataTxnContext()); for (Map.Entry<OperatorDescriptorId, IOperatorDescriptor> entry : operatorsMap.entrySet()) { IOperatorDescriptor opDesc = entry.getValue(); @@ -358,24 +428,39 @@ public class FeedOperations { return jobSpec; } + private static IStatementExecutor getSQLPPTranslator(MetadataProvider metadataProvider, + SessionOutput sessionOutput) { + List<Statement> stmts = new ArrayList<>(); + DefaultStatementExecutorFactory qtFactory = new DefaultStatementExecutorFactory(); + IStatementExecutor translator = qtFactory + .create(metadataProvider.getApplicationContext(), stmts, sessionOutput, new SqlppCompilationProvider(), + new StorageComponentProvider()); + return translator; + } + public static Pair<JobSpecification, AlgebricksAbsolutePartitionConstraint> buildStartFeedJob( - SessionOutput sessionOutput, MetadataProvider metadataProvider, Feed feed, - List<FeedConnection> feedConnections, ILangCompilationProvider compilationProvider, - IStorageComponentProvider storageComponentProvider, DefaultStatementExecutorFactory qtFactory, - IHyracksClientConnection hcc) throws Exception { + MetadataProvider metadataProvider, Feed feed, List<FeedConnection> feedConnections, + IStatementExecutor statementExecutor, IHyracksClientConnection hcc) throws Exception { FeedPolicyAccessor fpa = new FeedPolicyAccessor(new HashMap<>()); - // TODO: Change the default Datasource to use all possible partitions Pair<JobSpecification, IAdapterFactory> intakeInfo = buildFeedIntakeJobSpec(feed, metadataProvider, fpa); - //TODO: Add feed policy accessor List<JobSpecification> jobsList = new ArrayList<>(); + // TODO: Figure out a better way to handle insert/upsert per conn instead of per feed + Boolean insertFeed = ExternalDataUtils.isInsertFeed(feed.getAdapterConfiguration()); // Construct the ingestion Job JobSpecification intakeJob = intakeInfo.getLeft(); IAdapterFactory ingestionAdaptorFactory = intakeInfo.getRight(); String[] ingestionLocations = ingestionAdaptorFactory.getPartitionConstraint().getLocations(); + // Add metadata configs + metadataProvider.getConfig().put(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, Boolean.TRUE.toString()); + metadataProvider.getConfig() + .put(FeedActivityDetails.COLLECT_LOCATIONS, StringUtils.join(ingestionLocations, ',')); + // TODO: Once we deprecated AQL, this extra queryTranslator can be removed. + IStatementExecutor translator = + getSQLPPTranslator(metadataProvider, ((QueryTranslator) statementExecutor).getSessionOutput()); // Add connection job for (FeedConnection feedConnection : feedConnections) { - JobSpecification connectionJob = getConnectionJob(sessionOutput, metadataProvider, feedConnection, - ingestionLocations, compilationProvider, storageComponentProvider, qtFactory, hcc); + JobSpecification connectionJob = getConnectionJob(metadataProvider, feedConnection, translator, hcc, + insertFeed); jobsList.add(connectionJob); } return Pair.of(combineIntakeCollectJobs(metadataProvider, feed, intakeJob, jobsList, feedConnections, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/cross-dataverse/cross-dv15/cross-dv15.3.query.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/cross-dataverse/cross-dv15/cross-dv15.3.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/cross-dataverse/cross-dv15/cross-dv15.3.query.aql index 1d99878..2dc595b 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/cross-dataverse/cross-dv15/cross-dv15.3.query.aql +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/cross-dataverse/cross-dv15/cross-dv15.3.query.aql @@ -30,6 +30,5 @@ return { "DataverseName": $l.DataverseName, "Name": $l.Name, "Arity": $l.Arity, -"ReturnType": $l.ReturnType, -"Language": $l.Language +"ReturnType": $l.ReturnType }; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.1.ddl.aql deleted file mode 100644 index 631c3fb..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.1.ddl.aql +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed and apply two functions in the - * workflow. The output of the first function can be used in - * the second function. The function parameter can have any - * name. - * Expected Res : Success - * Date : 29th Mar 2017 - */ -drop dataverse experiments if exists; -create dataverse experiments; -use dataverse experiments; - -create type TwitterUser if not exists as open{ - screen-name: string, - friends_count: int32, - name: string, - followers_count: int32 -}; - -create dataset TwitterUsers(TwitterUser) primary key screen-name; - -create function test_func0($xyz) { - let $tty1 := if ($xyz.followers_count > 25000) then {"popularity":"Good!"} else {"popularity":"Bad!"} - return object_merge($tty1, $xyz) -} - -create function test_func1($anyname) { - let $tty2 := if ($anyname.popularity = "Good!") then {"true_popularity":"Indeed Good!"} else {"true_popularity":"Indeed Bad!"} - return object_merge($tty2, $anyname) -} - -create feed UserFeed using socket_adapter -( - ("sockets"="127.0.0.1:10001"), - ("address-type"="IP"), - ("type-name"="TwitterUser"), - ("format"="adm"), - ("upsert-feed"="true") -); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.2.update.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.2.update.aql deleted file mode 100644 index a5933a5..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.2.update.aql +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed and apply two functions in the - * workflow. The output of the first function can be used in - * the second function. The function parameter can have any - * name. - * Expected Res : Success - * Date : 29th Mar 2017 - */ -use dataverse experiments; -set wait-for-completion-feed "false"; - -connect feed UserFeed to dataset TwitterUsers apply function test_func0,test_func1; - -start feed UserFeed; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.3.server.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.3.server.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.3.server.aql deleted file mode 100644 index eacf623..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.3.server.aql +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed and apply two functions in the - * workflow. The output of the first function can be used in - * the second function. The function parameter can have any - * name. - * Expected Res : Success - * Date : 29th Mar 2017 - */ -start client 10001 file-client 127.0.0.1 ../asterix-app/data/tinysocial/twu.adm 500 50 1000 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.4.sleep.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.4.sleep.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.4.sleep.aql deleted file mode 100644 index dc5dae0..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.4.sleep.aql +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed and apply two functions in the - * workflow. The output of the first function can be used in - * the second function. The function parameter can have any - * name. - * Expected Res : Success - * Date : 29th Mar 2017 - */ -2000 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.5.update.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.5.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.5.update.aql deleted file mode 100644 index dcf2278..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.5.update.aql +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed and apply two functions in the - * workflow. The output of the first function can be used in - * the second function. The function parameter can have any - * name. - * Expected Res : Success - * Date : 29th Mar 2017 - */ -use dataverse experiments; -stop feed UserFeed; -disconnect feed UserFeed from dataset TwitterUsers; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.6.query.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.6.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.6.query.aql deleted file mode 100644 index 1a06334..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.6.query.aql +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed and apply two functions in the - * workflow. The output of the first function can be used in - * the second function. The function parameter can have any - * name. - * Expected Res : Success - * Date : 29th Mar 2017 - */ -use dataverse experiments; - -for $x in dataset TwitterUsers -order by $x.screen-name -return $x.true_popularity; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.7.server.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.7.server.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.7.server.aql deleted file mode 100644 index 4ba1c81..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.7.server.aql +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed and apply two functions in the - * workflow. The output of the first function can be used in - * the second function. The function parameter can have any - * name. - * Expected Res : Success - * Date : 29th Mar 2017 - */ - -stop 10001 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.8.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.8.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.8.ddl.aql deleted file mode 100644 index 7722945..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-aql-function/connect-feed-with-aql-function.8.ddl.aql +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed and apply two functions in the - * workflow. The output of the first function can be used in - * the second function. The function parameter can have any - * name. - * Expected Res : Success - * Date : 29th Mar 2017 - */ -use dataverse experiments; -drop dataverse experiments; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.1.ddl.aql new file mode 100644 index 0000000..631c3fb --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.1.ddl.aql @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Create a feed and apply two functions in the + * workflow. The output of the first function can be used in + * the second function. The function parameter can have any + * name. + * Expected Res : Success + * Date : 29th Mar 2017 + */ +drop dataverse experiments if exists; +create dataverse experiments; +use dataverse experiments; + +create type TwitterUser if not exists as open{ + screen-name: string, + friends_count: int32, + name: string, + followers_count: int32 +}; + +create dataset TwitterUsers(TwitterUser) primary key screen-name; + +create function test_func0($xyz) { + let $tty1 := if ($xyz.followers_count > 25000) then {"popularity":"Good!"} else {"popularity":"Bad!"} + return object_merge($tty1, $xyz) +} + +create function test_func1($anyname) { + let $tty2 := if ($anyname.popularity = "Good!") then {"true_popularity":"Indeed Good!"} else {"true_popularity":"Indeed Bad!"} + return object_merge($tty2, $anyname) +} + +create feed UserFeed using socket_adapter +( + ("sockets"="127.0.0.1:10001"), + ("address-type"="IP"), + ("type-name"="TwitterUser"), + ("format"="adm"), + ("upsert-feed"="true") +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.2.update.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.2.update.aql new file mode 100644 index 0000000..a5933a5 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.2.update.aql @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Create a feed and apply two functions in the + * workflow. The output of the first function can be used in + * the second function. The function parameter can have any + * name. + * Expected Res : Success + * Date : 29th Mar 2017 + */ +use dataverse experiments; +set wait-for-completion-feed "false"; + +connect feed UserFeed to dataset TwitterUsers apply function test_func0,test_func1; + +start feed UserFeed; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.3.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.3.ddl.aql new file mode 100644 index 0000000..46056b1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/connect-feed-with-function/connect-feed-with-function.3.ddl.aql @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Create a feed and apply two functions in the + * workflow. The output of the first function can be used in + * the second function. The function parameter can have any + * name. + * Expected Res : Success + * Date : 29th Mar 2017 + */ +drop dataverse experiments; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.1.ddl.aql new file mode 100644 index 0000000..8dba0fe --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.1.ddl.aql @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Create a feed with insert option. Push record twice + * with minor changes. The updated record will not be updated + * Expected Res : Success + * Date : 8th Oct 2017 + */ +drop dataverse experiments if exists; +create dataverse experiments; +use dataverse experiments; + +create type TwitterUser if not exists as open{ + screen-name: string, + friends_count: int32, + name: string, + followers_count: int32 +}; + +create dataset TwitterUsers(TwitterUser) primary key screen-name; + +create feed UserFeed using socket_adapter +( + ("sockets"="127.0.0.1:10001"), + ("address-type"="IP"), + ("type-name"="TwitterUser"), + ("format"="adm"), + ("insert-feed"="true") +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.2.update.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.2.update.aql new file mode 100644 index 0000000..2a9c11c --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.2.update.aql @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse experiments; +set wait-for-completion-feed "false"; + +connect feed UserFeed to dataset TwitterUsers; + +start feed UserFeed; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.3.server.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.3.server.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.3.server.aql new file mode 100644 index 0000000..873acae --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.3.server.aql @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +start client 10001 file-client 127.0.0.1 ../asterix-app/data/tinysocial/twu_update.adm 500 50 1000 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.4.sleep.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.4.sleep.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.4.sleep.aql new file mode 100644 index 0000000..5af9639 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.4.sleep.aql @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +1000 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.5.update.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.5.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.5.update.aql new file mode 100644 index 0000000..4904a5c --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.5.update.aql @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse experiments; +stop feed UserFeed; +disconnect feed UserFeed from dataset TwitterUsers; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.6.query.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.6.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.6.query.aql new file mode 100644 index 0000000..6085520 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.6.query.aql @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use dataverse experiments; + +for $x in dataset TwitterUsers +order by $x.screen-name +return $x; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.7.server.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.7.server.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.7.server.aql new file mode 100644 index 0000000..f9134bf --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.7.server.aql @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +stop 10001 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.8.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.8.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.8.ddl.aql new file mode 100644 index 0000000..94276a1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/insert-feed/insert-feed.8.ddl.aql @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse experiments; +drop dataverse experiments; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.1.ddl.aql deleted file mode 100644 index 14b857f..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.1.ddl.aql +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed with upsert option. Push record twice - * with minor changes. The updated record will not cause duplicate - * key exception and the data will be updated. - * Expected Res : Success - * Date : 13th Aug 2016 - */ -drop dataverse experiments if exists; -create dataverse experiments; -use dataverse experiments; - -create type TwitterUser if not exists as open{ - screen-name: string, - friends_count: int32, - name: string, - followers_count: int32 -}; - -create dataset TwitterUsers(TwitterUser) primary key screen-name; - -create feed UserFeed using socket_adapter -( - ("sockets"="127.0.0.1:10001"), - ("address-type"="IP"), - ("type-name"="TwitterUser"), - ("format"="adm"), - ("upsert-feed"="true") -); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.2.update.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.2.update.aql deleted file mode 100644 index bbf20dd..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.2.update.aql +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed with upsert option. Push record twice - * with minor changes. The updated record will not cause duplicate - * key exception and the data will be updated. - * Expected Res : Success - * Date : 13th Aug 2016 - */ -use dataverse experiments; -set wait-for-completion-feed "false"; - -connect feed UserFeed to dataset TwitterUsers; - -start feed UserFeed; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7e76a079/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.3.server.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.3.server.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.3.server.aql deleted file mode 100644 index 752ef6e..0000000 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/upsert-feed/upsert-feed.3.server.aql +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/* - * Description : Create a feed with upsert option. Push record twice - * with minor changes. The updated record will not cause duplicate - * key exception and the data will be updated. - * Expected Res : Success - * Date : 13th Aug 2016 - */ -start client 10001 file-client 127.0.0.1 ../asterix-app/data/tinysocial/twu_update.adm 500 50 1000 \ No newline at end of file
