Repository: asterixdb Updated Branches: refs/heads/master 26cc90802 -> b4d166b3c
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java new file mode 100644 index 0000000..c143a63 --- /dev/null +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.util; + +import org.apache.asterix.app.function.DatasetResourcesRewriter; +import org.apache.asterix.app.function.DatasetRewriter; +import org.apache.asterix.app.function.FeedRewriter; +import org.apache.asterix.app.function.StorageComponentsRewriter; +import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.asterix.om.utils.RecordUtil; + +public class MetadataBuiltinFunctions { + + static { + // Dataset function + BuiltinFunctions.addFunction(BuiltinFunctions.DATASET, DatasetRewriter.INSTANCE, true); + BuiltinFunctions.addUnnestFun(BuiltinFunctions.DATASET, false); + BuiltinFunctions.addDatasourceFunction(BuiltinFunctions.DATASET, DatasetRewriter.INSTANCE); + // Feed collect function + BuiltinFunctions.addPrivateFunction(BuiltinFunctions.FEED_COLLECT, FeedRewriter.INSTANCE, true); + BuiltinFunctions.addUnnestFun(BuiltinFunctions.FEED_COLLECT, false); + BuiltinFunctions.addDatasourceFunction(BuiltinFunctions.FEED_COLLECT, FeedRewriter.INSTANCE); + // Dataset resources function + BuiltinFunctions.addPrivateFunction(DatasetResourcesRewriter.DATASET_RESOURCES, + (expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true); + BuiltinFunctions.addUnnestFun(DatasetResourcesRewriter.DATASET_RESOURCES, false); + BuiltinFunctions.addDatasourceFunction(DatasetResourcesRewriter.DATASET_RESOURCES, + DatasetResourcesRewriter.INSTANCE); + // Dataset components function + BuiltinFunctions.addPrivateFunction(StorageComponentsRewriter.STORAGE_COMPONENTS, + (expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true); + BuiltinFunctions.addUnnestFun(StorageComponentsRewriter.STORAGE_COMPONENTS, false); + BuiltinFunctions.addDatasourceFunction(StorageComponentsRewriter.STORAGE_COMPONENTS, + StorageComponentsRewriter.INSTANCE); + + } + + private MetadataBuiltinFunctions() { + } + + public static void init() { + // Only execute the static block + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java index 9dddda4..ac249b7 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java @@ -54,7 +54,6 @@ import org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor; import org.apache.asterix.external.operators.FeedIntakeOperatorNodePushable; import org.apache.asterix.external.operators.FeedMetaOperatorDescriptor; import org.apache.asterix.external.util.ExternalDataUtils; -import org.apache.asterix.external.util.FeedConstants; import org.apache.asterix.external.util.FeedUtils; import org.apache.asterix.external.util.FeedUtils.FeedRuntimeType; import org.apache.asterix.file.StorageComponentProvider; @@ -87,6 +86,7 @@ import org.apache.asterix.metadata.entities.FeedConnection; import org.apache.asterix.metadata.entities.FeedPolicyEntity; import org.apache.asterix.metadata.feeds.FeedMetadataUtil; import org.apache.asterix.metadata.feeds.LocationConstraint; +import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.runtime.job.listener.JobEventListenerFactory; import org.apache.asterix.runtime.job.listener.MultiTransactionJobletEventListenerFactory; import org.apache.asterix.runtime.utils.RuntimeUtils; @@ -202,7 +202,7 @@ public class FeedOperations { addArgs(feedConnection.getDataverseName(), feedConnection.getFeedId().getEntityName(), feedConnection.getFeedId().getEntityName(), FeedRuntimeType.INTAKE.toString(), feedConnection.getDatasetName(), feedConnection.getOutputType()); - CallExpr datasrouceCallFunction = new CallExpr(new FunctionSignature(FeedConstants.FEED_COLLECT_FUN), exprList); + CallExpr datasrouceCallFunction = new CallExpr(new FunctionSignature(BuiltinFunctions.FEED_COLLECT), exprList); FromTerm fromterm = new FromTerm(datasrouceCallFunction, fromTermLeftExpr, null, null); FromClause fromClause = new FromClause(Arrays.asList(fromterm)); // TODO: This can be the place to add select predicate for ingestion http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.1.ddl.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.1.ddl.sqlpp new file mode 100644 index 0000000..ca38c20 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.1.ddl.sqlpp @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; +create dataverse test; + +use test; + + +create type test.LineItemType as + closed { + l_orderkey : bigint, + l_partkey : bigint, + l_suppkey : bigint, + l_linenumber : bigint, + l_quantity : double, + l_extendedprice : double, + l_discount : double, + l_tax : double, + l_returnflag : string, + l_linestatus : string, + l_shipdate : string, + l_commitdate : string, + l_receiptdate : string, + l_shipinstruct : string, + l_shipmode : string, + l_comment : string +}; + +create dataset LineItem(LineItemType) primary key l_orderkey,l_linenumber; + +create index idx_partkey on LineItem (l_partkey) type btree; + +create primary index sec_primary_idx on LineItem; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.2.update.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.2.update.sqlpp new file mode 100644 index 0000000..546a831 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.2.update.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; + + +load dataset LineItem using localfs ((`path`=`asterix_nc1://data/tpch0.001/lineitem.tbl`),(`format`=`delimited-text`),(`delimiter`=`|`)) pre-sorted; + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.3.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.3.query.sqlpp new file mode 100644 index 0000000..53af94f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.3.query.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +set `import-private-functions` `true`; +select value count(*) from dataset_resources('Metadata','Dataset') dsr; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.4.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.4.query.sqlpp new file mode 100644 index 0000000..33b4b33 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.4.query.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +set `import-private-functions` `true`; +select value (( select value count(*) from dataset_resources('test','LineItem') resource )[0] > 2); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.5.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.5.query.sqlpp new file mode 100644 index 0000000..b396dd4 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/dataset-resources/dataset-resources.5.query.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +set `import-private-functions` `true`; +select value (( select value count(*) from storage_components('test','LineItem') resource )[0] > 2); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.3.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.3.adm new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.3.adm @@ -0,0 +1 @@ +1 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.4.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.4.adm new file mode 100644 index 0000000..27ba77d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.4.adm @@ -0,0 +1 @@ +true http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.5.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.5.adm new file mode 100644 index 0000000..27ba77d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/dataset-resources/dataset-resources.5.adm @@ -0,0 +1 @@ +true http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 0c6493d..9ce7eb3 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -3445,6 +3445,11 @@ </test-group> <test-group name="misc"> <test-case FilePath="misc"> + <compilation-unit name="dataset-resources"> + <output-dir compare="Text">dataset-resources</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="misc"> <compilation-unit name="case_01"> <output-dir compare="Text">case_01</output-dir> </compilation-unit> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java index 1bb9c11..fc59f68 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.external.adapter.factory; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -36,6 +37,7 @@ import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController; import org.apache.asterix.external.dataset.adapter.FeedAdapter; import org.apache.asterix.external.dataset.adapter.GenericAdapter; import org.apache.asterix.external.indexing.ExternalFile; +import org.apache.asterix.external.parser.factory.ADMDataParserFactory; import org.apache.asterix.external.provider.DataflowControllerProvider; import org.apache.asterix.external.provider.DatasourceFactoryProvider; import org.apache.asterix.external.provider.ParserFactoryProvider; @@ -45,6 +47,7 @@ import org.apache.asterix.external.util.ExternalDataUtils; import org.apache.asterix.external.util.FeedLogManager; import org.apache.asterix.external.util.FeedUtils; import org.apache.asterix.om.types.ARecordType; +import org.apache.asterix.om.utils.RecordUtil; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.INCServiceContext; @@ -203,4 +206,20 @@ public class GenericAdapterFactory implements IIndexingAdapterFactory, IAdapterF public IExternalDataSourceFactory getDataSourceFactory() { return dataSourceFactory; } + + /** + * Use pre-configured datasource factory + * For function datasources + * + * @param dataSourceFactory + * the function datasource factory + * @throws AlgebricksException + */ + public void configure(IExternalDataSourceFactory dataSourceFactory) throws AlgebricksException { + this.dataSourceFactory = dataSourceFactory; + dataParserFactory = new ADMDataParserFactory(); + dataParserFactory.setRecordType(RecordUtil.FULLY_OPEN_RECORD_TYPE); + dataParserFactory.configure(Collections.emptyMap()); + configuration = Collections.emptyMap(); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedConstants.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedConstants.java index e2fa6db..a29d66c 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedConstants.java @@ -18,13 +18,8 @@ */ package org.apache.asterix.external.util; -import org.apache.asterix.om.functions.BuiltinFunctions; -import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; - public class FeedConstants { - public static final FunctionIdentifier FEED_COLLECT_FUN = BuiltinFunctions.FEED_COLLECT; - public final static String FEEDS_METADATA_DV = "feeds_metadata"; public final static String FAILED_TUPLE_DATASET = "failed_tuple"; public final static String FAILED_TUPLE_DATASET_TYPE = "FailedTupleType"; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IDatasourceFunction.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IDatasourceFunction.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IDatasourceFunction.java new file mode 100644 index 0000000..336c2dc --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IDatasourceFunction.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.api; + +import java.io.Serializable; + +import org.apache.asterix.external.api.IRecordReader; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; + +/** + * A function that is also a datasource + */ +public interface IDatasourceFunction extends Serializable { + + /** + * @return the locations on which the function is to be run + */ + AlgebricksAbsolutePartitionConstraint getPartitionConstraint(); + + /** + * The function record reader + * + * @param ctx + * @param partition + * @return + * @throws HyracksDataException + */ + IRecordReader<char[]> createRecordReader(IHyracksTaskContext ctx, int partition) throws HyracksDataException; + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java index d796fed..c4f5bcb 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java @@ -22,8 +22,6 @@ import org.apache.asterix.metadata.entities.Datatype; import org.apache.asterix.metadata.entities.Dataverse; import org.apache.asterix.metadata.utils.MetadataConstants; import org.apache.asterix.metadata.utils.MetadataUtil; -import org.apache.asterix.om.types.ARecordType; -import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.RecordUtil; import org.apache.asterix.runtime.formats.NonTaggedDataFormat; @@ -33,10 +31,8 @@ public class MetadataBuiltinEntities { public static final Dataverse DEFAULT_DATAVERSE = new Dataverse(DEFAULT_DATAVERSE_NAME, NonTaggedDataFormat.class.getName(), MetadataUtil.PENDING_NO_OP); //--------------------------------------- Datatypes -----------------------------------------// - public static final ARecordType ANY_OBJECT_RECORD_TYPE = - new ARecordType("AnyObject", new String[0], new IAType[0], true); public static final Datatype ANY_OBJECT_DATATYPE = new Datatype(MetadataConstants.METADATA_DATAVERSE_NAME, - ANY_OBJECT_RECORD_TYPE.getTypeName(), RecordUtil.FULLY_OPEN_RECORD_TYPE, false); + RecordUtil.FULLY_OPEN_RECORD_TYPE.getTypeName(), RecordUtil.FULLY_OPEN_RECORD_TYPE, false); private MetadataBuiltinEntities() { } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AbstractDatasourceFunction.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AbstractDatasourceFunction.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AbstractDatasourceFunction.java new file mode 100644 index 0000000..0aff4c3 --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AbstractDatasourceFunction.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.declared; + +import org.apache.asterix.metadata.api.IDatasourceFunction; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; + +public abstract class AbstractDatasourceFunction implements IDatasourceFunction { + + private static final long serialVersionUID = 1L; + private final transient AlgebricksAbsolutePartitionConstraint locations; + + public AbstractDatasourceFunction(AlgebricksAbsolutePartitionConstraint locations) { + this.locations = locations; + } + + @Override + public final AlgebricksAbsolutePartitionConstraint getPartitionConstraint() { + return locations; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java index fa874d5..ca22567 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java @@ -57,6 +57,7 @@ public abstract class DataSource implements IDataSource<DataSourceId> { public static final byte EXTERNAL_DATASET = 0x01; public static final byte FEED = 0x02; public static final byte LOADABLE = 0x03; + public static final byte FUNCTION = 0x04; // Hide implicit public constructor private Type() { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java new file mode 100644 index 0000000..d2b9871 --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.declared; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.asterix.common.cluster.IClusterStateManager; +import org.apache.asterix.external.adapter.factory.GenericAdapterFactory; +import org.apache.asterix.metadata.api.IDatasourceFunction; +import org.apache.asterix.om.types.IAType; +import org.apache.asterix.om.utils.RecordUtil; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; +import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; +import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource; +import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; +import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain; +import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty; +import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector; +import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext; +import org.apache.hyracks.api.dataflow.IOperatorDescriptor; +import org.apache.hyracks.api.job.JobSpecification; + +public abstract class FunctionDataSource extends DataSource { + + public FunctionDataSource(DataSourceId id, INodeDomain domain) throws AlgebricksException { + super(id, RecordUtil.FULLY_OPEN_RECORD_TYPE, null, DataSource.Type.FUNCTION, domain); + schemaTypes = new IAType[] { itemType }; + } + + @Override + public boolean isScanAccessPathALeaf() { + return true; + } + + @Override + public IDataSourcePropertiesProvider getPropertiesProvider() { + // Unordered Random partitioning on all nodes + return scanVariables -> new StructuralPropertiesVector(new RandomPartitioningProperty(domain), + Collections.emptyList()); + } + + @Override + public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDatasourceScanRuntime( + MetadataProvider metadataProvider, IDataSource<DataSourceId> dataSource, + List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed, + List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema, + IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig) + throws AlgebricksException { + GenericAdapterFactory adapterFactory = new GenericAdapterFactory(); + adapterFactory.setOutputType(RecordUtil.FULLY_OPEN_RECORD_TYPE); + IClusterStateManager csm = metadataProvider.getApplicationContext().getClusterStateManager(); + FunctionDataSourceFactory factory = + new FunctionDataSourceFactory(createFunction(metadataProvider, getLocations(csm))); + adapterFactory.configure(factory); + return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory); + } + + protected abstract IDatasourceFunction createFunction(MetadataProvider metadataProvider, + AlgebricksAbsolutePartitionConstraint locations); + + protected static AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) { + String[] allPartitions = csm.getClusterLocations().getLocations(); + Set<String> ncs = new HashSet<>(Arrays.asList(allPartitions)); + return new AlgebricksAbsolutePartitionConstraint(ncs.toArray(new String[ncs.size()])); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java new file mode 100644 index 0000000..1e0ddbf --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSourceFactory.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.declared; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.asterix.external.api.IRecordReader; +import org.apache.asterix.external.api.IRecordReaderFactory; +import org.apache.asterix.metadata.api.IDatasourceFunction; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.api.application.IServiceContext; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; + +public class FunctionDataSourceFactory implements IRecordReaderFactory<char[]> { + + private static final long serialVersionUID = 1L; + private final IDatasourceFunction function; + + public FunctionDataSourceFactory(IDatasourceFunction function) { + this.function = function; + } + + @Override + public final DataSourceType getDataSourceType() { + return DataSourceType.RECORDS; + } + + @Override + public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() throws AlgebricksException { + return function.getPartitionConstraint(); + } + + @Override + public void configure(IServiceContext ctx, Map<String, String> configuration) + throws AlgebricksException, HyracksDataException { + // No Op + } + + @Override + public IRecordReader<? extends char[]> createRecordReader(IHyracksTaskContext ctx, int partition) + throws HyracksDataException { + return function.createRecordReader(ctx, partition); + } + + @Override + public Class<?> getRecordClass() { + return char[].class; + } + + @Override + public List<String> getRecordReaderNames() { + return Collections.emptyList(); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/MetadataBuiltinFunctions.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/MetadataBuiltinFunctions.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/MetadataBuiltinFunctions.java deleted file mode 100644 index 137e625..0000000 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/MetadataBuiltinFunctions.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.metadata.functions; - -import org.apache.asterix.metadata.declared.MetadataProvider; -import org.apache.asterix.metadata.entities.Dataset; -import org.apache.asterix.om.functions.BuiltinFunctions; -import org.apache.asterix.om.typecomputer.base.IResultTypeComputer; -import org.apache.asterix.om.types.ATypeTag; -import org.apache.asterix.om.types.BuiltinType; -import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.utils.ConstantExpressionUtil; -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -import org.apache.hyracks.algebricks.common.utils.Pair; -import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; -import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; -import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; -import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider; - -public class MetadataBuiltinFunctions { - - static { - addMetadataBuiltinFunctions(); - BuiltinFunctions.addUnnestFun(BuiltinFunctions.DATASET, false); - BuiltinFunctions.addDatasetFunction(BuiltinFunctions.DATASET); - BuiltinFunctions.addUnnestFun(BuiltinFunctions.FEED_COLLECT, false); - BuiltinFunctions.addDatasetFunction(BuiltinFunctions.FEED_COLLECT); - BuiltinFunctions.addUnnestFun(BuiltinFunctions.FEED_INTERCEPT, false); - BuiltinFunctions.addDatasetFunction(BuiltinFunctions.FEED_INTERCEPT); - } - - public static void addMetadataBuiltinFunctions() { - - BuiltinFunctions.addFunction(BuiltinFunctions.DATASET, new IResultTypeComputer() { - - @Override - public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, - IMetadataProvider<?, ?> mp) throws AlgebricksException { - AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression; - if (f.getArguments().size() != 1) { - throw new AlgebricksException("dataset arity is 1, not " + f.getArguments().size()); - } - ILogicalExpression a1 = f.getArguments().get(0).getValue(); - IAType t1 = (IAType) env.getType(a1); - if (t1.getTypeTag() == ATypeTag.ANY) { - return BuiltinType.ANY; - } - if (t1.getTypeTag() != ATypeTag.STRING) { - throw new AlgebricksException("Illegal type " + t1 + " for dataset() argument."); - } - String datasetArg = ConstantExpressionUtil.getStringConstant(a1); - if (datasetArg == null) { - return BuiltinType.ANY; - } - MetadataProvider metadata = (MetadataProvider) mp; - Pair<String, String> datasetInfo = getDatasetInfo(metadata, datasetArg); - String dataverseName = datasetInfo.first; - String datasetName = datasetInfo.second; - if (dataverseName == null) { - throw new AlgebricksException("Unspecified dataverse!"); - } - Dataset dataset = metadata.findDataset(dataverseName, datasetName); - if (dataset == null) { - throw new AlgebricksException( - "Could not find dataset " + datasetName + " in dataverse " + dataverseName); - } - String tn = dataset.getItemTypeName(); - IAType t2 = metadata.findType(dataset.getItemTypeDataverseName(), tn); - if (t2 == null) { - throw new AlgebricksException("No type for dataset " + datasetName); - } - return t2; - } - }, true); - - BuiltinFunctions.addPrivateFunction(BuiltinFunctions.FEED_COLLECT, new IResultTypeComputer() { - - @Override - public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, - IMetadataProvider<?, ?> mp) throws AlgebricksException { - AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression; - if (f.getArguments().size() != BuiltinFunctions.FEED_COLLECT.getArity()) { - throw new AlgebricksException("Incorrect number of arguments -> arity is " - + BuiltinFunctions.FEED_COLLECT.getArity() + ", not " + f.getArguments().size()); - } - ILogicalExpression a1 = f.getArguments().get(5).getValue(); - IAType t1 = (IAType) env.getType(a1); - if (t1.getTypeTag() == ATypeTag.ANY) { - return BuiltinType.ANY; - } - if (t1.getTypeTag() != ATypeTag.STRING) { - throw new AlgebricksException("Illegal type " + t1 + " for feed-ingest argument."); - } - String typeArg = ConstantExpressionUtil.getStringConstant(a1); - if (typeArg == null) { - return BuiltinType.ANY; - } - MetadataProvider metadata = (MetadataProvider) mp; - Pair<String, String> argInfo = getDatasetInfo(metadata, typeArg); - String dataverseName = argInfo.first; - String typeName = argInfo.second; - if (dataverseName == null) { - throw new AlgebricksException("Unspecified dataverse!"); - } - IAType t2 = metadata.findType(dataverseName, typeName); - if (t2 == null) { - throw new AlgebricksException("Unknown type " + typeName); - } - return t2; - } - }, true); - - BuiltinFunctions.addFunction(BuiltinFunctions.FEED_INTERCEPT, new IResultTypeComputer() { - - @Override - public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, - IMetadataProvider<?, ?> mp) throws AlgebricksException { - AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression; - if (f.getArguments().size() != 1) { - throw new AlgebricksException("dataset arity is 1, not " + f.getArguments().size()); - } - ILogicalExpression a1 = f.getArguments().get(0).getValue(); - IAType t1 = (IAType) env.getType(a1); - if (t1.getTypeTag() == ATypeTag.ANY) { - return BuiltinType.ANY; - } - if (t1.getTypeTag() != ATypeTag.STRING) { - throw new AlgebricksException("Illegal type " + t1 + " for dataset() argument."); - } - String datasetArg = ConstantExpressionUtil.getStringConstant(a1); - if (datasetArg == null) { - return BuiltinType.ANY; - } - MetadataProvider metadata = (MetadataProvider) mp; - Pair<String, String> datasetInfo = getDatasetInfo(metadata, datasetArg); - String dataverseName = datasetInfo.first; - String datasetName = datasetInfo.second; - if (dataverseName == null) { - throw new AlgebricksException("Unspecified dataverse!"); - } - Dataset dataset = metadata.findDataset(dataverseName, datasetName); - if (dataset == null) { - throw new AlgebricksException( - "Could not find dataset " + datasetName + " in dataverse " + dataverseName); - } - String tn = dataset.getItemTypeName(); - IAType t2 = metadata.findType(dataset.getItemTypeDataverseName(), tn); - if (t2 == null) { - throw new AlgebricksException("No type for dataset " + datasetName); - } - return t2; - } - }, true); - } - - private static Pair<String, String> getDatasetInfo(MetadataProvider metadata, String datasetArg) { - String[] nameComponents = datasetArg.split("\\."); - String first; - String second; - if (nameComponents.length == 1) { - first = metadata.getDefaultDataverse() == null ? null : metadata.getDefaultDataverse().getDataverseName(); - second = nameComponents[0]; - } else { - first = nameComponents[0]; - second = nameComponents[1]; - } - return new Pair<String, String>(first, second); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java index cf663eb..e38894c 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java @@ -547,4 +547,19 @@ public class DatasetUtil { MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(nodeGroup, new ArrayList<>(ncNames))); return nodeGroup; } + + // This doesn't work if the dataset or the dataverse name contains a '.' + public static Pair<String, String> getDatasetInfo(MetadataProvider metadata, String datasetArg) { + String[] nameComponents = datasetArg.split("\\."); + String first; + String second; + if (nameComponents.length == 1) { + first = metadata.getDefaultDataverse() == null ? null : metadata.getDefaultDataverse().getDataverseName(); + second = nameComponents[0]; + } else { + first = nameComponents[0]; + second = nameComponents[1]; + } + return new Pair<>(first, second); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java index e1f5bb0..dde61f9 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java @@ -134,7 +134,7 @@ public class BuiltinFunctions { private static final Map<IFunctionInfo, IResultTypeComputer> funTypeComputer = new HashMap<>(); private static final Set<IFunctionInfo> builtinAggregateFunctions = new HashSet<>(); - private static final Set<IFunctionInfo> datasetFunctions = new HashSet<>(); + private static final Map<IFunctionInfo, IFunctionToDataSourceRewriter> datasourceFunctions = new HashMap<>(); private static final Set<IFunctionInfo> similarityFunctions = new HashSet<>(); private static final Set<IFunctionInfo> globalAggregateFunctions = new HashSet<>(); private static final Map<IFunctionInfo, IFunctionInfo> aggregateToLocalAggregate = new HashMap<>(); @@ -1280,13 +1280,6 @@ public class BuiltinFunctions { // unnesting function addPrivateFunction(SCAN_COLLECTION, CollectionMemberResultType.INSTANCE, true); - String metadataFunctionLoaderClassName = "org.apache.asterix.metadata.functions.MetadataBuiltinFunctions"; - try { - Class.forName(metadataFunctionLoaderClassName); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } static { @@ -1527,27 +1520,17 @@ public class BuiltinFunctions { } static { - datasetFunctions.add(getAsterixFunctionInfo(DATASET)); - datasetFunctions.add(getAsterixFunctionInfo(FEED_COLLECT)); - datasetFunctions.add(getAsterixFunctionInfo(FEED_INTERCEPT)); - datasetFunctions.add(getAsterixFunctionInfo(INDEX_SEARCH)); - } - - static { - addUnnestFun(DATASET, false); - addUnnestFun(FEED_COLLECT, false); - addUnnestFun(FEED_INTERCEPT, false); addUnnestFun(RANGE, true); addUnnestFun(SCAN_COLLECTION, false); addUnnestFun(SUBSET_COLLECTION, false); } - public static void addDatasetFunction(FunctionIdentifier fi) { - datasetFunctions.add(getAsterixFunctionInfo(fi)); + public static void addDatasourceFunction(FunctionIdentifier fi, IFunctionToDataSourceRewriter transformer) { + datasourceFunctions.put(getAsterixFunctionInfo(fi), transformer); } - public static boolean isDatasetFunction(FunctionIdentifier fi) { - return datasetFunctions.contains(getAsterixFunctionInfo(fi)); + public static IFunctionToDataSourceRewriter getDatasourceTransformer(FunctionIdentifier fi) { + return datasourceFunctions.getOrDefault(getAsterixFunctionInfo(fi), IFunctionToDataSourceRewriter.NOOP); } public static boolean isBuiltinCompilerFunction(FunctionSignature signature, boolean includePrivateFunctions) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionToDataSourceRewriter.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionToDataSourceRewriter.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionToDataSourceRewriter.java new file mode 100644 index 0000000..2793c4d --- /dev/null +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionToDataSourceRewriter.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.om.functions; + +import org.apache.commons.lang3.mutable.Mutable; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; +import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; + +public interface IFunctionToDataSourceRewriter { + public static final IFunctionToDataSourceRewriter NOOP = (o, c) -> false; + + /** + * Replace the unnest operator by a datasource operator + * + * @param opRef + * UnnestOperator to be replaced by DataSourceScanOperator + * @param context + * optimization context + * @return true if transformed, false otherwise + * @throws AlgebricksException + */ + boolean rewrite(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException; +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4d166b3/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/RecordUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/RecordUtil.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/RecordUtil.java index 031669a..e4a057f 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/RecordUtil.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/utils/RecordUtil.java @@ -29,7 +29,7 @@ public class RecordUtil { * A fully open record type which has the name OpenRecord */ public static final ARecordType FULLY_OPEN_RECORD_TYPE = - new ARecordType("OpenRecord", new String[0], new IAType[0], true); + new ARecordType("AnyObject", new String[0], new IAType[0], true); private RecordUtil() { } @@ -73,6 +73,7 @@ public class RecordUtil { */ public static int computeNullBitmapSize(ARecordType recordType) { return NonTaggedFormatUtil.hasOptionalField(recordType) - ? (int) Math.ceil(recordType.getFieldNames().length / 4.0) : 0; + ? (int) Math.ceil(recordType.getFieldNames().length / 4.0) + : 0; } }
