This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2364a2191b82e1499ed5f6646388f0a5e586b55d Merge: b4ea906 bc0f7e0 Author: Michael Blow <[email protected]> AuthorDate: Wed May 6 10:30:06 2020 -0400 Merge commit 'bc0f7e0' from branch 'gerrit/mad-hatter' Change-Id: I629850e0f6ac7e1afae3fb5c6e1f1f859b2ead72 asterixdb/asterix-app/pom.xml | 33 +++ .../asterix/app/translator/QueryTranslator.java | 24 +- .../asterix/hyracks/bootstrap/CCApplication.java | 9 +- .../org/apache/asterix/utils/FeedOperations.java | 12 +- .../aws/AwsS3ExternalDatasetTest.java | 215 ++++++++++++++++++ .../resources/runtimets/only_external_dataset.xml | 23 ++ .../avg_mixed/avg_mixed.3.query.sqlpp | 3 +- .../serial_avg_double.1.ddl.sqlpp} | 19 +- .../serial_avg_double.2.update.sqlpp} | 18 +- .../serial_avg_double.3.query.sqlpp} | 11 +- .../serial_avg_double_null.1.ddl.sqlpp} | 19 +- .../serial_avg_double_null.2.update.sqlpp} | 18 +- .../serial_avg_double_null.3.query.sqlpp} | 11 +- .../serial_avg_empty.1.ddl.sqlpp} | 19 +- .../serial_avg_empty.2.update.sqlpp} | 1 - .../serial_avg_empty.3.query.sqlpp} | 6 + .../serial_avg_float.1.ddl.sqlpp} | 19 +- .../serial_avg_float.2.update.sqlpp} | 18 +- .../serial_avg_float.3.query.sqlpp} | 6 + .../serial_avg_float_null.1.ddl.sqlpp} | 19 +- .../serial_avg_float_null.2.update.sqlpp} | 18 +- .../serial_avg_float_null.3.query.sqlpp} | 6 + .../serial_avg_int16.1.ddl.sqlpp} | 19 +- .../serial_avg_int16.2.update.sqlpp} | 18 +- .../serial_avg_int16.3.query.sqlpp} | 6 + .../serial_avg_int16_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int16_null.2.update.sqlpp} | 18 +- .../serial_avg_int16_null.3.query.sqlpp} | 6 + .../serial_avg_int32.1.ddl.sqlpp} | 19 +- .../serial_avg_int32.2.update.sqlpp} | 18 +- .../serial_avg_int32.3.query.sqlpp} | 6 + .../serial_avg_int32_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int32_null.2.update.sqlpp} | 18 +- .../serial_avg_int32_null.3.query.sqlpp} | 6 + .../serial_avg_int64.1.ddl.sqlpp} | 19 +- .../serial_avg_int64.2.update.sqlpp} | 18 +- .../serial_avg_int64.3.query.sqlpp} | 6 + .../serial_avg_int64_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int64_null.2.update.sqlpp} | 18 +- .../serial_avg_int64_null.3.query.sqlpp} | 6 + .../serial_avg_int8.1.ddl.sqlpp} | 19 +- .../serial_avg_int8.2.update.sqlpp} | 18 +- .../serial_avg_int8.3.query.sqlpp} | 6 + .../serial_avg_int8_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int8_null.2.update.sqlpp} | 18 +- .../serial_avg_int8_null.3.query.sqlpp} | 6 + .../serial_avg_mixed.1.ddl.sqlpp} | 19 +- .../serial_avg_mixed.2.update.sqlpp} | 16 +- .../serial_avg_mixed.3.query.sqlpp} | 18 +- .../aggregate/avg_mixed/avg_mixed.3.query.sqlpp | 1 + .../serial_avg_double.1.ddl.sqlpp} | 19 +- .../serial_avg_double.2.update.sqlpp} | 18 +- .../serial_avg_double.3.query.sqlpp} | 11 +- .../serial_avg_double_null.1.ddl.sqlpp} | 19 +- .../serial_avg_double_null.2.update.sqlpp} | 18 +- .../serial_avg_double_null.3.query.sqlpp} | 11 +- .../serial_avg_empty/serial_avg_empty.1.ddl.sqlpp} | 19 +- .../serial_avg_empty.2.update.sqlpp} | 1 - .../serial_avg_empty.3.query.sqlpp} | 11 +- .../serial_avg_float/serial_avg_float.1.ddl.sqlpp} | 19 +- .../serial_avg_float.2.update.sqlpp} | 18 +- .../serial_avg_float.3.query.sqlpp} | 11 +- .../serial_avg_float_null.1.ddl.sqlpp} | 19 +- .../serial_avg_float_null.2.update.sqlpp} | 18 +- .../serial_avg_float_null.3.query.sqlpp} | 11 +- .../serial_avg_int16/serial_avg_int16.1.ddl.sqlpp} | 19 +- .../serial_avg_int16.2.update.sqlpp} | 18 +- .../serial_avg_int16.3.query.sqlpp} | 11 +- .../serial_avg_int16_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int16_null.2.update.sqlpp} | 18 +- .../serial_avg_int16_null.3.query.sqlpp} | 11 +- .../serial_avg_int32/serial_avg_int32.1.ddl.sqlpp} | 19 +- .../serial_avg_int32.2.update.sqlpp} | 18 +- .../serial_avg_int32.3.query.sqlpp} | 11 +- .../serial_avg_int32_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int32_null.2.update.sqlpp} | 18 +- .../serial_avg_int32_null.3.query.sqlpp} | 11 +- .../serial_avg_int64/serial_avg_int64.1.ddl.sqlpp} | 19 +- .../serial_avg_int64.2.update.sqlpp} | 18 +- .../serial_avg_int64.3.query.sqlpp} | 11 +- .../serial_avg_int64_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int64_null.2.update.sqlpp} | 18 +- .../serial_avg_int64_null.3.query.sqlpp} | 11 +- .../serial_avg_int8/serial_avg_int8.1.ddl.sqlpp} | 19 +- .../serial_avg_int8.2.update.sqlpp} | 18 +- .../serial_avg_int8/serial_avg_int8.3.query.sqlpp} | 11 +- .../serial_avg_int8_null.1.ddl.sqlpp} | 19 +- .../serial_avg_int8_null.2.update.sqlpp} | 18 +- .../serial_avg_int8_null.3.query.sqlpp} | 11 +- .../serial_avg_mixed/serial_avg_mixed.1.ddl.sqlpp} | 19 +- .../serial_avg_mixed.2.update.sqlpp} | 16 +- .../serial_avg_mixed.3.query.sqlpp} | 17 +- .../aws/s3/000/external_dataset.000.ddl.sqlpp} | 23 +- .../aws/s3/000/external_dataset.001.query.sqlpp} | 6 + .../aws/s3/000/external_dataset.002.ddl.sqlpp} | 1 + .../aggregate-sql/avg_mixed/avg_mixed.1.adm | 2 +- .../serial_avg_double/serial_avg_double.1.adm | 1 + .../serial_avg_double_null.1.adm | 1 + .../serial_avg_empty/serial_sum_empty.1.adm} | 0 .../serial_avg_float/serial_avg_float.1.adm | 1 + .../serial_avg_float_null.1.adm | 1 + .../serial_avg_int16/serial_avg_int16.1.adm | 1 + .../serial_avg_int16_null.1.adm | 1 + .../serial_avg_int32/serial_avg_int32.1.adm | 1 + .../serial_avg_int32_null.1.adm | 1 + .../serial_avg_int64/serial_avg_int64.1.adm | 1 + .../serial_avg_int64_null.1.adm | 1 + .../serial_avg_int8/serial_avg_int8.1.adm | 1 + .../serial_avg_int8_null.1.adm | 1 + .../serial_avg_mixed/serial_avg_mixed.1.adm | 1 + .../results/aggregate/avg_mixed/avg_mixed.1.adm | 2 +- .../serial_avg_double/serial_avg_double.1.adm | 1 + .../serial_avg_double_null.1.adm | 1 + .../serial_avg_empty/serial_sum_empty.1.adm} | 0 .../serial_avg_float/serial_avg_float.1.adm | 1 + .../serial_avg_float_null.1.adm | 1 + .../serial_avg_int16/serial_avg_int16.1.adm | 1 + .../serial_avg_int16_null.1.adm | 1 + .../serial_avg_int32/serial_avg_int32.1.adm | 1 + .../serial_avg_int32_null.1.adm | 1 + .../serial_avg_int64/serial_avg_int64.1.adm | 1 + .../serial_avg_int64_null.1.adm | 1 + .../serial_avg_int8/serial_avg_int8.1.adm | 1 + .../serial_avg_int8_null.1.adm | 1 + .../serial_avg_mixed/serial_avg_mixed.1.adm | 1 + .../aws/s3/000/external_dataset.001.adm | 14 ++ .../results/feeds/feeds_01/feeds_01.1.adm | 2 +- .../aggregate-sql/avg_mixed/avg_mixed.1.ast | 0 .../aggregate-sql/avg_mixed/avg_mixed.2.ast | 0 .../aggregate-sql/avg_mixed/avg_mixed.3.ast | 2 +- .../runtimets/testsuite_external_dataset.xml | 28 +++ .../test/resources/runtimets/testsuite_sqlpp.xml | 150 ++++++++++++- .../common/dataflow/ICcApplicationContext.java | 16 ++ .../asterix/common/external}/IAdapterFactory.java | 31 +-- .../common/external/IAdapterFactoryService.java} | 16 +- .../common/external}/IDataSourceAdapter.java | 2 +- asterixdb/asterix-external-data/pom.xml | 12 + .../factory/AdapterFactoryService.java} | 18 +- .../adapter/factory/GenericAdapterFactory.java | 6 +- .../external/api/IIndexingAdapterFactory.java | 2 +- .../asterix/external/api/ITypedAdapterFactory.java | 54 +++++ .../external/dataset/adapter/FeedAdapter.java | 2 +- .../external/dataset/adapter/GenericAdapter.java | 2 +- .../input/record/reader/aws/AwsS3InputStream.java | 163 ++++++++++++++ .../record/reader/aws/AwsS3InputStreamFactory.java | 250 +++++++++++++++++++++ .../record/reader/aws/AwsS3ReaderFactory.java | 90 ++++++++ .../operators/ExternalScanOperatorDescriptor.java | 8 +- .../operators/FeedIntakeOperatorDescriptor.java | 16 +- .../operators/FeedIntakeOperatorNodePushable.java | 4 +- .../external/provider/AdapterFactoryProvider.java | 9 +- .../provider/DatasourceFactoryProvider.java | 1 + .../util/ExternalDataCompatibilityUtils.java | 7 +- .../external/util/ExternalDataConstants.java | 23 ++ ...pache.asterix.external.api.IRecordReaderFactory | 1 + .../library/adapter/TestTypedAdapterFactory.java | 6 +- .../asterix/lang/common/statement/DatasetDecl.java | 4 +- .../common/util/DatasetDeclParametersUtil.java | 18 +- .../metadata/bootstrap/MetadataBootstrap.java | 7 +- .../metadata/declared/DatasetDataSource.java | 4 +- .../metadata/declared/LoadableDataSource.java | 4 +- .../metadata/declared/MetadataProvider.java | 18 +- .../metadata/entities/DatasourceAdapter.java | 2 +- .../DatasourceAdapterTupleTranslator.java | 2 +- .../asterix/metadata/feeds/FeedMetadataUtil.java | 22 +- .../metadata/utils/ExternalIndexingOperations.java | 4 +- .../AbstractSerializableAvgAggregateFunction.java | 36 ++- .../std/AbstractAvgAggregateFunction.java | 35 ++- .../runtime/utils/CcApplicationContext.java | 12 +- asterixdb/asterix-server/pom.xml | 6 + asterixdb/pom.xml | 79 +++++++ ...streams_reactive-streams-jvm_v1.0.2_COPYING.txt | 121 ++++++++++ ...streams_reactive-streams-jvm_v1.0.2_LICENSE.txt | 8 + 172 files changed, 2402 insertions(+), 559 deletions(-) diff --cc asterixdb/asterix-app/pom.xml index e5e978a,219595b..a26a335 --- a/asterixdb/asterix-app/pom.xml +++ b/asterixdb/asterix-app/pom.xml @@@ -819,18 -699,38 +819,51 @@@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.mindrot</groupId> + <artifactId>jbcrypt</artifactId> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-csv</artifactId> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpmime</artifactId> + <scope>test</scope> + </dependency> + <!-- AWS --> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>sdk-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>s3</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>regions</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>auth</artifactId> + <scope>test</scope> + </dependency> + <!-- Mock for AWS S3 --> + <dependency> + <groupId>io.findify</groupId> + <artifactId>s3mock_2.12</artifactId> + <scope>test</scope> + </dependency> + <!-- Needed for the s3 mock --> + <dependency> + <groupId>com.typesafe.akka</groupId> + <artifactId>akka-http-core_2.12</artifactId> + <scope>test</scope> + </dependency> </dependencies> </project> diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 067ee6e,b8a048d..eefb7f3 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@@ -78,17 -76,11 +78,17 @@@ import org.apache.asterix.common.except import org.apache.asterix.common.exceptions.RuntimeDataException; import org.apache.asterix.common.exceptions.WarningCollector; import org.apache.asterix.common.exceptions.WarningUtil; ++import org.apache.asterix.common.external.IDataSourceAdapter; +import org.apache.asterix.common.functions.ExternalFunctionLanguage; import org.apache.asterix.common.functions.FunctionSignature; +import org.apache.asterix.common.messaging.api.ICCMessageBroker; +import org.apache.asterix.common.metadata.DataverseName; +import org.apache.asterix.common.metadata.IMetadataLockUtil; import org.apache.asterix.common.utils.JobUtils; import org.apache.asterix.common.utils.JobUtils.ProgressState; import org.apache.asterix.common.utils.StorageConstants; import org.apache.asterix.compiler.provider.ILangCompilationProvider; - import org.apache.asterix.external.api.IDataSourceAdapter; +import org.apache.asterix.external.dataset.adapter.AdapterIdentifier; import org.apache.asterix.external.indexing.ExternalFile; import org.apache.asterix.external.indexing.IndexingConstants; import org.apache.asterix.external.operators.FeedIntakeOperatorNodePushable; @@@ -168,7 -153,10 +168,9 @@@ import org.apache.asterix.metadata.util import org.apache.asterix.metadata.utils.IndexUtil; import org.apache.asterix.metadata.utils.KeyFieldTypeUtil; import org.apache.asterix.metadata.utils.MetadataConstants; -import org.apache.asterix.metadata.utils.MetadataLockUtil; import org.apache.asterix.metadata.utils.MetadataUtil; + import org.apache.asterix.object.base.AdmObjectNode; + import org.apache.asterix.object.base.AdmStringNode; import org.apache.asterix.om.base.IAObject; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java index bfc8154,fc912b0..ee5b957 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java @@@ -73,10 -69,11 +73,12 @@@ import org.apache.asterix.common.config import org.apache.asterix.common.config.ReplicationProperties; import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.dataflow.ICcApplicationContext; + import org.apache.asterix.common.external.IAdapterFactoryService; import org.apache.asterix.common.library.ILibraryManager; +import org.apache.asterix.common.metadata.IMetadataLockUtil; import org.apache.asterix.common.replication.INcLifecycleCoordinator; import org.apache.asterix.common.utils.Servlets; + import org.apache.asterix.external.adapter.factory.AdapterFactoryService; import org.apache.asterix.external.library.ExternalLibraryManager; import org.apache.asterix.file.StorageComponentProvider; import org.apache.asterix.messaging.CCMessageBroker; @@@ -154,13 -149,14 +156,13 @@@ public class CCApplication extends Base ReplicationProperties repProp = new ReplicationProperties(PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig())); INcLifecycleCoordinator lifecycleCoordinator = createNcLifeCycleCoordinator(repProp.isReplicationEnabled()); - ExternalLibraryUtils.setUpInstalledLibraries(libraryManager, false, ccServiceCtx.getServerCtx().getAppDir()); componentProvider = new StorageComponentProvider(); - ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions())); IGlobalRecoveryManager globalRecoveryManager = createGlobalRecoveryManager(); + ILibraryManager libraryManager = new ExternalLibraryManager(ccServiceCtx.getServerCtx().getAppDir(), + ccServiceCtx.getPersistedResourceRegistry()); appCtx = createApplicationContext(libraryManager, globalRecoveryManager, lifecycleCoordinator, - () -> new Receptionist("CC"), ConfigValidator::new, ccExtensionManager); + () -> new Receptionist("CC"), ConfigValidator::new, ccExtensionManager, new AdapterFactoryService()); final CCConfig ccConfig = controllerService.getCCConfig(); if (System.getProperty("java.rmi.server.hostname") == null) { System.setProperty("java.rmi.server.hostname", ccConfig.getClusterPublicAddress()); @@@ -208,11 -184,12 +210,12 @@@ protected ICcApplicationContext createApplicationContext(ILibraryManager libraryManager, IGlobalRecoveryManager globalRecoveryManager, INcLifecycleCoordinator lifecycleCoordinator, IReceptionistFactory receptionistFactory, IConfigValidatorFactory configValidatorFactory, - CCExtensionManager ccExtensionManager) throws AlgebricksException, IOException { + CCExtensionManager ccExtensionManager, IAdapterFactoryService adapterFactoryService) + throws AlgebricksException, IOException { return new CcApplicationContext(ccServiceCtx, getHcc(), libraryManager, () -> MetadataManager.INSTANCE, globalRecoveryManager, lifecycleCoordinator, new ActiveNotificationHandler(), componentProvider, - new MetadataLockManager(), receptionistFactory, configValidatorFactory, ccExtensionManager, - adapterFactoryService); + new MetadataLockManager(), createMetadataLockUtil(), receptionistFactory, configValidatorFactory, - ccExtensionManager); ++ ccExtensionManager, adapterFactoryService); } protected IGlobalRecoveryManager createGlobalRecoveryManager() throws Exception { diff --cc asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java index d764b98,5fc1bb7..ad90814 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java @@@ -26,8 -27,8 +27,9 @@@ import org.apache.asterix.common.cluste import org.apache.asterix.common.cluster.IGlobalRecoveryManager; import org.apache.asterix.common.config.ExtensionProperties; import org.apache.asterix.common.context.IStorageComponentProvider; + import org.apache.asterix.common.external.IAdapterFactoryService; import org.apache.asterix.common.metadata.IMetadataBootstrap; +import org.apache.asterix.common.metadata.IMetadataLockUtil; import org.apache.asterix.common.replication.INcLifecycleCoordinator; import org.apache.asterix.common.storage.ICompressionManager; import org.apache.asterix.common.transactions.IResourceIdManager; diff --cc asterixdb/asterix-external-data/pom.xml index 08ffb16,30e7770..f6ebdba --- a/asterixdb/asterix-external-data/pom.xml +++ b/asterixdb/asterix-external-data/pom.xml @@@ -445,16 -436,16 +445,28 @@@ <artifactId>netty-all</artifactId> </dependency> <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>net.razorvine</groupId> + <artifactId>pyrolite</artifactId> + </dependency> + <dependency> + <groupId>net.razorvine</groupId> + <artifactId>serpent</artifactId> + </dependency> ++ <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>s3</artifactId> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>regions</artifactId> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>auth</artifactId> + </dependency> </dependencies> </project> diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java index 3b2d95f,d63e8a8..779b05f --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java @@@ -24,13 -24,10 +24,13 @@@ import org.apache.asterix.active.Entity import org.apache.asterix.common.api.INcApplicationContext; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.exceptions.RuntimeDataException; +import org.apache.asterix.common.functions.ExternalFunctionLanguage; +import org.apache.asterix.common.library.ILibrary; import org.apache.asterix.common.library.ILibraryManager; - import org.apache.asterix.external.api.IAdapterFactory; + import org.apache.asterix.external.api.ITypedAdapterFactory; import org.apache.asterix.external.feed.api.IFeed; import org.apache.asterix.external.feed.policy.FeedPolicyAccessor; +import org.apache.asterix.external.library.JavaLibrary; import org.apache.asterix.om.types.ARecordType; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.IOperatorNodePushable; @@@ -108,15 -105,10 +108,15 @@@ public class FeedIntakeOperatorDescript INcApplicationContext runtimeCtx = (INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext(); ILibraryManager libraryManager = runtimeCtx.getLibraryManager(); - ClassLoader classLoader = libraryManager.getLibraryClassLoader(feedId.getDataverse(), adaptorLibraryName); + + ILibrary lib = libraryManager.getLibrary(feedId.getDataverseName(), adaptorLibraryName); + if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) { + throw new HyracksDataException("Unexpected library language: " + lib.getLanguage()); + } + ClassLoader classLoader = ((JavaLibrary) lib).getClassLoader(); if (classLoader != null) { try { - adapterFactory = (IAdapterFactory) (classLoader.loadClass(adaptorFactoryClassName).newInstance()); + adapterFactory = (ITypedAdapterFactory) (classLoader.loadClass(adaptorFactoryClassName).newInstance()); adapterFactory.setOutputType(adapterOutputType); adapterFactory.configure(ctx.getJobletContext().getServiceContext(), adaptorConfiguration); } catch (Exception e) { diff --cc asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java index b6a3d66,45fc33a..811a494 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java @@@ -45,22 -44,30 +45,22 @@@ public class DatasetDecl extends Abstra protected final DatasetType datasetType; protected final IDatasetDetailsDecl datasetDetailsDecl; protected final Map<String, String> hints; - private final AdmObjectNode withObjectNode; + private AdmObjectNode withObjectNode; protected final boolean ifNotExists; - public DatasetDecl(Identifier dataverse, Identifier name, Identifier itemTypeDataverse, Identifier itemTypeName, - Identifier metaItemTypeDataverse, Identifier metaItemTypeName, Identifier nodeGroupName, - Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd, RecordConstructor withRecord, - boolean ifNotExists) throws CompilationException { + public DatasetDecl(DataverseName dataverse, Identifier name, DataverseName itemTypeDataverse, + Identifier itemTypeName, DataverseName metaItemTypeDataverse, Identifier metaItemTypeName, + Identifier nodeGroupName, Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd, + RecordConstructor withRecord, boolean ifNotExists) throws CompilationException { this.dataverse = dataverse; this.name = name; this.itemTypeName = itemTypeName; - if (itemTypeDataverse.getValue() == null) { - this.itemTypeDataverse = dataverse; - } else { - this.itemTypeDataverse = itemTypeDataverse; - } + this.itemTypeDataverse = itemTypeDataverse == null ? dataverse : itemTypeDataverse; this.metaItemTypeName = metaItemTypeName; - if (metaItemTypeDataverse == null || metaItemTypeDataverse.getValue() == null) { - this.metaItemTypeDataverse = dataverse; - } else { - this.metaItemTypeDataverse = metaItemTypeDataverse; - } + this.metaItemTypeDataverse = metaItemTypeDataverse == null ? dataverse : metaItemTypeDataverse; this.nodegroupName = nodeGroupName; this.hints = hints; - this.withObjectNode = DatasetDeclParametersUtil.validateAndGetWithObjectNode(withRecord); + this.withObjectNode = DatasetDeclParametersUtil.validateAndGetWithObjectNode(withRecord, datasetType); this.ifNotExists = ifNotExists; this.datasetType = datasetType; this.datasetDetailsDecl = idd; diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java index f5ca79a,07bbc57..81646e20 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java @@@ -23,8 -23,7 +23,8 @@@ import java.util.List import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.common.metadata.DataverseName; - import org.apache.asterix.external.api.IAdapterFactory; + import org.apache.asterix.external.api.ITypedAdapterFactory; import org.apache.asterix.metadata.IDatasetDetails; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.entities.Dataset; diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java index e1ad67e,c2983af..4d4b17a --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java @@@ -26,8 -26,7 +26,8 @@@ import java.util.Map import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.common.metadata.DataverseName; - import org.apache.asterix.external.api.IAdapterFactory; + import org.apache.asterix.external.api.ITypedAdapterFactory; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.InternalDatasetDetails; import org.apache.asterix.om.types.ARecordType; diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index ff68e78,5bdf2a7..93e214e --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@@ -40,7 -39,7 +40,8 @@@ import org.apache.asterix.common.contex import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.dataflow.LSMTreeInsertDeleteOperatorDescriptor; import org.apache.asterix.common.exceptions.AsterixException; + import org.apache.asterix.common.external.IDataSourceAdapter; +import org.apache.asterix.common.metadata.DataverseName; import org.apache.asterix.common.metadata.LockList; import org.apache.asterix.common.storage.ICompressionManager; import org.apache.asterix.common.transactions.ITxnIdFactory; @@@ -817,11 -775,11 +817,11 @@@ public class MetadataProvider implement return numElementsHint / numPartitions; } - protected IAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName, + protected ITypedAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName, Map<String, String> configuration, ARecordType itemType, ARecordType metaType) throws AlgebricksException { try { - configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName()); + configuration.put(ExternalDataConstants.KEY_DATAVERSE, dataset.getDataverseName().getCanonicalForm()); - IAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory( + ITypedAdapterFactory adapterFactory = AdapterFactoryProvider.getAdapterFactory( getApplicationContext().getServiceContext(), adapterName, configuration, itemType, metaType); // check to see if dataset is indexed diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java index a94adfa,9e65c08..eb37a60 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java @@@ -19,13 -19,14 +19,13 @@@ package org.apache.asterix.metadata.entitytupletranslators; -import java.io.ByteArrayInputStream; -import java.io.DataInput; -import java.io.DataInputStream; +import static org.apache.asterix.metadata.bootstrap.MetadataRecordTypes.DATASOURCE_ARECORD_FUNCTION_LIBRARY_FIELD_NAME; + import java.util.Calendar; + import org.apache.asterix.common.external.IDataSourceAdapter; +import org.apache.asterix.common.metadata.DataverseName; - import org.apache.asterix.external.api.IDataSourceAdapter; import org.apache.asterix.external.dataset.adapter.AdapterIdentifier; -import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.DatasourceAdapter; diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java index 790d7dd,7ed53e4..09eaa8b --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java @@@ -28,15 -28,11 +28,15 @@@ import org.apache.asterix.common.except import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.exceptions.MetadataException; + import org.apache.asterix.common.external.IDataSourceAdapter; + import org.apache.asterix.common.external.IDataSourceAdapter.AdapterType; +import org.apache.asterix.common.functions.ExternalFunctionLanguage; +import org.apache.asterix.common.library.ILibrary; +import org.apache.asterix.common.metadata.DataverseName; - import org.apache.asterix.external.api.IAdapterFactory; - import org.apache.asterix.external.api.IDataSourceAdapter; - import org.apache.asterix.external.api.IDataSourceAdapter.AdapterType; + import org.apache.asterix.external.api.ITypedAdapterFactory; import org.apache.asterix.external.feed.api.IFeed; import org.apache.asterix.external.feed.policy.FeedPolicyAccessor; +import org.apache.asterix.external.library.JavaLibrary; import org.apache.asterix.external.provider.AdapterFactoryProvider; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.ExternalDataUtils; @@@ -133,12 -128,9 +133,12 @@@ public class FeedMetadataUtil case EXTERNAL: String[] anameComponents = adapterName.split("#"); String libraryName = anameComponents[0]; - ClassLoader cl = - appCtx.getLibraryManager().getLibraryClassLoader(feed.getDataverseName(), libraryName); + ILibrary lib = appCtx.getLibraryManager().getLibrary(feed.getDataverseName(), libraryName); + if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) { + throw new HyracksDataException("Unexpected library language: " + lib.getLanguage()); + } + ClassLoader cl = ((JavaLibrary) lib).getClassLoader(); - adapterFactory = (IAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance(); + adapterFactory = (ITypedAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance(); break; default: throw new AsterixException("Unknown Adapter type " + adapterType); @@@ -209,12 -201,9 +209,12 @@@ case EXTERNAL: String[] anameComponents = adapterName.split("#"); String libraryName = anameComponents[0]; - ClassLoader cl = - appCtx.getLibraryManager().getLibraryClassLoader(feed.getDataverseName(), libraryName); + ILibrary lib = appCtx.getLibraryManager().getLibrary(feed.getDataverseName(), libraryName); + if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) { + throw new HyracksDataException("Unexpected library language: " + lib.getLanguage()); + } + ClassLoader cl = ((JavaLibrary) lib).getClassLoader(); - adapterFactory = (IAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance(); + adapterFactory = (ITypedAdapterFactory) cl.loadClass(adapterFactoryClassname).newInstance(); break; default: throw new AsterixException("Unknown Adapter type " + adapterType); diff --cc asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java index 3de4a88,3366ac1..6def41a --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java @@@ -45,9 -45,9 +45,10 @@@ import org.apache.asterix.common.config import org.apache.asterix.common.config.TransactionProperties; import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.dataflow.ICcApplicationContext; + import org.apache.asterix.common.external.IAdapterFactoryService; import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.common.metadata.IMetadataBootstrap; +import org.apache.asterix.common.metadata.IMetadataLockUtil; import org.apache.asterix.common.replication.INcLifecycleCoordinator; import org.apache.asterix.common.storage.ICompressionManager; import org.apache.asterix.common.transactions.IResourceIdManager; @@@ -105,9 -105,9 +107,9 @@@ public class CcApplicationContext imple ILibraryManager libraryManager, Supplier<IMetadataBootstrap> metadataBootstrapSupplier, IGlobalRecoveryManager globalRecoveryManager, INcLifecycleCoordinator ftStrategy, IJobLifecycleListener activeLifeCycleListener, IStorageComponentProvider storageComponentProvider, - IMetadataLockManager mdLockManager, IReceptionistFactory receptionistFactory, + IMetadataLockManager mdLockManager, IMetadataLockUtil mdLockUtil, IReceptionistFactory receptionistFactory, - IConfigValidatorFactory configValidatorFactory, Object extensionManager) - throws AlgebricksException, IOException { + IConfigValidatorFactory configValidatorFactory, Object extensionManager, + IAdapterFactoryService adapterFactoryService) throws AlgebricksException, IOException { this.ccServiceCtx = ccServiceCtx; this.hcc = hcc; this.libraryManager = libraryManager; diff --cc asterixdb/asterix-server/pom.xml index e6d7b69,7acfc04..b660379 --- a/asterixdb/asterix-server/pom.xml +++ b/asterixdb/asterix-server/pom.xml @@@ -173,9 -173,10 +173,14 @@@ <noticeUrl>https://raw.githubusercontent.com/netty/netty/netty-4.1.46.Final/NOTICE.txt</noticeUrl> </override> <override> + <gav>org.reactivestreams:reactive-streams:1.0.2</gav> + <noticeUrl>https://raw.githubusercontent.com/reactive-streams/reactive-streams-jvm/v1.0.2/COPYING.txt</noticeUrl> + <url>https://raw.githubusercontent.com/reactive-streams/reactive-streams-jvm/v1.0.2/LICENSE.txt</url> + </override> ++ <override> + <gav>org.mindrot:jbcrypt:0.4</gav> + <url>http://www.mindrot.org/files/jBCrypt/LICENSE</url> + </override> </overrides> <licenses> <license> diff --cc asterixdb/pom.xml index 69ee90c,76ec3d2..7d58b73 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@@ -1358,40 -1342,83 +1359,118 @@@ <version>0.9.12</version> </dependency> <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>s3</artifactId> + <version>${awsjavasdk.version}</version> + <exclusions> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-codec-http</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-codec-http2</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-codec</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-common</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-buffer</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-handler</artifactId> + </exclusion> + <exclusion> + <groupId>io.netty</groupId> + <artifactId>netty-transport-native-epoll</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>regions</artifactId> + <version>${awsjavasdk.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>auth</artifactId> + <version>${awsjavasdk.version}</version> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>sdk-core</artifactId> + <version>${awsjavasdk.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <!-- Mock for AWS S3 --> + <dependency> + <groupId>io.findify</groupId> + <artifactId>s3mock_2.12</artifactId> + <version>0.2.5</version> + </dependency> + <!-- Needed for the s3 mock --> + <dependency> + <groupId>com.typesafe.akka</groupId> + <artifactId>akka-http-core_2.12</artifactId> + <version>10.1.0</version> + </dependency> ++ <dependency> + <groupId>org.mindrot</groupId> + <artifactId>jbcrypt</artifactId> + <version>0.4</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-csv</artifactId> + <version>1.8</version> + </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>postgresql</artifactId> + <version>1.13.0</version> + </dependency> + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>42.2.10</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpmime</artifactId> + <version>4.5.11</version> + </dependency> + <dependency> + <groupId>net.razorvine</groupId> + <artifactId>pyrolite</artifactId> + <version>4.30</version> + </dependency> + <dependency> + <groupId>net.razorvine</groupId> + <artifactId>serpent</artifactId> + <version>1.23</version> + </dependency> </dependencies> </dependencyManagement>
