Dmitry Lychagin has submitted this change and it was merged. ( https://asterix-gerrit.ics.uci.edu/3378 )
Change subject: [NO ISSUE][CONF] Add compilation provider to NC application context ...................................................................... [NO ISSUE][CONF] Add compilation provider to NC application context - user model changes: no - storage format changes: no - interface changes: no Details: - Create NCExtensionManager in NCApplication instead of NCAppRuntimeContext (aligned with how CCApplication creates CCExtensionManager) - Pass extension manager to application context via constructor instead of setter method (for both CC and NC contexts) - Move getExtensionManager() from ICcApplicationContext to IApplicationContext, so it's available in NC application context - Add compilation provider to NC application context (NCExtensionManager.getCompilationProvider()) - Add compilation provider to NCQueryServiceServlet constructor Change-Id: Ibb4dc6d8dbc00c8a701fddb1a3205d1070078cb4 Reviewed-on: https://asterix-gerrit.ics.uci.edu/3378 Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java 8 files changed, 61 insertions(+), 50 deletions(-) Approvals: Jenkins: Verified; ; Verified Till Westmann: Looks good to me, approved Objections: Jenkins: Violations found diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java index 5a6bdd7..8923ae8 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java @@ -39,6 +39,7 @@ import org.apache.asterix.common.exceptions.RuntimeDataException; import org.apache.asterix.common.messaging.api.INCMessageBroker; import org.apache.asterix.common.messaging.api.MessageFuture; +import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.translator.IStatementExecutor; import org.apache.asterix.translator.ResultProperties; @@ -63,9 +64,9 @@ public class NCQueryServiceServlet extends QueryServiceServlet { public NCQueryServiceServlet(ConcurrentMap<String, Object> ctx, String[] paths, IApplicationContext appCtx, - ILangExtension.Language queryLanguage, + ILangExtension.Language queryLanguage, ILangCompilationProvider compilationProvider, Function<IServletRequest, Map<String, String>> optionalParamProvider) { - super(ctx, paths, appCtx, queryLanguage, null, null, null, optionalParamProvider); + super(ctx, paths, appCtx, queryLanguage, compilationProvider, null, null, optionalParamProvider); } @Override diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java index 182f982..e11302b 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java @@ -21,9 +21,7 @@ import java.io.IOException; import java.rmi.RemoteException; import java.rmi.server.UnicastRemoteObject; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -42,21 +40,18 @@ import org.apache.asterix.common.api.IReceptionistFactory; import org.apache.asterix.common.cluster.ClusterPartition; import org.apache.asterix.common.config.ActiveProperties; -import org.apache.asterix.common.config.AsterixExtension; import org.apache.asterix.common.config.BuildProperties; import org.apache.asterix.common.config.CompilerProperties; import org.apache.asterix.common.config.ExternalProperties; import org.apache.asterix.common.config.MessagingProperties; import org.apache.asterix.common.config.MetadataProperties; import org.apache.asterix.common.config.NodeProperties; -import org.apache.asterix.common.config.PropertiesAccessor; import org.apache.asterix.common.config.ReplicationProperties; import org.apache.asterix.common.config.StorageProperties; import org.apache.asterix.common.config.TransactionProperties; import org.apache.asterix.common.context.DatasetLifecycleManager; import org.apache.asterix.common.context.DatasetMemoryManager; import org.apache.asterix.common.context.IStorageComponentProvider; -import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.common.replication.IReplicationChannel; import org.apache.asterix.common.replication.IReplicationManager; @@ -157,10 +152,8 @@ private ICacheManager cacheManager; private IConfigValidator configValidator; - public NCAppRuntimeContext(INCServiceContext ncServiceContext, List<AsterixExtension> extensions, - IPropertiesFactory propertiesFactory) throws AsterixException, InstantiationException, - IllegalAccessException, ClassNotFoundException, IOException { - List<AsterixExtension> allExtensions = new ArrayList<>(); + public NCAppRuntimeContext(INCServiceContext ncServiceContext, NCExtensionManager extensionManager, + IPropertiesFactory propertiesFactory) { this.ncServiceContext = ncServiceContext; compilerProperties = propertiesFactory.newCompilerProperties(); externalProperties = propertiesFactory.newExternalProperties(); @@ -173,12 +166,7 @@ messagingProperties = propertiesFactory.newMessagingProperties(); nodeProperties = propertiesFactory.newNodeProperties(); libraryManager = new ExternalLibraryManager(); - if (extensions != null) { - allExtensions.addAll(extensions); - } - PropertiesAccessor propertiesAccessor = PropertiesAccessor.getInstance(ncServiceContext.getAppConfig()); - allExtensions.addAll(propertiesAccessor.getExtensions()); - ncExtensionManager = new NCExtensionManager(allExtensions); + ncExtensionManager = extensionManager; componentProvider = new StorageComponentProvider(); resourceIdFactory = new GlobalResourceIdFactoryProvider(ncServiceContext).createResourceIdFactory(); persistedResourceRegistry = ncServiceContext.getPersistedResourceRegistry(); @@ -484,7 +472,8 @@ .setMetadataNode(metadataNodeStub); } - public NCExtensionManager getNcExtensionManager() { + @Override + public NCExtensionManager getExtensionManager() { return ncExtensionManager; } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java index 6ea97d3..a1405a6 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java @@ -22,13 +22,19 @@ import java.util.ArrayList; import java.util.List; +import org.apache.asterix.algebra.base.ILangExtension; +import org.apache.asterix.common.api.ExtensionId; import org.apache.asterix.common.api.IExtension; import org.apache.asterix.common.config.AsterixExtension; import org.apache.asterix.common.exceptions.ACIDException; +import org.apache.asterix.compiler.provider.AqlCompilationProvider; +import org.apache.asterix.compiler.provider.ILangCompilationProvider; +import org.apache.asterix.compiler.provider.SqlppCompilationProvider; import org.apache.asterix.metadata.api.IMetadataExtension; import org.apache.asterix.metadata.api.INCExtensionManager; import org.apache.asterix.metadata.entitytupletranslators.MetadataTupleTranslatorProvider; import org.apache.asterix.utils.ExtensionUtil; +import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.application.INCServiceContext; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -38,6 +44,8 @@ */ public class NCExtensionManager implements INCExtensionManager { + private final ILangCompilationProvider aqlCompilationProvider; + private final ILangCompilationProvider sqlppCompilationProvider; private final MetadataTupleTranslatorProvider tupleTranslatorProvider; private final List<IMetadataExtension> mdExtensions; @@ -57,6 +65,8 @@ */ public NCExtensionManager(List<AsterixExtension> list) throws InstantiationException, IllegalAccessException, ClassNotFoundException, HyracksDataException { + Pair<ExtensionId, ILangCompilationProvider> aqlcp = null; + Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null; IMetadataExtension tupleTranslatorProviderExtension = null; mdExtensions = new ArrayList<>(); if (list != null) { @@ -64,6 +74,12 @@ IExtension extension = (IExtension) Class.forName(extensionConf.getClassName()).newInstance(); extension.configure(extensionConf.getArgs()); switch (extension.getExtensionKind()) { + case LANG: + ILangExtension le = (ILangExtension) extension; + aqlcp = ExtensionUtil.extendLangCompilationProvider(ILangExtension.Language.AQL, aqlcp, le); + sqlppcp = + ExtensionUtil.extendLangCompilationProvider(ILangExtension.Language.SQLPP, sqlppcp, le); + break; case METADATA: IMetadataExtension mde = (IMetadataExtension) extension; mdExtensions.add(mde); @@ -75,10 +91,23 @@ } } } + this.aqlCompilationProvider = aqlcp == null ? new AqlCompilationProvider() : aqlcp.second; + this.sqlppCompilationProvider = sqlppcp == null ? new SqlppCompilationProvider() : sqlppcp.second; this.tupleTranslatorProvider = tupleTranslatorProviderExtension == null ? new MetadataTupleTranslatorProvider() : tupleTranslatorProviderExtension.getMetadataTupleTranslatorProvider(); } + public ILangCompilationProvider getCompilationProvider(ILangExtension.Language lang) { + switch (lang) { + case AQL: + return aqlCompilationProvider; + case SQLPP: + return sqlppCompilationProvider; + default: + throw new IllegalArgumentException(String.valueOf(lang)); + } + } + public List<IMetadataExtension> getMetadataExtensions() { return mdExtensions; } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java index 76e1d88..99286a2 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java @@ -151,13 +151,10 @@ ExternalLibraryUtils.setUpExternaLibraries(libraryManager, false); componentProvider = new StorageComponentProvider(); - List<AsterixExtension> extensions = new ArrayList<>(); - extensions.addAll(getExtensions()); - ccExtensionManager = new CCExtensionManager(extensions); + ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions())); IGlobalRecoveryManager globalRecoveryManager = createGlobalRecoveryManager(); appCtx = createApplicationContext(libraryManager, globalRecoveryManager, lifecycleCoordinator, - () -> new Receptionist("CC"), ConfigValidator::new); - appCtx.setExtensionManager(ccExtensionManager); + () -> new Receptionist("CC"), ConfigValidator::new, ccExtensionManager); final CCConfig ccConfig = controllerService.getCCConfig(); if (System.getProperty("java.rmi.server.hostname") == null) { System.setProperty("java.rmi.server.hostname", ccConfig.getClusterPublicAddress()); @@ -184,11 +181,11 @@ protected ICcApplicationContext createApplicationContext(ILibraryManager libraryManager, IGlobalRecoveryManager globalRecoveryManager, INcLifecycleCoordinator lifecycleCoordinator, - IReceptionistFactory receptionistFactory, IConfigValidatorFactory configValidatorFactory) - throws AlgebricksException, IOException { + IReceptionistFactory receptionistFactory, IConfigValidatorFactory configValidatorFactory, + CCExtensionManager ccExtensionManager) throws AlgebricksException, IOException { return new CcApplicationContext(ccServiceCtx, getHcc(), libraryManager, () -> MetadataManager.INSTANCE, globalRecoveryManager, lifecycleCoordinator, new ActiveNotificationHandler(), componentProvider, - new MetadataLockManager(), receptionistFactory, configValidatorFactory); + new MetadataLockManager(), receptionistFactory, configValidatorFactory, ccExtensionManager); } protected IGlobalRecoveryManager createGlobalRecoveryManager() throws Exception { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java index 9135688..c3402f9 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java @@ -19,8 +19,8 @@ package org.apache.asterix.hyracks.bootstrap; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Set; @@ -30,6 +30,7 @@ import org.apache.asterix.app.config.ConfigValidator; import org.apache.asterix.app.io.PersistedResourceRegistry; import org.apache.asterix.app.nc.NCAppRuntimeContext; +import org.apache.asterix.app.nc.NCExtensionManager; import org.apache.asterix.app.nc.RecoveryManager; import org.apache.asterix.app.replication.message.RegistrationTasksRequestMessage; import org.apache.asterix.common.api.AsterixThreadFactory; @@ -38,6 +39,7 @@ import org.apache.asterix.common.api.IPropertiesFactory; import org.apache.asterix.common.api.IReceptionistFactory; import org.apache.asterix.common.config.AsterixExtension; +import org.apache.asterix.common.config.ExtensionProperties; import org.apache.asterix.common.config.ExternalProperties; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.config.MessagingProperties; @@ -84,6 +86,7 @@ private static final Logger LOGGER = LogManager.getLogger(); protected INCServiceContext ncServiceCtx; + protected NCExtensionManager ncExtensionManager; private INcApplicationContext runtimeContext; private String nodeId; private boolean stopInitiated; @@ -124,7 +127,9 @@ (controllerService).getConfiguration().getClusterPublicAddress()); } MetadataBuiltinFunctions.init(); - runtimeContext = new NCAppRuntimeContext(ncServiceCtx, getExtensions(), getPropertiesFactory()); + + ncExtensionManager = new NCExtensionManager(new ArrayList<>(getExtensions())); + runtimeContext = new NCAppRuntimeContext(ncServiceCtx, ncExtensionManager, getPropertiesFactory()); MetadataProperties metadataProperties = runtimeContext.getMetadataProperties(); if (!metadataProperties.getNodeNames().contains(this.ncServiceCtx.getNodeId())) { if (LOGGER.isInfoEnabled()) { @@ -187,8 +192,8 @@ webManager.add(apiServer); } - protected List<AsterixExtension> getExtensions() { - return Collections.emptyList(); + protected List<AsterixExtension> getExtensions() throws Exception { + return new ExtensionProperties(PropertiesAccessor.getInstance(ncServiceCtx.getAppConfig())).getExtensions(); } protected IPropertiesFactory getPropertiesFactory() throws IOException, AsterixException { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java index a9b306f..0340898 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IApplicationContext.java @@ -83,4 +83,11 @@ * @return the configuration validator */ IConfigValidator getConfigValidator(); + + /** + * Returns the extension manager + * + * @return the extension manager instance + */ + Object getExtensionManager(); } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java index e4b70f6..3389962 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/ICcApplicationContext.java @@ -87,13 +87,6 @@ IStorageComponentProvider getStorageComponentProvider(); /** - * Returns the extension manager - * - * @return the extension manager instance - */ - Object getExtensionManager(); - - /** * @return the metadata lock manager */ IMetadataLockManager getMetadataLockManager(); @@ -107,13 +100,6 @@ * @return the cluster state manager */ IClusterStateManager getClusterStateManager(); - - /** - * Set the extension manager - * - * @param extensionManager - */ - void setExtensionManager(Object extensionManager); /** * @return the extension properties diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java index c72bf54..a0b10c6 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/CcApplicationContext.java @@ -104,11 +104,13 @@ IGlobalRecoveryManager globalRecoveryManager, INcLifecycleCoordinator ftStrategy, IJobLifecycleListener activeLifeCycleListener, IStorageComponentProvider storageComponentProvider, IMetadataLockManager mdLockManager, IReceptionistFactory receptionistFactory, - IConfigValidatorFactory configValidatorFactory) throws AlgebricksException, IOException { + IConfigValidatorFactory configValidatorFactory, Object extensionManager) + throws AlgebricksException, IOException { this.ccServiceCtx = ccServiceCtx; this.hcc = hcc; this.libraryManager = libraryManager; this.activeLifeCycleListener = activeLifeCycleListener; + this.extensionManager = extensionManager; // Determine whether to use old-style asterix-configuration.xml or new-style configuration. // QQQ strip this out eventually PropertiesAccessor propertiesAccessor = PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig()); @@ -218,11 +220,6 @@ @Override public Object getExtensionManager() { return extensionManager; - } - - @Override - public void setExtensionManager(Object extensionManager) { - this.extensionManager = extensionManager; } @Override -- To view, visit https://asterix-gerrit.ics.uci.edu/3378 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-MessageType: merged Gerrit-Change-Id: Ibb4dc6d8dbc00c8a701fddb1a3205d1070078cb4 Gerrit-Change-Number: 3378 Gerrit-PatchSet: 4 Gerrit-Owner: Dmitry Lychagin <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose (1000171) Gerrit-Reviewer: Anon. E. Moose <[email protected]> Gerrit-Reviewer: Dmitry Lychagin <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
