This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new ac782ed17c [ASTERIXDB-3322][FUN] Adapt external artifacts to DB
ac782ed17c is described below
commit ac782ed17c211fe2386d3aa4ba0974eebaad930a
Author: Ian Maxon <[email protected]>
AuthorDate: Tue Dec 12 08:08:58 2023 -0800
[ASTERIXDB-3322][FUN] Adapt external artifacts to DB
- user model changes: yes
- storage format changes: yes
- interface changes: yes
Details:
Adapt libraries and consumers of libraries to use
the new Database level when appropriate. Add a test
to run some of the library-related tests with this.
Change-Id: I809109b232bc5a5db0c47a52cb98c838ff55ea7e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17988
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
---
asterixdb/asterix-app/pom.xml | 4 +-
.../asterix/api/http/server/NCUdfApiServlet.java | 6 +-
.../app/external/ExternalLibraryJobUtils.java | 27 +++---
.../asterix/app/translator/QueryTranslator.java | 22 ++---
.../org/apache/asterix/utils/FeedOperations.java | 6 +-
.../test/cloud_storage/CloudPythonTest.java | 96 +++++++++++++++++++
.../test/runtime/ExternalPythonFunctionIT.java | 2 +-
.../library_list_api.5.regexjson | 6 +-
.../resources/runtimets/testsuite_it_python.xml | 1 +
.../asterix/common/api/INamespacePathResolver.java | 2 +
.../asterix/common/library/ILibraryManager.java | 14 +--
.../apache/asterix/common/metadata/Namespace.java | 10 +-
.../common/metadata/NamespacePathResolver.java | 5 +
.../adapter/factory/ExternalAdapterFactory.java | 14 ++-
.../external/library/ExternalLibraryManager.java | 106 ++++++++++++---------
.../ExternalScalarJavaFunctionEvaluator.java | 6 +-
.../PythonLibraryDomainSocketEvaluator.java | 5 +-
.../external/library/PythonLibraryEvaluatorId.java | 16 ++--
.../library/PythonLibraryTCPSocketEvaluator.java | 5 +-
.../AbstractLibraryOperatorDescriptor.java | 12 +--
.../operators/FeedIntakeOperatorDescriptor.java | 16 +++-
.../LibraryDeployAbortOperatorDescriptor.java | 8 +-
.../LibraryDeployCommitOperatorDescriptor.java | 8 +-
.../LibraryDeployPrepareOperatorDescriptor.java | 8 +-
.../LibraryUndeployOperatorDescriptor.java | 8 +-
.../asterix/external/util/ExternalDataUtils.java | 10 +-
.../external/util/ExternalLibraryUtils.java | 10 +-
.../metadata/declared/MetadataProvider.java | 6 +-
.../LibraryTupleTranslator.java | 3 +-
.../asterix/metadata/feeds/FeedMetadataUtil.java | 2 +-
.../functions/ExternalFunctionCompilerUtil.java | 7 +-
.../functions/ExternalScalarFunctionInfo.java | 9 +-
.../asterix/om/functions/ExternalFunctionInfo.java | 16 ++--
.../om/functions/IExternalFunctionInfo.java | 4 +-
34 files changed, 315 insertions(+), 165 deletions(-)
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index ed6f5e3261..f285611975 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -448,7 +448,7 @@
<profile>
<id>asterix-gerrit-asterix-app</id>
<properties>
-
<test.excludes>**/CloudStorageTest.java,**/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/SqlppExecutionColumnTest.java,**/*StaticPartitioning*Test.java,**/*Ssl*Test.java,**/Podman*.java,**/*AnalyzedExecutionTest.java,**/SqlppProfiledExecutionTest.java</test.excludes>
+
<test.excludes>**/CloudStorageTest.java,**/SqlppExecutionWithCancellationTest.java,**/DmlTest.java,**/RepeatedTest.java,**/SqlppExecutionTest.java,**/SqlppExecutionColumnTest.java,**/*StaticPartitioning*Test.java,**/*Ssl*Test.java,**/Podman*.java,**/*AnalyzedExecutionTest.java,**/SqlppProfiledExecutionTest.java,**/CloudPythonTest.java</test.excludes>
<itest.excludes>**/*.java</itest.excludes>
</properties>
<build>
@@ -482,7 +482,7 @@
<id>asterix-gerrit-asterix-app-sql-execution</id>
<properties>
<test.excludes>**/*.java</test.excludes>
-
<itest.includes>**/SqlppExecution*IT.java,**/ExternalPythonFunctionIT.java</itest.includes>
+
<itest.includes>**/SqlppExecution*IT.java,**/ExternalPythonFunctionIT.java,**/CloudPythonTest.java</itest.includes>
<failIfNoTests>false</failIfNoTests>
</properties>
</profile>
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
index 9698f3b136..74beb46b62 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCUdfApiServlet.java
@@ -181,13 +181,13 @@ public class NCUdfApiServlet extends AbstractNCUdfServlet
{
try {
if (localPath.equals("/") || localPath.equals("")) {
//TODO: nicer way to get this into display form?
- Map<DataverseName, Map<String, String>> dvToLibHashes =
+ Map<Namespace, Map<String, String>> dvToLibHashes =
ExternalLibraryUtils.produceLibraryListing(libraryManager);
List<Map<String, Object>> libraryList = new ArrayList<>();
- for (Map.Entry<DataverseName, Map<String, String>> dvAndLibs :
dvToLibHashes.entrySet()) {
+ for (Map.Entry<Namespace, Map<String, String>> dvAndLibs :
dvToLibHashes.entrySet()) {
for (Map.Entry<String, String> libsInDv :
dvAndLibs.getValue().entrySet()) {
Map<String, Object> libraryEntry = new HashMap<>();
- libraryEntry.put(getDataverseKey(),
dvAndLibs.getKey().getCanonicalForm());
+ libraryEntry.put(getDataverseKey(),
libraryManager.getNsOrDv(dvAndLibs.getKey()));
libraryEntry.put(NAME_KEY, libsInDv.getKey());
libraryEntry.put(FIELD_HASH, libsInDv.getValue());
libraryList.add(libraryEntry);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java
index 6c6691b674..f32b8ef246 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java
@@ -28,7 +28,7 @@ import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.common.utils.StoragePathUtil;
import
org.apache.asterix.external.operators.LibraryDeployAbortOperatorDescriptor;
import
org.apache.asterix.external.operators.LibraryDeployCommitOperatorDescriptor;
@@ -51,30 +51,29 @@ public class ExternalLibraryJobUtils {
}
public static Triple<JobSpecification, JobSpecification, JobSpecification>
buildCreateLibraryJobSpec(
- DataverseName dataverseName, String libraryName,
ExternalFunctionLanguage language, URI downloadURI,
+ Namespace namespace, String libraryName, ExternalFunctionLanguage
language, URI downloadURI,
String authToken, MetadataProvider metadataProvider) {
ICcApplicationContext appCtx =
metadataProvider.getApplicationContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
splitsAndConstraint = getSplitsAndConstraints(appCtx);
- JobSpecification prepareJobSpec =
createLibraryPrepareJobSpec(dataverseName, libraryName, language, downloadURI,
+ JobSpecification prepareJobSpec =
createLibraryPrepareJobSpec(namespace, libraryName, language, downloadURI,
authToken, appCtx, splitsAndConstraint);
JobSpecification commitJobSpec =
- createLibraryCommitJobSpec(dataverseName, libraryName, appCtx,
splitsAndConstraint);
+ createLibraryCommitJobSpec(namespace, libraryName, appCtx,
splitsAndConstraint);
- JobSpecification abortJobSpec =
- createLibraryAbortJobSpec(dataverseName, libraryName, appCtx,
splitsAndConstraint);
+ JobSpecification abortJobSpec = createLibraryAbortJobSpec(namespace,
libraryName, appCtx, splitsAndConstraint);
return new Triple<>(prepareJobSpec, commitJobSpec, abortJobSpec);
}
- private static JobSpecification createLibraryPrepareJobSpec(DataverseName
dataverseName, String libraryName,
+ private static JobSpecification createLibraryPrepareJobSpec(Namespace
namespace, String libraryName,
ExternalFunctionLanguage language, URI downloadURI, String
authToken, ICcApplicationContext appCtx,
Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
splitsAndConstraint) {
JobSpecification jobSpec = RuntimeUtils.createJobSpecification(appCtx);
- IOperatorDescriptor opDesc = new
LibraryDeployPrepareOperatorDescriptor(jobSpec, dataverseName, libraryName,
+ IOperatorDescriptor opDesc = new
LibraryDeployPrepareOperatorDescriptor(jobSpec, namespace, libraryName,
language, downloadURI, authToken);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec,
opDesc,
splitsAndConstraint.second);
@@ -82,32 +81,32 @@ public class ExternalLibraryJobUtils {
return jobSpec;
}
- private static JobSpecification createLibraryCommitJobSpec(DataverseName
dataverseName, String libraryName,
+ private static JobSpecification createLibraryCommitJobSpec(Namespace
namespace, String libraryName,
ICcApplicationContext appCtx, Pair<IFileSplitProvider,
AlgebricksPartitionConstraint> splitsAndConstraint) {
JobSpecification jobSpec = RuntimeUtils.createJobSpecification(appCtx);
- IOperatorDescriptor opDesc = new
LibraryDeployCommitOperatorDescriptor(jobSpec, dataverseName, libraryName);
+ IOperatorDescriptor opDesc = new
LibraryDeployCommitOperatorDescriptor(jobSpec, namespace, libraryName);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec,
opDesc,
splitsAndConstraint.second);
return jobSpec;
}
- private static JobSpecification createLibraryAbortJobSpec(DataverseName
dataverseName, String libraryName,
+ private static JobSpecification createLibraryAbortJobSpec(Namespace
namespace, String libraryName,
ICcApplicationContext appCtx, Pair<IFileSplitProvider,
AlgebricksPartitionConstraint> splitsAndConstraint) {
JobSpecification jobSpec = RuntimeUtils.createJobSpecification(appCtx);
- IOperatorDescriptor opDesc = new
LibraryDeployAbortOperatorDescriptor(jobSpec, dataverseName, libraryName);
+ IOperatorDescriptor opDesc = new
LibraryDeployAbortOperatorDescriptor(jobSpec, namespace, libraryName);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec,
opDesc,
splitsAndConstraint.second);
return jobSpec;
}
- public static JobSpecification buildDropLibraryJobSpec(DataverseName
dataverseName, String libraryName,
+ public static JobSpecification buildDropLibraryJobSpec(Namespace
namespace, String libraryName,
MetadataProvider metadataProvider) {
ICcApplicationContext appCtx =
metadataProvider.getApplicationContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
splitsAndConstraint = getSplitsAndConstraints(appCtx);
JobSpecification jobSpec = RuntimeUtils.createJobSpecification(appCtx);
- IOperatorDescriptor opDesc = new
LibraryUndeployOperatorDescriptor(jobSpec, dataverseName, libraryName);
+ IOperatorDescriptor opDesc = new
LibraryUndeployOperatorDescriptor(jobSpec, namespace, libraryName);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec,
opDesc,
splitsAndConstraint.second);
jobSpec.addRoot(opDesc);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 8071fd8a14..f058c41282 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -2220,9 +2220,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
List<Library> libraries =
MetadataManager.INSTANCE.getDataverseLibraries(mdTxnCtx,
databaseName, dataverseName);
for (Library library : libraries) {
- //TODO(DB): library database
-
jobsToExecute.add(ExternalLibraryJobUtils.buildDropLibraryJobSpec(dataverseName,
library.getName(),
- metadataProvider));
+
jobsToExecute.add(ExternalLibraryJobUtils.buildDropLibraryJobSpec(stmtDropDataverse.getNamespace(),
+ library.getName(), metadataProvider));
}
jobsToExecute.add(DataverseUtil.dropDataverseJobSpec(dv,
metadataProvider));
@@ -3577,6 +3576,7 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
MetadataTransactionContext mdTxnCtx =
MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ Namespace stmtActiveNamespace = getActiveNamespace(cls.getNamespace());
try {
Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx,
databaseName, dataverseName);
if (dv == null) {
@@ -3600,9 +3600,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
// #. prepare to create library artifacts in NC.
- //TODO(DB):
Triple<JobSpecification, JobSpecification, JobSpecification>
jobSpecs =
-
ExternalLibraryJobUtils.buildCreateLibraryJobSpec(dataverseName, libraryName,
language,
+
ExternalLibraryJobUtils.buildCreateLibraryJobSpec(stmtActiveNamespace,
libraryName, language,
cls.getLocation(), cls.getAuthToken(),
metadataProvider);
JobSpecification prepareJobSpec = jobSpecs.first;
JobSpecification commitJobSpec = jobSpecs.second;
@@ -3645,9 +3644,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
} else if (existingLibrary == null) {
// 'commit' job failed for a new library -> try removing
the library
try {
- //TODO(DB):
JobSpecification dropLibraryJobSpec =
ExternalLibraryJobUtils
- .buildDropLibraryJobSpec(dataverseName,
libraryName, metadataProvider);
+ .buildDropLibraryJobSpec(stmtActiveNamespace,
libraryName, metadataProvider);
runJob(hcc, dropLibraryJobSpec, jobFlags);
} catch (Exception e2) {
e.addSuppressed(e2);
@@ -3711,6 +3709,7 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
MetadataTransactionContext mdTxnCtx =
MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ Namespace stmtActiveNamespace =
getActiveNamespace(stmtDropLibrary.getNamespace());
try {
Dataverse dataverse =
MetadataManager.INSTANCE.getDataverse(mdTxnCtx, databaseName, dataverseName);
if (dataverse == null) {
@@ -3741,9 +3740,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
library.getLanguage(), library.getHash(),
MetadataUtil.PENDING_DROP_OP));
// #. drop library artifacts in NCs.
- //TODO(DB):
JobSpecification jobSpec =
-
ExternalLibraryJobUtils.buildDropLibraryJobSpec(dataverseName, libraryName,
metadataProvider);
+
ExternalLibraryJobUtils.buildDropLibraryJobSpec(stmtActiveNamespace,
libraryName, metadataProvider);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
@@ -5690,11 +5688,11 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
List<Dataset> datasets = addDatasetDropJob(metadataProvider,
sourceLoc, databaseName, mdTxnCtx, jobsToExecute);
// #. prepare jobs which will drop corresponding libraries
- //TODO(DB): library database
List<Library> libraries =
MetadataManager.INSTANCE.getDatabaseLibraries(mdTxnCtx, databaseName);
for (Library library : libraries) {
-
jobsToExecute.add(ExternalLibraryJobUtils.buildDropLibraryJobSpec(library.getDataverseName(),
- library.getName(), metadataProvider));
+ jobsToExecute.add(ExternalLibraryJobUtils.buildDropLibraryJobSpec(
+ new Namespace(library.getDatabaseName(),
library.getDataverseName()), library.getName(),
+ metadataProvider));
}
// #. prepare jobs which will drop the database
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
index 85014ab373..93233497dd 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/FeedOperations.java
@@ -249,9 +249,9 @@ public class FeedOperations {
(FeedIntakeOperatorDescriptor)
intakeJob.getOperatorMap().get(new OperatorDescriptorId(0));
FeedIntakeOperatorDescriptor ingestionOp;
if (firstOp.getAdaptorFactory() == null) {
- ingestionOp = new FeedIntakeOperatorDescriptor(jobSpec, feed,
firstOp.getAdaptorLibraryDataverse(),
- firstOp.getAdaptorLibraryName(),
firstOp.getAdaptorFactoryClassName(),
- firstOp.getAdapterOutputType(),
firstOp.getPolicyAccessor(),
+ ingestionOp = new FeedIntakeOperatorDescriptor(jobSpec, feed,
firstOp.getAdaptorLibraryDatabase(),
+ firstOp.getAdaptorLibraryDataverse(),
firstOp.getAdaptorLibraryName(),
+ firstOp.getAdaptorFactoryClassName(),
firstOp.getAdapterOutputType(), firstOp.getPolicyAccessor(),
firstOp.getOutputRecordDescriptors()[0]);
} else {
ingestionOp = new FeedIntakeOperatorDescriptor(jobSpec, feed,
firstOp.getAdaptorFactory(),
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java
new file mode 100644
index 0000000000..643ed802af
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudPythonTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.cloud_storage;
+
+import static
org.apache.asterix.test.runtime.ExternalPythonFunctionIT.setNcEndpoints;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.asterix.api.common.LocalCloudUtil;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.test.runtime.LangExecutionUtil;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.asterix.testframework.xml.Description;
+import org.apache.asterix.testframework.xml.TestCase;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.MethodSorters;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Run tests in cloud deployment environment
+ */
+@RunWith(Parameterized.class)
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class CloudPythonTest {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final TestCaseContext tcCtx;
+ private static final String SUITE_TESTS = "testsuite_it_python.xml";
+ private static final String ONLY_TESTS =
"testsuite_cloud_storage_only.xml";
+ private static final String CONFIG_FILE_NAME =
"src/test/resources/cc-cloud-storage.conf";
+ private static final String DELTA_RESULT_PATH = "results_cloud";
+ private static final String EXCLUDED_TESTS = "MP";
+
+ public CloudPythonTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ LocalCloudUtil.startS3CloudEnvironment(true);
+ TestExecutor testExecutor = new TestExecutor(DELTA_RESULT_PATH);
+ testExecutor.executorId = "cloud";
+ testExecutor.stripSubstring = "//DB:";
+ LangExecutionUtil.setUp(CONFIG_FILE_NAME, testExecutor);
+ setNcEndpoints(testExecutor);
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY,
CONFIG_FILE_NAME);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ }
+
+ @Parameters(name = "CloudPythonTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
+ }
+
+ @Test
+ public void test() throws Exception {
+ List<TestCase.CompilationUnit> cu =
tcCtx.getTestCase().getCompilationUnit();
+ Assume.assumeTrue(cu.size() > 1 ||
!EXCLUDED_TESTS.equals(getText(cu.get(0).getDescription())));
+ LangExecutionUtil.test(tcCtx);
+ }
+
+ private static String getText(Description description) {
+ return description == null ? "" : description.getValue();
+ }
+}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java
index d72d494d15..2674f41b99 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExternalPythonFunctionIT.java
@@ -72,7 +72,7 @@ public class ExternalPythonFunctionIT {
LangExecutionUtil.test(tcCtx);
}
- private static void setNcEndpoints(TestExecutor testExecutor) {
+ public static void setNcEndpoints(TestExecutor testExecutor) {
final NodeControllerService[] ncs =
ExecutionTestUtil.integrationUtil.ncs;
final Map<String, InetSocketAddress> ncEndPoints = new HashMap<>();
final String ip = InetAddress.getLoopbackAddress().getHostAddress();
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_api.5.regexjson
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_api.5.regexjson
index f0cb7cf4dd..de8723176d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_api.5.regexjson
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/library_list_api_multipart/library_list_api.5.regexjson
@@ -1,5 +1,5 @@
[{
- "dataverse": "external/lib/test",
+ "dataverse": "external.lib.test",
"hash_md5": "R{[a-zA-Z0-9-]+}",
"name": "testlib"
},
@@ -9,12 +9,12 @@
"name": "testlib"
},
{
- "dataverse": "externallibtest/foo",
+ "dataverse": "externallibtest.foo",
"hash_md5": "R{[a-zA-Z0-9-]+}",
"name": "testlib"
},
{
- "dataverse": "externallibtest/foo/bar",
+ "dataverse": "externallibtest.foo.bar",
"hash_md5": "R{[a-zA-Z0-9-]+}",
"name": "testlib"
}]
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
index 284c2fd242..c6a7cf5a82 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
@@ -30,6 +30,7 @@
</test-case>
<test-case FilePath="external-library">
<compilation-unit name="mysentiment_multipart">
+ <description>MP</description>
<output-dir compare="Text">mysentiment</output-dir>
</compilation-unit>
</test-case>
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
index 6f33d6f29a..b5fd3a64f3 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INamespacePathResolver.java
@@ -27,4 +27,6 @@ public interface INamespacePathResolver {
String resolve(String databaseName, DataverseName dataverseName);
+ boolean usingDatabase();
+
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
index acbce6d4d6..2098060ec2 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/library/ILibraryManager.java
@@ -26,7 +26,7 @@ import java.security.MessageDigest;
import java.util.List;
import java.util.function.Function;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.ipc.ExternalFunctionResultRouter;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -38,22 +38,24 @@ import org.apache.hyracks.ipc.impl.IPCSystem;
public interface ILibraryManager {
- List<Pair<DataverseName, String>> getLibraryListing() throws IOException;
+ List<Pair<Namespace, String>> getLibraryListing() throws IOException;
- String getLibraryHash(DataverseName dataverseName, String libraryName)
throws IOException;
+ String getLibraryHash(Namespace namespace, String libraryName) throws
IOException;
- ILibrary getLibrary(DataverseName dataverseName, String libraryName)
throws HyracksDataException;
+ ILibrary getLibrary(Namespace namespace, String libraryName) throws
HyracksDataException;
- void closeLibrary(DataverseName dataverseName, String libraryName) throws
HyracksDataException;
+ void closeLibrary(Namespace namespace, String libraryName) throws
HyracksDataException;
// deployment helpers
FileReference getStorageDir();
- FileReference getLibraryDir(DataverseName dataverseName, String
libraryName) throws HyracksDataException;
+ FileReference getLibraryDir(Namespace namespace, String libraryName)
throws HyracksDataException;
FileReference getDistributionDir();
+ String getNsOrDv(Namespace ns);
+
void dropLibraryPath(FileReference fileRef) throws HyracksDataException;
byte[] serializeLibraryDescriptor(LibraryDescriptor libraryDescriptor)
throws HyracksDataException;
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
index 5b75c797ca..1a14be4742 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/Namespace.java
@@ -22,7 +22,7 @@ package org.apache.asterix.common.metadata;
import java.io.Serializable;
import java.util.Objects;
-public final class Namespace implements Serializable {
+public final class Namespace implements Serializable, Comparable<Namespace> {
private static final long serialVersionUID = 1L;
@@ -61,4 +61,12 @@ public final class Namespace implements Serializable {
return Objects.equals(databaseName, that.databaseName) &&
Objects.equals(dataverseName, that.dataverseName);
}
+ @Override
+ public int compareTo(Namespace that) {
+ int dbComp = databaseName.compareTo(that.getDatabaseName());
+ if (dbComp == 0) {
+ return dataverseName.compareTo(that.getDataverseName());
+ }
+ return dbComp;
+ }
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
index 910c3ef68f..fa1c0b5837 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/NamespacePathResolver.java
@@ -57,4 +57,9 @@ public class NamespacePathResolver implements
INamespacePathResolver {
return StoragePathUtil.prepareDataverseName(dataverseName);
}
}
+
+ @Override
+ public boolean usingDatabase() {
+ return usingDatabase;
+ }
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java
index 21735436dc..ac34471d63 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/ExternalAdapterFactory.java
@@ -29,6 +29,7 @@ import
org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.api.IExternalDataSourceFactory;
import org.apache.asterix.external.api.ITypedAdapterFactory;
import
org.apache.asterix.external.input.filter.NoOpExternalFilterEvaluatorFactory;
@@ -44,13 +45,14 @@ import org.apache.hyracks.api.exceptions.IWarningCollector;
public final class ExternalAdapterFactory implements ITypedAdapterFactory {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
private final DataverseName libraryDataverse;
private final String libraryName;
private final String className;
+ private final String libraryDatabase;
private ARecordType outputType;
@@ -60,7 +62,9 @@ public final class ExternalAdapterFactory implements
ITypedAdapterFactory {
private transient ICCServiceContext serviceContext;
- public ExternalAdapterFactory(DataverseName libraryDataverse, String
libraryName, String className) {
+ public ExternalAdapterFactory(String libraryDatabase, DataverseName
libraryDataverse, String libraryName,
+ String className) {
+ this.libraryDatabase = libraryDatabase;
this.libraryDataverse = libraryDataverse;
this.libraryName = libraryName;
this.className = className;
@@ -85,7 +89,7 @@ public final class ExternalAdapterFactory implements
ITypedAdapterFactory {
INCServiceContext serviceCtx =
ctx.getJobletContext().getServiceContext();
INcApplicationContext appCtx = (INcApplicationContext)
serviceCtx.getApplicationContext();
ILibraryManager libraryManager = appCtx.getLibraryManager();
- ILibrary library = libraryManager.getLibrary(libraryDataverse,
libraryName);
+ ILibrary library = libraryManager.getLibrary(new
Namespace(libraryDatabase, libraryDataverse), libraryName);
if (ExternalFunctionLanguage.JAVA != library.getLanguage()) {
throw new HyracksDataException("Unexpected library language: " +
library.getLanguage());
}
@@ -127,6 +131,10 @@ public final class ExternalAdapterFactory implements
ITypedAdapterFactory {
return "external:" + className;
}
+ public String getLibraryDatabase() {
+ return libraryDatabase;
+ }
+
public DataverseName getLibraryDataverse() {
return libraryDataverse;
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
index 59b1a02cb7..47a17680b9 100755
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalLibraryManager.java
@@ -64,6 +64,8 @@ import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.library.LibraryDescriptor;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.MetadataConstants;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.external.ipc.ExternalFunctionResultRouter;
import org.apache.commons.codec.digest.DigestUtils;
@@ -139,7 +141,7 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
private final FileReference distDir;
private final Path trashDirPath;
//TODO(DB): change for database
- private final Map<Pair<DataverseName, String>, ILibrary> libraries = new
HashMap<>();
+ private final Map<Pair<Namespace, String>, ILibrary> libraries = new
HashMap<>();
private IPCSystem pythonIPC;
private final ExternalFunctionResultRouter router;
private final IIOManager ioManager;
@@ -219,7 +221,7 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
@Override
public void stop(boolean dumpState, OutputStream ouputStream) {
synchronized (this) {
- for (Map.Entry<Pair<DataverseName, String>, ILibrary> p :
libraries.entrySet()) {
+ for (Map.Entry<Pair<Namespace, String>, ILibrary> p :
libraries.entrySet()) {
ILibrary library = p.getValue();
try {
library.close();
@@ -235,13 +237,13 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
return storageDir;
}
- private FileReference getDataverseDir(DataverseName dataverseName) throws
HyracksDataException {
- return getChildFileRef(storageDir,
StoragePathUtil.prepareDataverseName(dataverseName));
+ private FileReference getDataverseDir(Namespace namespace) throws
HyracksDataException {
+ return getChildFileRef(storageDir,
namespacePathResolver.resolve(namespace));
}
@Override
- public FileReference getLibraryDir(DataverseName dataverseName, String
libraryName) throws HyracksDataException {
- FileReference dataverseDir = getDataverseDir(dataverseName);
+ public FileReference getLibraryDir(Namespace namespace, String
libraryName) throws HyracksDataException {
+ FileReference dataverseDir = getDataverseDir(namespace);
return getChildFileRef(dataverseDir, libraryName);
}
@@ -251,8 +253,8 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
}
@Override
- public List<Pair<DataverseName, String>> getLibraryListing() throws
IOException {
- List<Pair<DataverseName, String>> libs = new ArrayList<>();
+ public List<Pair<Namespace, String>> getLibraryListing() throws
IOException {
+ List<Pair<Namespace, String>> libs = new ArrayList<>();
Files.walkFileTree(storageDirPath, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path currPath,
BasicFileAttributes attrs) {
@@ -277,30 +279,41 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
//? shouldn't happen
return FileVisitResult.TERMINATE;
}
- //add first part, then all multiparts
- dvParts.add(tokens[0]);
- int currToken = 1;
- for (; currToken < tokens.length && tokens[currToken]
- .codePointAt(0) ==
StoragePathUtil.DATAVERSE_CONTINUATION_MARKER; currToken++) {
- dvParts.add(tokens[currToken].substring(1));
- }
- //we should only arrive at foo/^bar/^baz/.../^bat/lib
- //anything else is fishy or empty
- if (currToken != tokens.length - 1) {
- return FileVisitResult.SKIP_SUBTREE;
- }
- String candidateLib = tokens[currToken];
- DataverseName candidateDv;
try {
+ String candidateDb = MetadataConstants.DEFAULT_DATABASE;
+ if (namespacePathResolver.usingDatabase()) {
+ if (tokens.length < 3) {
+ return FileVisitResult.TERMINATE;
+ }
+ libs.add(new Pair<>(new Namespace(candidateDb,
DataverseName.create(List.of(tokens[1]))),
+ tokens[3]));
+ return FileVisitResult.SKIP_SUBTREE;
+
+ }
+ //add first part, then look for multiparts
+ dvParts.add(tokens[0]);
+ int currToken = 1;
+ for (; currToken < tokens.length && tokens[currToken]
+ .codePointAt(0) ==
StoragePathUtil.DATAVERSE_CONTINUATION_MARKER; currToken++) {
+ dvParts.add(tokens[currToken].substring(1));
+ }
+ //we should only arrive at foo/^bar/^baz/.../^bat/lib
+ //anything else is fishy or empty
+ if (currToken != tokens.length - 1) {
+ return FileVisitResult.SKIP_SUBTREE;
+ }
+ String candidateLib = tokens[currToken];
+ DataverseName candidateDv;
candidateDv = DataverseName.create(dvParts);
+ Namespace candidateNs = new Namespace(candidateDb,
candidateDv);
+ FileReference candidateLibPath =
findLibraryRevDir(candidateNs, candidateLib);
+ if (candidateLibPath != null) {
+ libs.add(new Pair<>(candidateNs, candidateLib));
+ }
} catch (AsterixException e) {
// shouldn't happen
throw HyracksDataException.create(e);
}
- FileReference candidateLibPath =
findLibraryRevDir(candidateDv, candidateLib);
- if (candidateLibPath != null) {
- libs.add(new Pair<>(candidateDv, candidateLib));
- }
return FileVisitResult.SKIP_SUBTREE;
}
});
@@ -308,8 +321,8 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
}
@Override
- public String getLibraryHash(DataverseName dataverseName, String
libraryName) throws IOException {
- FileReference revDir = findLibraryRevDir(dataverseName, libraryName);
+ public String getLibraryHash(Namespace namespace, String libraryName)
throws IOException {
+ FileReference revDir = findLibraryRevDir(namespace, libraryName);
if (revDir == null) {
throw HyracksDataException
.create(AsterixException.create(ErrorCode.EXTERNAL_UDF_EXCEPTION, "Library does
not exist"));
@@ -319,26 +332,26 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
}
@Override
- public ILibrary getLibrary(DataverseName dataverseName, String
libraryName) throws HyracksDataException {
- Pair<DataverseName, String> key = getKey(dataverseName, libraryName);
+ public ILibrary getLibrary(Namespace namespace, String libraryName) throws
HyracksDataException {
+ Pair<Namespace, String> key = getKey(namespace, libraryName);
synchronized (this) {
ILibrary library = libraries.get(key);
if (library == null) {
- library = loadLibrary(dataverseName, libraryName);
+ library = loadLibrary(namespace, libraryName);
libraries.put(key, library);
}
return library;
}
}
- private ILibrary loadLibrary(DataverseName dataverseName, String
libraryName) throws HyracksDataException {
- FileReference libRevDir = findLibraryRevDir(dataverseName,
libraryName);
+ private ILibrary loadLibrary(Namespace namespace, String libraryName)
throws HyracksDataException {
+ FileReference libRevDir = findLibraryRevDir(namespace, libraryName);
if (libRevDir == null) {
- throw new HyracksDataException("Cannot find library: " +
dataverseName + '.' + libraryName);
+ throw new HyracksDataException("Cannot find library: " + namespace
+ '.' + libraryName);
}
FileReference libContentsDir = libRevDir.getChild(CONTENTS_DIR_NAME);
if (!libContentsDir.getFile().isDirectory()) {
- throw new HyracksDataException("Cannot find library: " +
dataverseName + '.' + libraryName);
+ throw new HyracksDataException("Cannot find library: " + namespace
+ '.' + libraryName);
}
try {
ExternalFunctionLanguage libLang =
getLibraryDescriptor(libRevDir).getLanguage();
@@ -351,7 +364,7 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
throw new HyracksDataException("Invalid language: " +
libraryName);
}
} catch (IOException e) {
- LOGGER.error("Failed to initialize library " + dataverseName + '.'
+ libraryName, e);
+ LOGGER.error("Failed to initialize library " + namespace + '.' +
libraryName, e);
throw HyracksDataException.create(e);
}
}
@@ -377,9 +390,8 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
}
- private FileReference findLibraryRevDir(DataverseName dataverseName,
String libraryName)
- throws HyracksDataException {
- FileReference libraryBaseDir = getLibraryDir(dataverseName,
libraryName);
+ private FileReference findLibraryRevDir(Namespace namespace, String
libraryName) throws HyracksDataException {
+ FileReference libraryBaseDir = getLibraryDir(namespace, libraryName);
if (!libraryBaseDir.getFile().isDirectory()) {
return null;
}
@@ -395,8 +407,8 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
}
@Override
- public void closeLibrary(DataverseName dataverseName, String libraryName)
throws HyracksDataException {
- Pair<DataverseName, String> key = getKey(dataverseName, libraryName);
+ public void closeLibrary(Namespace namespace, String libraryName) throws
HyracksDataException {
+ Pair<Namespace, String> key = getKey(namespace, libraryName);
ILibrary library;
synchronized (this) {
library = libraries.remove(key);
@@ -410,8 +422,16 @@ public class ExternalLibraryManager implements
ILibraryManager, ILifeCycleCompon
public void dumpState(OutputStream os) {
}
- private static Pair<DataverseName, String> getKey(DataverseName
dataverseName, String libraryName) {
- return new Pair<>(dataverseName, libraryName);
+ private static Pair<Namespace, String> getKey(Namespace namespace, String
libraryName) {
+ return new Pair<>(namespace, libraryName);
+ }
+
+ @Override
+ public String getNsOrDv(Namespace ns) {
+ if (namespacePathResolver.usingDatabase()) {
+ return ns.toString();
+ }
+ return ns.getDataverseName().toString();
}
public Path zipAllLibs() throws IOException {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
index 4aa5f421a7..d190f5e1dc 100755
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
@@ -23,7 +23,7 @@ import static
org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.api.IExternalScalarFunction;
import org.apache.asterix.external.api.IFunctionFactory;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
@@ -49,9 +49,9 @@ class ExternalScalarJavaFunctionEvaluator extends
ExternalScalarFunctionEvaluato
IEvaluatorContext context) throws HyracksDataException {
super(finfo, args, argTypes, context);
- DataverseName libraryDataverseName = finfo.getLibraryDataverseName();
+ Namespace libraryNamespace = finfo.getLibraryNamespace();
String libraryName = finfo.getLibraryName();
- JavaLibrary library = (JavaLibrary)
libraryManager.getLibrary(libraryDataverseName, libraryName);
+ JavaLibrary library = (JavaLibrary)
libraryManager.getLibrary(libraryNamespace, libraryName);
String classname = finfo.getExternalIdentifier().get(0);
try {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java
index 056aa9a014..32186338d9 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryDomainSocketEvaluator.java
@@ -62,8 +62,7 @@ public class PythonLibraryDomainSocketEvaluator extends
AbstractLibrarySocketEva
public void start() throws IOException, AsterixException {
PythonLibraryEvaluatorId fnId = (PythonLibraryEvaluatorId) id;
- PythonLibrary library =
- (PythonLibrary)
libMgr.getLibrary(fnId.getLibraryDataverseName(), fnId.getLibraryName());
+ PythonLibrary library = (PythonLibrary)
libMgr.getLibrary(fnId.getLibraryNamespace(), fnId.getLibraryName());
String wd = library.getFile().getAbsolutePath();
MethodHandles.Lookup lookup = MethodHandles.lookup();
SocketAddress sockAddr;
@@ -107,7 +106,7 @@ public class PythonLibraryDomainSocketEvaluator extends
AbstractLibrarySocketEva
static PythonLibraryDomainSocketEvaluator
getInstance(IExternalFunctionInfo finfo, ILibraryManager libMgr,
IHyracksTaskContext ctx, IWarningCollector warningCollector,
SourceLocation sourceLoc)
throws IOException, AsterixException {
- PythonLibraryEvaluatorId evaluatorId = new
PythonLibraryEvaluatorId(finfo.getLibraryDataverseName(),
+ PythonLibraryEvaluatorId evaluatorId = new
PythonLibraryEvaluatorId(finfo.getLibraryNamespace(),
finfo.getLibraryName(), Thread.currentThread());
PythonLibraryDomainSocketEvaluator evaluator =
(PythonLibraryDomainSocketEvaluator)
ctx.getStateObject(evaluatorId);
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorId.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorId.java
index c2f6f00518..4a6a5e5964 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorId.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryEvaluatorId.java
@@ -20,18 +20,18 @@ package org.apache.asterix.external.library;
import java.util.Objects;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
final class PythonLibraryEvaluatorId {
- private final DataverseName libraryDataverseName;
+ private final Namespace libraryNamespace;
private final String libraryName;
private final Thread thread;
- PythonLibraryEvaluatorId(DataverseName libraryDataverseName, String
libraryName, Thread thread) {
- this.libraryDataverseName =
Objects.requireNonNull(libraryDataverseName);
+ PythonLibraryEvaluatorId(Namespace libraryNamespace, String libraryName,
Thread thread) {
+ this.libraryNamespace = Objects.requireNonNull(libraryNamespace);
this.libraryName = Objects.requireNonNull(libraryName);
this.thread = Objects.requireNonNull(thread);
}
@@ -43,17 +43,17 @@ final class PythonLibraryEvaluatorId {
if (o == null || getClass() != o.getClass())
return false;
PythonLibraryEvaluatorId that = (PythonLibraryEvaluatorId) o;
- return libraryDataverseName.equals(that.libraryDataverseName) &&
libraryName.equals(that.libraryName)
+ return libraryNamespace.equals(that.libraryNamespace) &&
libraryName.equals(that.libraryName)
&& thread.equals(that.thread);
}
@Override
public int hashCode() {
- return Objects.hash(libraryDataverseName, libraryName);
+ return Objects.hash(libraryNamespace, libraryName);
}
- public DataverseName getLibraryDataverseName() {
- return libraryDataverseName;
+ public Namespace getLibraryNamespace() {
+ return libraryNamespace;
}
public String getLibraryName() {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java
index 385d738f83..4406cc89eb 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/PythonLibraryTCPSocketEvaluator.java
@@ -69,8 +69,7 @@ public class PythonLibraryTCPSocketEvaluator extends
AbstractLibrarySocketEvalua
@Override
public void start() throws IOException, AsterixException {
PythonLibraryEvaluatorId fnId = (PythonLibraryEvaluatorId) id;
- PythonLibrary library =
- (PythonLibrary)
libMgr.getLibrary(fnId.getLibraryDataverseName(), fnId.getLibraryName());
+ PythonLibrary library = (PythonLibrary)
libMgr.getLibrary(fnId.getLibraryNamespace(), fnId.getLibraryName());
String wd = library.getFile().getAbsolutePath();
int port = ipcSys.getSocketAddress().getPort();
List<String> args = new ArrayList<>();
@@ -110,7 +109,7 @@ public class PythonLibraryTCPSocketEvaluator extends
AbstractLibrarySocketEvalua
ExternalFunctionResultRouter router, IPCSystem ipcSys, File
pythonHome, IHyracksTaskContext ctx,
String sitePkgs, List<String> pythonArgs, Map<String, String>
pythonEnv, IWarningCollector warningCollector,
SourceLocation sourceLoc) throws IOException, AsterixException {
- PythonLibraryEvaluatorId evaluatorId = new
PythonLibraryEvaluatorId(finfo.getLibraryDataverseName(),
+ PythonLibraryEvaluatorId evaluatorId = new
PythonLibraryEvaluatorId(finfo.getLibraryNamespace(),
finfo.getLibraryName(), Thread.currentThread());
PythonLibraryTCPSocketEvaluator evaluator =
(PythonLibraryTCPSocketEvaluator) ctx.getStateObject(evaluatorId);
if (evaluator == null) {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractLibraryOperatorDescriptor.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractLibraryOperatorDescriptor.java
index ec05dfe31a..b25483657f 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractLibraryOperatorDescriptor.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractLibraryOperatorDescriptor.java
@@ -27,7 +27,7 @@ import java.nio.file.StandardCopyOption;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.library.ILibraryManager;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -44,14 +44,14 @@ abstract class AbstractLibraryOperatorDescriptor extends
AbstractSingleActivityO
private static final long serialVersionUID = 1L;
- protected final DataverseName dataverseName;
+ protected final Namespace namespace;
protected final String libraryName;
- public AbstractLibraryOperatorDescriptor(IOperatorDescriptorRegistry spec,
DataverseName dataverseName,
+ public AbstractLibraryOperatorDescriptor(IOperatorDescriptorRegistry spec,
Namespace namespace,
String libraryName) {
super(spec, 0, 0);
- this.dataverseName = dataverseName;
+ this.namespace = namespace;
this.libraryName = libraryName;
}
@@ -77,7 +77,7 @@ abstract class AbstractLibraryOperatorDescriptor extends
AbstractSingleActivityO
(INcApplicationContext)
ctx.getJobletContext().getServiceContext().getApplicationContext();
ioManager = runtimeCtx.getIoManager();
libraryManager = runtimeCtx.getLibraryManager();
- libraryDir = libraryManager.getLibraryDir(dataverseName,
libraryName);
+ libraryDir = libraryManager.getLibraryDir(namespace, libraryName);
try {
execute();
} catch (IOException e) {
@@ -130,7 +130,7 @@ abstract class AbstractLibraryOperatorDescriptor extends
AbstractSingleActivityO
}
protected void closeLibrary() throws HyracksDataException {
- libraryManager.closeLibrary(dataverseName, libraryName);
+ libraryManager.closeLibrary(namespace, libraryName);
}
@Override
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
index dbc571496c..6a80750a82 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorDescriptor.java
@@ -28,6 +28,7 @@ import
org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.api.ITypedAdapterFactory;
import org.apache.asterix.external.feed.api.IFeed;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
@@ -63,6 +64,7 @@ public class FeedIntakeOperatorDescriptor extends
AbstractSingleActivityOperator
private final FeedPolicyAccessor policyAccessor;
private final ARecordType adapterOutputType;
+ private String adaptorLibraryDatabase;
/**
* The adaptor factory that is used to create an instance of the feed
adaptor
**/
@@ -93,13 +95,14 @@ public class FeedIntakeOperatorDescriptor extends
AbstractSingleActivityOperator
this.outRecDescs[0] = rDesc;
}
- public FeedIntakeOperatorDescriptor(JobSpecification spec, IFeed feed,
DataverseName adapterLibraryDataverse,
- String adapterLibraryName, String adapterFactoryClassName,
ARecordType adapterOutputType,
- FeedPolicyAccessor policyAccessor, RecordDescriptor rDesc) {
+ public FeedIntakeOperatorDescriptor(JobSpecification spec, IFeed feed,
String databaseName,
+ DataverseName adapterLibraryDataverse, String adapterLibraryName,
String adapterFactoryClassName,
+ ARecordType adapterOutputType, FeedPolicyAccessor policyAccessor,
RecordDescriptor rDesc) {
super(spec, 0, 1);
this.feedId =
new EntityId(FEED_EXTENSION_NAME, feed.getDatabaseName(),
feed.getDataverseName(), feed.getFeedName());
this.adaptorFactoryClassName = adapterFactoryClassName;
+ this.adaptorLibraryDatabase = databaseName;
this.adaptorLibraryDataverse = adapterLibraryDataverse;
this.adaptorLibraryName = adapterLibraryName;
this.adaptorConfiguration = feed.getConfiguration();
@@ -122,7 +125,8 @@ public class FeedIntakeOperatorDescriptor extends
AbstractSingleActivityOperator
INcApplicationContext runtimeCtx =
(INcApplicationContext)
ctx.getJobletContext().getServiceContext().getApplicationContext();
ILibraryManager libraryManager = runtimeCtx.getLibraryManager();
- ILibrary lib = libraryManager.getLibrary(adaptorLibraryDataverse,
adaptorLibraryName);
+ ILibrary lib = libraryManager.getLibrary(new
Namespace(adaptorLibraryDatabase, adaptorLibraryDataverse),
+ adaptorLibraryName);
if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
throw new HyracksDataException("Unexpected library language: " +
lib.getLanguage());
}
@@ -166,6 +170,10 @@ public class FeedIntakeOperatorDescriptor extends
AbstractSingleActivityOperator
return this.policyAccessor;
}
+ public String getAdaptorLibraryDatabase() {
+ return adaptorLibraryDatabase;
+ }
+
public DataverseName getAdaptorLibraryDataverse() {
return adaptorLibraryDataverse;
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployAbortOperatorDescriptor.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployAbortOperatorDescriptor.java
index 97eb3e8ea4..4fc8788208 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployAbortOperatorDescriptor.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployAbortOperatorDescriptor.java
@@ -21,7 +21,7 @@ package org.apache.asterix.external.operators;
import java.io.IOException;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
@@ -36,9 +36,9 @@ public final class LibraryDeployAbortOperatorDescriptor
extends AbstractLibraryO
private static final Logger LOGGER =
LogManager.getLogger(LibraryDeployAbortOperatorDescriptor.class);
- public LibraryDeployAbortOperatorDescriptor(IOperatorDescriptorRegistry
spec, DataverseName dataverseName,
+ public LibraryDeployAbortOperatorDescriptor(IOperatorDescriptorRegistry
spec, Namespace namespace,
String libraryName) {
- super(spec, dataverseName, libraryName);
+ super(spec, namespace, libraryName);
}
@Override
@@ -48,7 +48,7 @@ public final class LibraryDeployAbortOperatorDescriptor
extends AbstractLibraryO
@Override
protected void execute() throws IOException {
if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Abort deployment of library {}.{}",
dataverseName, libraryName);
+ LOGGER.info("Abort deployment of library {}.{}",
namespace, libraryName);
}
FileReference libDir = getLibraryDir();
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployCommitOperatorDescriptor.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployCommitOperatorDescriptor.java
index 6cc3df7f7b..c7ca8e46af 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployCommitOperatorDescriptor.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployCommitOperatorDescriptor.java
@@ -21,7 +21,7 @@ package org.apache.asterix.external.operators;
import java.io.IOException;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
@@ -36,9 +36,9 @@ public final class LibraryDeployCommitOperatorDescriptor
extends AbstractLibrary
private static final Logger LOGGER =
LogManager.getLogger(LibraryDeployCommitOperatorDescriptor.class);
- public LibraryDeployCommitOperatorDescriptor(IOperatorDescriptorRegistry
spec, DataverseName dataverseName,
+ public LibraryDeployCommitOperatorDescriptor(IOperatorDescriptorRegistry
spec, Namespace namespace,
String libraryName) {
- super(spec, dataverseName, libraryName);
+ super(spec, namespace, libraryName);
}
@Override
@@ -48,7 +48,7 @@ public final class LibraryDeployCommitOperatorDescriptor
extends AbstractLibrary
@Override
protected void execute() throws IOException {
if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Commit deployment of library {}.{}",
dataverseName, libraryName);
+ LOGGER.info("Commit deployment of library {}.{}",
namespace, libraryName);
}
// #. rename 'stage' dir into 'rev_1' dir
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java
index c12bb58cc7..0c12d4338b 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryDeployPrepareOperatorDescriptor.java
@@ -33,7 +33,7 @@ import java.security.MessageDigest;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.library.LibraryDescriptor;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.external.util.ExternalLibraryUtils;
import org.apache.commons.io.FilenameUtils;
@@ -56,9 +56,9 @@ public class LibraryDeployPrepareOperatorDescriptor extends
AbstractLibraryOpera
private final URI libLocation;
private final String authToken;
- public LibraryDeployPrepareOperatorDescriptor(IOperatorDescriptorRegistry
spec, DataverseName dataverseName,
+ public LibraryDeployPrepareOperatorDescriptor(IOperatorDescriptorRegistry
spec, Namespace namespace,
String libraryName, ExternalFunctionLanguage language, URI
libLocation, String authToken) {
- super(spec, dataverseName, libraryName);
+ super(spec, namespace, libraryName);
this.language = language;
this.libLocation = libLocation;
this.authToken = authToken;
@@ -74,7 +74,7 @@ public class LibraryDeployPrepareOperatorDescriptor extends
AbstractLibraryOpera
@Override
protected void execute() throws IOException {
if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Prepare deployment of library {}.{}",
dataverseName, libraryName);
+ LOGGER.info("Prepare deployment of library {}.{}",
namespace, libraryName);
}
// #. create library dir if necessary, clean 'stage' dir
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryUndeployOperatorDescriptor.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryUndeployOperatorDescriptor.java
index a146ee30e4..04c9c95d8e 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryUndeployOperatorDescriptor.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/LibraryUndeployOperatorDescriptor.java
@@ -21,7 +21,7 @@ package org.apache.asterix.external.operators;
import java.io.IOException;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
@@ -37,9 +37,9 @@ public final class LibraryUndeployOperatorDescriptor extends
AbstractLibraryOper
private static final Logger LOGGER =
LogManager.getLogger(LibraryUndeployOperatorDescriptor.class);
- public LibraryUndeployOperatorDescriptor(IOperatorDescriptorRegistry spec,
DataverseName dataverseName,
+ public LibraryUndeployOperatorDescriptor(IOperatorDescriptorRegistry spec,
Namespace namespace,
String libraryName) {
- super(spec, dataverseName, libraryName);
+ super(spec, namespace, libraryName);
}
@Override
@@ -49,7 +49,7 @@ public final class LibraryUndeployOperatorDescriptor extends
AbstractLibraryOper
@Override
protected void execute() throws IOException {
if (LOGGER.isInfoEnabled()) {
- LOGGER.info("Undeploying library {}.{}", dataverseName,
libraryName);
+ LOGGER.info("Undeploying library {}.{}", namespace,
libraryName);
}
try {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 439e8529d8..189e0d8e78 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.external.util;
+import static
org.apache.asterix.common.metadata.MetadataConstants.DEFAULT_DATABASE;
import static
org.apache.asterix.external.util.ExternalDataConstants.DEFINITION_FIELD_NAME;
import static
org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
import static
org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE;
@@ -62,6 +63,7 @@ import
org.apache.asterix.common.functions.ExternalFunctionLanguage;
import org.apache.asterix.common.library.ILibrary;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.external.api.IDataParserFactory;
import
org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType;
import org.apache.asterix.external.api.IInputStreamFactory;
@@ -194,11 +196,11 @@ public class ExternalDataUtils {
}
public static IInputStreamFactory
createExternalInputStreamFactory(ILibraryManager libraryManager,
- DataverseName dataverse, String stream) throws
HyracksDataException {
+ Namespace namespace, String stream) throws HyracksDataException {
try {
String libraryName = getLibraryName(stream);
String className = getExternalClassName(stream);
- ILibrary lib = libraryManager.getLibrary(dataverse, libraryName);
+ ILibrary lib = libraryManager.getLibrary(namespace, libraryName);
if (lib.getLanguage() != ExternalFunctionLanguage.JAVA) {
throw new HyracksDataException("Unexpected library language: "
+ lib.getLanguage());
}
@@ -287,7 +289,7 @@ public class ExternalDataUtils {
String libraryName = dataverseAndLibrary[1];
ILibrary lib;
try {
- lib = libraryManager.getLibrary(dataverseName, libraryName);
+ lib = libraryManager.getLibrary(new Namespace(DEFAULT_DATABASE,
dataverseName), libraryName);
} catch (HyracksDataException e) {
throw new AsterixException("Cannot load library", e);
}
@@ -310,7 +312,7 @@ public class ExternalDataUtils {
parserFactoryName.indexOf(ExternalDataConstants.EXTERNAL_LIBRARY_SEPARATOR));
ILibrary lib;
try {
- lib = libraryManager.getLibrary(dataverse, library);
+ lib = libraryManager.getLibrary(new
Namespace(DEFAULT_DATABASE, dataverse), library);
} catch (HyracksDataException e) {
throw new AsterixException("Cannot load library", e);
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java
index 9ab07eb03e..57fc90d9b7 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalLibraryUtils.java
@@ -26,7 +26,7 @@ import java.util.Map;
import java.util.TreeMap;
import org.apache.asterix.common.library.ILibraryManager;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.util.bytes.HexPrinter;
@@ -42,11 +42,11 @@ public class ExternalLibraryUtils {
return hashBuilder.toString();
}
- public static Map<DataverseName, Map<String, String>>
produceLibraryListing(ILibraryManager libraryManager)
+ public static Map<Namespace, Map<String, String>>
produceLibraryListing(ILibraryManager libraryManager)
throws IOException {
- List<Pair<DataverseName, String>> libs =
libraryManager.getLibraryListing();
- Map<DataverseName, Map<String, String>> dvToLibHashes = new
TreeMap<>();
- for (Pair<DataverseName, String> lib : libs) {
+ List<Pair<Namespace, String>> libs =
libraryManager.getLibraryListing();
+ Map<Namespace, Map<String, String>> dvToLibHashes = new TreeMap<>();
+ for (Pair<Namespace, String> lib : libs) {
dvToLibHashes.computeIfAbsent(lib.first, h -> new
TreeMap<>()).put(lib.getSecond(),
libraryManager.getLibraryHash(lib.first, lib.second));
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index b310271270..0aa5f460c1 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -578,9 +578,9 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
break;
case EXTERNAL:
ExternalAdapterFactory extAdapterFactory =
(ExternalAdapterFactory) adapterFactory;
- feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, feed,
extAdapterFactory.getLibraryDataverse(),
- extAdapterFactory.getLibraryName(),
extAdapterFactory.getClassName(), recordType,
- policyAccessor, factoryOutput.second);
+ feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, feed,
extAdapterFactory.getLibraryDatabase(),
+ extAdapterFactory.getLibraryDataverse(),
extAdapterFactory.getLibraryName(),
+ extAdapterFactory.getClassName(), recordType,
policyAccessor, factoryOutput.second);
break;
default:
break;
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
index 63a252ee29..534a692986 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
@@ -84,12 +84,12 @@ public class LibraryTupleTranslator extends
AbstractTupleTranslator<Library> {
String dataverseCanonicalName =
library.getDataverseName().getCanonicalForm();
// write the key in the first 2 fields of the tuple
+ tupleBuilder.reset();
if (libraryEntity.databaseNameIndex() >= 0) {
aString.setValue(library.getDatabaseName());
stringSerde.serialize(aString, tupleBuilder.getDataOutput());
tupleBuilder.addFieldEndOffset();
}
- tupleBuilder.reset();
aString.setValue(dataverseCanonicalName);
stringSerde.serialize(aString, tupleBuilder.getDataOutput());
tupleBuilder.addFieldEndOffset();
@@ -106,6 +106,7 @@ public class LibraryTupleTranslator extends
AbstractTupleTranslator<Library> {
stringSerde.serialize(aString, fieldValue.getDataOutput());
recordBuilder.addField(libraryEntity.databaseNameIndex(),
fieldValue);
}
+
// write field 0
fieldValue.reset();
aString.setValue(dataverseCanonicalName);
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index ca50fcdae2..4750591870 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -188,7 +188,7 @@ public class FeedMetadataUtil {
if
(!ExternalFunctionLanguage.JAVA.name().equals(library.getLanguage())) {
throw new HyracksDataException("Unexpected library language: " +
library.getLanguage());
}
- return new ExternalAdapterFactory(libraryDataverse, libraryName,
adapterFactoryClassname);
+ return new ExternalAdapterFactory(libraryDatabaseName,
libraryDataverse, libraryName, adapterFactoryClassname);
}
@SuppressWarnings("rawtypes")
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
index 5f42089077..404c4e7dbc 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
@@ -26,6 +26,7 @@ import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
@@ -81,8 +82,10 @@ public class ExternalFunctionCompilerUtil {
//TODO(DB): review
return new
ExternalScalarFunctionInfo(function.getSignature().createFunctionIdentifier(),
paramTypes,
- returnType, typeComputer, lang,
function.getLibraryDataverseName(), function.getLibraryName(),
- function.getExternalIdentifier(), function.getResources(),
deterministic, function.getNullCall());
+ returnType, typeComputer, lang,
+ new Namespace(function.getLibraryDatabaseName(),
function.getLibraryDataverseName()),
+ function.getLibraryName(), function.getExternalIdentifier(),
function.getResources(), deterministic,
+ function.getNullCall());
}
private static IFunctionInfo getUnnestFunctionInfo(MetadataProvider
metadataProvider, Function function) {
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
index 82f74d9d7b..441d118f65 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.om.functions.ExternalFunctionInfo;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
@@ -34,10 +34,9 @@ public class ExternalScalarFunctionInfo extends
ExternalFunctionInfo {
private static final long serialVersionUID = 3L;
public ExternalScalarFunctionInfo(FunctionIdentifier fid, List<IAType>
parameterTypes, IAType returnType,
- IResultTypeComputer rtc, ExternalFunctionLanguage language,
DataverseName libraryDataverseName,
- String libraryName, List<String> externalIdentifier, Map<String,
String> resources, boolean deterministic,
- boolean nullCall) {
- super(fid, FunctionKind.SCALAR, parameterTypes, returnType, rtc,
language, libraryDataverseName, libraryName,
+ IResultTypeComputer rtc, ExternalFunctionLanguage language,
Namespace libraryNamespace, String libraryName,
+ List<String> externalIdentifier, Map<String, String> resources,
boolean deterministic, boolean nullCall) {
+ super(fid, FunctionKind.SCALAR, parameterTypes, returnType, rtc,
language, libraryNamespace, libraryName,
externalIdentifier, resources, deterministic, nullCall);
}
}
diff --git
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
index 8a2cefa02a..700c9f2444 100644
---
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
+++
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
@@ -36,22 +36,22 @@ public class ExternalFunctionInfo extends FunctionInfo
implements IExternalFunct
private final List<IAType> parameterTypes;
private final IAType returnType;
private final ExternalFunctionLanguage language;
- private final DataverseName libraryDataverseName;
+ private final Namespace libraryNamespace;
private final String libraryName;
private final List<String> externalIdentifier;
private final Map<String, String> resources;
private final boolean nullCall;
public ExternalFunctionInfo(FunctionIdentifier fid, FunctionKind kind,
List<IAType> parameterTypes,
- IAType returnType, IResultTypeComputer rtc,
ExternalFunctionLanguage language,
- DataverseName libraryDataverseName, String libraryName,
List<String> externalIdentifier,
- Map<String, String> resources, boolean deterministic, boolean
nullCall) {
+ IAType returnType, IResultTypeComputer rtc,
ExternalFunctionLanguage language, Namespace libraryNamespace,
+ String libraryName, List<String> externalIdentifier, Map<String,
String> resources, boolean deterministic,
+ boolean nullCall) {
super(fid, rtc, deterministic);
this.kind = kind;
this.parameterTypes = parameterTypes;
this.returnType = returnType;
this.language = language;
- this.libraryDataverseName = libraryDataverseName;
+ this.libraryNamespace = libraryNamespace;
this.libraryName = libraryName;
this.externalIdentifier = externalIdentifier;
this.resources = resources;
@@ -78,8 +78,8 @@ public class ExternalFunctionInfo extends FunctionInfo
implements IExternalFunct
return language;
}
- public DataverseName getLibraryDataverseName() {
- return libraryDataverseName;
+ public Namespace getLibraryNamespace() {
+ return libraryNamespace;
}
@Override
diff --git
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
index 9eb9875655..e2662e662a 100644
---
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
+++
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.common.functions.ExternalFunctionLanguage;
-import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.common.metadata.Namespace;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
@@ -40,7 +40,7 @@ public interface IExternalFunctionInfo extends IFunctionInfo {
ExternalFunctionLanguage getLanguage();
- DataverseName getLibraryDataverseName();
+ Namespace getLibraryNamespace();
String getLibraryName();
