Xikui Wang has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2078
Change subject: [NO ISSUE][*DB] Enable loading external UDF for
AsterixHyracksITUtil
......................................................................
[NO ISSUE][*DB] Enable loading external UDF for AsterixHyracksITUtil
- user model changes: no
- storage format changes: no
- interface changes: no
Add external UDF support for AsterixHyracksIntegrationUtil. Developer
may use following system properties to install external UDF for local
IT env.
"external.lib" : udf package path.
"external.lib.dataverse": dataverse name (default: test).
"external.lib.libname": library name (default: testlib).
Change-Id: I333238b944b1c6d48a2b80b87a25f600522e0cc4
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
R
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
R
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
7 files changed, 43 insertions(+), 20 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/78/2078/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index f5e94b1..678926e 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -29,11 +29,14 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.asterix.app.external.ExternalUDFLibrarian;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
+import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.config.PropertiesAccessor;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.hyracks.bootstrap.CCApplication;
import org.apache.asterix.hyracks.bootstrap.NCApplication;
import org.apache.commons.io.FileUtils;
@@ -90,7 +93,7 @@
ncApplication.registerConfig(ncConfigManager);
nodeControllers.add(
new
NodeControllerService(fixupIODevices(createNCConfig(nodeId, ncConfigManager)),
ncApplication));
- } ;
+ }
cc.start();
@@ -119,6 +122,22 @@
((ICcApplicationContext)
cc.getApplicationContext()).getClusterStateManager().waitForState(ClusterState.ACTIVE);
hcc = new HyracksConnection(cc.getConfig().getClientListenAddress(),
cc.getConfig().getClientListenPort());
this.ncs = nodeControllers.toArray(new
NodeControllerService[nodeControllers.size()]);
+ }
+
+ public void initLocalInstance(boolean deleteOldInstanceData, String
externalLibPath) throws Exception {
+ List<ILibraryManager> libraryManagers = new ArrayList<>();
+ ExternalUDFLibrarian librarian = new
ExternalUDFLibrarian(libraryManagers);
+ librarian.cleanup();
+ init(deleteOldInstanceData);
+ if (externalLibPath != null && externalLibPath.length() != 0) {
+ libraryManagers.add(((ICcApplicationContext)
cc.getApplicationContext()).getLibraryManager());
+ for (NodeControllerService nc : ncs) {
+ INcApplicationContext runtimeCtx = (INcApplicationContext)
nc.getApplicationContext();
+ libraryManagers.add(runtimeCtx.getLibraryManager());
+ }
+ librarian.install(System.getProperty("external.lib.dataverse",
"test"),
+ System.getProperty("external.lib.libname", "testlib"),
externalLibPath);
+ }
}
public ClusterControllerService getClusterControllerService() {
@@ -263,14 +282,16 @@
public static void main(String[] args) throws Exception {
AsterixHyracksIntegrationUtil integrationUtil = new
AsterixHyracksIntegrationUtil();
try {
- integrationUtil.run(Boolean.getBoolean("cleanup.start"),
Boolean.getBoolean("cleanup.shutdown"));
+
integrationUtil.runLocalInstance(Boolean.getBoolean("cleanup.start"),
+ Boolean.getBoolean("cleanup.shutdown"),
System.getProperty("external.lib", ""));
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Unexpected exception", e);
System.exit(1);
}
}
- protected void run(boolean cleanupOnStart, boolean cleanupOnShutdown)
throws Exception {
+ protected void runLocalInstance(boolean cleanupOnStart, boolean
cleanupOnShutdown, String loadExternalLibs)
+ throws Exception {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
@@ -283,7 +304,7 @@
});
System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY,
"asterix-build-configuration.xml");
- init(cleanupOnStart);
+ initLocalInstance(cleanupOnStart, loadExternalLibs);
while (true) {
Thread.sleep(10000);
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index 5f86c28..4271d55 100755
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -346,7 +346,7 @@
if (jarsInLibDir.length > 1) {
throw new Exception("Incorrect library structure: found multiple
library jars");
}
- if (jarsInLibDir.length < 0) {
+ if (jarsInLibDir.length <= 0) {
throw new Exception("Incorrect library structure: could not find
library jar");
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
similarity index 95%
rename from
asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
rename to
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
index 76fa308..69b7e3c 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalUDFLibrarian.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.asterix.app.external;
import java.io.File;
@@ -28,17 +29,16 @@
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.event.service.AsterixEventServiceUtil;
-import org.apache.asterix.test.common.ITestLibrarian;
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.algebricks.common.utils.Pair;
-public class TestLibrarian implements ITestLibrarian {
+public class ExternalUDFLibrarian implements IExternalUDFLibrarian {
// The following list includes a library manager for the CC
// and library managers for NCs (one-per-NC).
private final List<ILibraryManager> libraryManagers;
- public TestLibrarian(List<ILibraryManager> libraryManagers) {
+ public ExternalUDFLibrarian(List<ILibraryManager> libraryManagers) {
this.libraryManagers = libraryManagers;
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
similarity index 93%
rename from
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java
rename to
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
index d661710..9b1d19d 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ITestLibrarian.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/IExternalUDFLibrarian.java
@@ -16,7 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.test.common;
+
+package org.apache.asterix.app.external;
import java.io.IOException;
import java.rmi.RemoteException;
@@ -24,7 +25,7 @@
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
-public interface ITestLibrarian {
+public interface IExternalUDFLibrarian {
public void install(String dvName, String libName, String libPath) throws
IOException, Exception;
public void uninstall(String dvName, String libName) throws
RemoteException, AsterixException, ACIDException;
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index 104f80b..432e4f1 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -26,7 +26,7 @@
import java.util.Map;
import java.util.logging.Logger;
-import org.apache.asterix.app.external.TestLibrarian;
+import org.apache.asterix.app.external.ExternalUDFLibrarian;
import org.apache.asterix.app.nc.NCAppRuntimeContext;
import org.apache.asterix.app.nc.TransactionSubsystem;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
@@ -129,7 +129,7 @@
File outdir = new File(PATH_ACTUAL);
outdir.mkdirs();
// remove library directory
- TestLibrarian.removeLibraryDir();
+ ExternalUDFLibrarian.removeLibraryDir();
ExecutionTestUtil.setUp(cleanupOnStart,
testConfigFileName == null ? TEST_CONFIG_FILE_NAME :
testConfigFileName,
ExecutionTestUtil.integrationUtil, runHDFS);
@@ -143,7 +143,7 @@
}
public void deInit() throws Exception {
- TestLibrarian.removeLibraryDir();
+ ExternalUDFLibrarian.removeLibraryDir();
ExecutionTestUtil.tearDown(cleanupOnStop);
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 4ca9131..0d66256 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -58,6 +58,7 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.asterix.app.external.IExternalUDFLibrarian;
import org.apache.asterix.common.api.Duration;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.utils.Servlets;
@@ -134,7 +135,7 @@
*/
protected final List<InetSocketAddress> endpoints;
protected int endpointSelector;
- protected ITestLibrarian librarian;
+ protected IExternalUDFLibrarian librarian;
private Map<File, TestLoop> testLoops = new HashMap<>();
public TestExecutor() {
@@ -153,7 +154,7 @@
this.endpoints = endpoints;
}
- public void setLibrarian(ITestLibrarian librarian) {
+ public void setLibrarian(IExternalUDFLibrarian librarian) {
this.librarian = librarian;
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
index 03f42f5..b0e5785 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -31,7 +31,7 @@
import java.util.Collection;
import java.util.List;
-import org.apache.asterix.app.external.TestLibrarian;
+import org.apache.asterix.app.external.ExternalUDFLibrarian;
import org.apache.asterix.common.config.ClusterProperties;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.test.common.TestExecutor;
@@ -56,7 +56,7 @@
private static final List<String> badTestCases = new ArrayList<>();
private static TestExecutor testExecutor;
- private static TestLibrarian librarian;
+ private static ExternalUDFLibrarian librarian;
private static final int repeat = Integer.getInteger("test.repeat", 1);
public static void setUp(String configFile, TestExecutor executor) throws
Exception {
@@ -64,8 +64,8 @@
File outdir = new File(PATH_ACTUAL);
outdir.mkdirs();
List<ILibraryManager> libraryManagers =
ExecutionTestUtil.setUp(cleanupOnStart, configFile);
- TestLibrarian.removeLibraryDir();
- librarian = new TestLibrarian(libraryManagers);
+ ExternalUDFLibrarian.removeLibraryDir();
+ librarian = new ExternalUDFLibrarian(libraryManagers);
testExecutor.setLibrarian(librarian);
if (repeat != 1) {
System.out.println("FYI: each test will be run " + repeat + "
times.");
@@ -79,7 +79,7 @@
// Check whether there are leaked threads.
checkThreadLeaks();
} finally {
- TestLibrarian.removeLibraryDir();
+ ExternalUDFLibrarian.removeLibraryDir();
ExecutionTestUtil.tearDown(cleanupOnStop);
ExecutionTestUtil.integrationUtil.removeTestStorageFiles();
if (!badTestCases.isEmpty()) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/2078
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I333238b944b1c6d48a2b80b87a25f600522e0cc4
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <[email protected]>
