APEX-125 #resolve #comment add long-lived option for app package
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/cecdf4c7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/cecdf4c7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/cecdf4c7 Branch: refs/heads/feature-module Commit: cecdf4c7351eb3605acc7864bb50b1724993181d Parents: 2cd917d Author: siyuan <[email protected]> Authored: Tue Sep 15 13:10:38 2015 -0700 Committer: siyuan <[email protected]> Committed: Tue Sep 15 13:10:38 2015 -0700 ---------------------------------------------------------------------- .../datatorrent/stram/client/AppPackage.java | 47 ++++++++++++++++++-- .../stram/client/AppPackageTest.java | 13 ++++++ 2 files changed, 57 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/cecdf4c7/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java ---------------------------------------------------------------------- diff --git a/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java b/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java index 5698807..f67860d 100644 --- a/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java +++ b/engine/src/main/java/com/datatorrent/stram/client/AppPackage.java @@ -36,7 +36,7 @@ import org.slf4j.LoggerFactory; * * @since 1.0.3 */ -public class AppPackage extends JarFile implements Closeable +public class AppPackage extends JarFile { public static final String ATTRIBUTE_DT_ENGINE_VERSION = "DT-Engine-Version"; public static final String ATTRIBUTE_DT_APP_PACKAGE_NAME = "DT-App-Package-Name"; @@ -63,6 +63,7 @@ public class AppPackage extends JarFile implements Closeable private final Set<String> configs = new TreeSet<String>(); private final File resourcesDirectory; + private final boolean cleanOnClose; public static class AppInfo { @@ -97,14 +98,27 @@ public class AppPackage extends JarFile implements Closeable * If app directory is to be processed, there may be resource leak in the class loader. Only pass true for short-lived * applications * + * If contentFolder is not null, it will try to create the contentFolder, file will be retained on disk after App Package is closed + * If contentFolder is null, temp folder will be created and will be cleaned on close() + * * @param file + * @param contentFolder the folder that the app package will be extracted to * @param processAppDirectory * @throws java.io.IOException * @throws net.lingala.zip4j.exception.ZipException */ - public AppPackage(File file, boolean processAppDirectory) throws IOException, ZipException + public AppPackage(File file, File contentFolder, boolean processAppDirectory) throws IOException, ZipException { super(file); + + if (contentFolder != null) { + FileUtils.forceMkdir(contentFolder); + cleanOnClose = false; + } else { + cleanOnClose = true; + contentFolder = new File("/tmp/dt-appPackage-" + Long.toString(System.nanoTime())); + } + Manifest manifest = getManifest(); if (manifest == null) { throw new IOException("Not a valid app package. MANIFEST.MF is not present."); @@ -120,7 +134,7 @@ public class AppPackage extends JarFile implements Closeable throw new IOException("Not a valid app package. Class-Path is missing from MANIFEST.MF"); } classPath.addAll(Arrays.asList(StringUtils.split(classPathString, " "))); - directory = new File("/tmp/dt-appPackage-" + Long.toString(System.nanoTime())); + directory = contentFolder; extractToDirectory(directory, file); if (processAppDirectory) { processAppDirectory(new File(directory, "app")); @@ -148,6 +162,25 @@ public class AppPackage extends JarFile implements Closeable } } + /** + * Creates an App Package object. + * + * If app directory is to be processed, there may be resource leak in the class loader. Only pass true for short-lived + * applications + * + * Files in app package will be extracted to tmp folder and will be cleaned on close() + * The close() method could be explicitly called or implicitly called by GC finalize() + * + * @param file + * @param processAppDirectory + * @throws java.io.IOException + * @throws net.lingala.zip4j.exception.ZipException + */ + public AppPackage(File file, boolean processAppDirectory) throws IOException, ZipException + { + this(file, null, processAppDirectory); + } + public static void extractToDirectory(File directory, File appPackageFile) throws ZipException { ZipFile zipFile = new ZipFile(appPackageFile); @@ -177,7 +210,15 @@ public class AppPackage extends JarFile implements Closeable public void close() throws IOException { super.close(); + if (cleanOnClose) { + cleanContent(); + } + } + + public void cleanContent() throws IOException + { FileUtils.deleteDirectory(directory); + LOG.debug("App Package {}-{} folder {} is removed", appPackageName, appPackageVersion, directory.getAbsolutePath()); } public String getAppPackageName() http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/cecdf4c7/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java ---------------------------------------------------------------------- diff --git a/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java b/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java index bf41270..6d70eeb 100644 --- a/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java +++ b/engine/src/test/java/com/datatorrent/stram/client/AppPackageTest.java @@ -18,6 +18,7 @@ package com.datatorrent.stram.client; import com.datatorrent.stram.support.StramTestSupport; import com.datatorrent.stram.util.JSONSerializationProvider; import net.lingala.zip4j.exception.ZipException; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -37,6 +38,8 @@ import org.junit.BeforeClass; public class AppPackageTest { private static AppPackage ap; + //yet another app package which retains the files + private static AppPackage yap; private static JSONSerializationProvider jomp; private static JSONObject json; @@ -49,6 +52,9 @@ public class AppPackageTest File file = StramTestSupport.createAppPackageFile(); // Set up test instance ap = new AppPackage(file, true); + // set up another instance + File testfolder = new File("target/testapp"); + yap = new AppPackage(file, testfolder, false); jomp = new JSONSerializationProvider(); json = new JSONObject(jomp.getContext(null).writeValueAsString(ap)); @@ -58,6 +64,9 @@ public class AppPackageTest throw new RuntimeException(e); } catch (JSONException e) { throw new RuntimeException(e); + } finally { + IOUtils.closeQuietly(ap); + IOUtils.closeQuietly(yap); } } @@ -82,6 +91,10 @@ public class AppPackageTest JSONObject dag = application.getJSONObject("dag"); Assert.assertTrue("There is at least one stream", dag.getJSONArray("streams").length() >= 1); Assert.assertEquals("There are two operator", 2, dag.getJSONArray("operators").length()); + + Assert.assertTrue("app package extraction folder should be retained", new File("target/testapp").exists()); + yap.cleanContent(); + Assert.assertTrue("app package extraction folder should be removed", !new File("target/testapp").exists()); } @Test
