[OODT-978] Fixed crawler launcher hanging issue in file manager avro version by adding a close() method to the FileManagerClient
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/1c49a9bb Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/1c49a9bb Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/1c49a9bb Branch: refs/heads/OODT-978 Commit: 1c49a9bbbc8e7c95bd23106c3dc58560c5af7488 Parents: 19e5162 Author: Imesha Sudasingha <imesha.sudasin...@gmail.com> Authored: Wed May 16 15:56:29 2018 +0530 Committer: Imesha Sudasingha <imesha.sudasin...@gmail.com> Committed: Wed May 16 15:56:29 2018 +0530 ---------------------------------------------------------------------- .../apache/oodt/cas/crawl/CrawlerLauncher.java | 4 ++- .../apache/oodt/cas/crawl/ProductCrawler.java | 19 ++++++++++-- .../cli/action/CrawlerLauncherCliAction.java | 5 +++- .../oodt/cas/filemgr/ingest/CachedIngester.java | 13 ++++++-- .../cas/filemgr/ingest/CmdLineIngester.java | 1 + .../oodt/cas/filemgr/ingest/Ingester.java | 3 +- .../oodt/cas/filemgr/ingest/StdIngester.java | 31 +++++++++++++------- .../filemgr/system/AvroFileManagerClient.java | 17 +++++++---- .../cas/filemgr/system/FileManagerClient.java | 5 +++- .../filemgr/system/XmlRpcFileManagerClient.java | 5 ++++ .../cli/action/DummyFileManagerClient.java | 5 ++++ 11 files changed, 81 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java ---------------------------------------------------------------------- diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java index 9fd682c..b47c7e2 100644 --- a/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java +++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java @@ -18,13 +18,14 @@ package org.apache.oodt.cas.crawl; //JDK imports import java.io.IOException; +import java.util.Set; //OODT imports import org.apache.oodt.cas.cli.CmdLineUtility; /** * A command line interface to the new Spring enabled crawler. - * + * * @author bfoster (Brian Foster) * @version $Revision$ * @since OODT-190 @@ -34,5 +35,6 @@ public class CrawlerLauncher { public static void main(String[] args) throws IOException { CmdLineUtility cmdLineUtility = new CmdLineUtility(); cmdLineUtility.run(args); + System.out.println("Exiting crawler launcher"); } } http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java ---------------------------------------------------------------------- diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java index 3f5d254..494b2c2 100644 --- a/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java +++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java @@ -17,8 +17,6 @@ package org.apache.oodt.cas.crawl; import com.google.common.annotations.VisibleForTesting; - -//OODT imports import org.apache.oodt.cas.crawl.action.CrawlerAction; import org.apache.oodt.cas.crawl.action.CrawlerActionRepo; import org.apache.oodt.cas.crawl.config.ProductCrawlerBean; @@ -29,6 +27,7 @@ import org.apache.oodt.cas.metadata.Metadata; import java.io.File; import java.io.FileFilter; +import java.io.IOException; import java.net.URL; import java.util.Collections; import java.util.List; @@ -37,6 +36,8 @@ import java.util.Vector; import java.util.logging.Level; import java.util.logging.Logger; +//OODT imports + /** * An abstract base class for Product Crawling. This class provides methods to @@ -71,11 +72,15 @@ public abstract class ProductCrawler extends ProductCrawlerBean { protected Ingester ingester; public void crawl() { + LOG.info("Crawl operation invoked"); crawl(new File(getProductPath())); } public void crawl(File dirRoot) { - // Reset ingest status. + LOG.info(String.format("Start crawling dir: %s", dirRoot)); + + // Reset ingest status.a + ingestStatus.clear(); // Load actions. @@ -114,6 +119,8 @@ public abstract class ProductCrawler extends ProductCrawlerBean { } } } + + LOG.info(String.format("Finished crawling dir: %s", dirRoot)); } public IngestStatus handleFile(File product) { @@ -363,4 +370,10 @@ public abstract class ProductCrawler extends ProductCrawlerBean { } return allSucceeded; } + + public void shutdown() throws IOException { + if (ingester != null) { + ingester.close(); + } + } } http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java ---------------------------------------------------------------------- diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java index cce8ea8..b7b70ef 100644 --- a/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java +++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java @@ -27,7 +27,7 @@ import org.springframework.context.support.FileSystemXmlApplicationContext; /** * A {@link CmdLineAction} which is responsible for launching crawlers. - * + * * @author bfoster (Brian Foster) */ public class CrawlerLauncherCliAction extends CmdLineAction { @@ -54,9 +54,12 @@ public class CrawlerLauncherCliAction extends CmdLineAction { if (pc.getDaemonPort() != -1 && pc.getDaemonWait() != -1) { new CrawlDaemon(pc.getDaemonWait(), pc, pc.getDaemonPort()) .startCrawling(); + printer.println("Finished crawler daemon"); } else { pc.crawl(); + printer.println("Finished crawling"); } + pc.shutdown(); } catch (Exception e) { throw new CmdLineActionException("Failed to launch crawler : " + e.getMessage(), e); http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java index 8e96be1..52e5c9f 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java @@ -25,6 +25,7 @@ import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; import java.net.URISyntaxException; import java.net.URL; @@ -39,7 +40,7 @@ import java.util.logging.Logger; * * <p> * An extension of the {@link StdIngester} that uses a {@link Cache} to keep - * track of {@link Product} ingestion status. If the existing {@link Cache} used + * track of {@link org.apache.oodt.cas.filemgr.structs.Product} ingestion status. If the existing {@link Cache} used * is already sync'ed to the requested File Manager (specified by the * <code>fmUrl</code> parameter in {@link #hasProduct(URL, File)} or * {@link #hasProduct(URL, String)}), then the {@link Cache} will simply return @@ -58,7 +59,7 @@ public class CachedIngester extends StdIngester { /** * @param transferService * The underlying data transfer service to use to ingest - * {@link Product}s. + * {@link org.apache.oodt.cas.filemgr.structs.Product}s. * @param cacheServiceFactory * The {@link CacheFactory} to use to construct this * {@link Ingester}'s {@link Cache}. @@ -94,7 +95,7 @@ public class CachedIngester extends StdIngester { * * @param transferService * The underlying data transfer service to use to ingest - * {@link Product}s. + * {@link org.apache.oodt.cas.filemgr.structs.Product}s. * @param cache * The {@link Cache} that this {@link Ingester} will use. * @throws InstantiationException @@ -167,4 +168,10 @@ public class CachedIngester extends StdIngester { + e.getMessage()); } } + + @Override + public void close() throws IOException { + cache.clear(); + super.close(); + } } http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java index 59d0736..8d6807b 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java @@ -128,6 +128,7 @@ public class CmdLineIngester extends StdIngester { System.out.println("Result: " + productID); } + ingester.close(); } private static List<String> readProdFilesFromStdin() { http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java index b12dfd4..8e19273 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java @@ -24,6 +24,7 @@ import org.apache.oodt.cas.metadata.MetExtractor; import org.apache.oodt.cas.metadata.Metadata; //JDK imports +import java.io.Closeable; import java.io.File; import java.net.URL; import java.util.List; @@ -37,7 +38,7 @@ import java.util.List; * An interface for ingesting {@link Product}s * </p>. */ -public interface Ingester { +public interface Ingester extends Closeable { /** * Ingests a {@link Product} to the file manager service object identified http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java index 2421873..0195bb6 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java @@ -18,6 +18,7 @@ package org.apache.oodt.cas.filemgr.ingest; //OODT imports + import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys; import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.ProductType; @@ -26,15 +27,15 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException; import org.apache.oodt.cas.filemgr.structs.exceptions.IngestException; import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import org.apache.oodt.cas.filemgr.system.FileManagerClient; -import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory; +import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory; import org.apache.oodt.cas.filemgr.versioning.VersioningUtils; import org.apache.oodt.cas.metadata.MetExtractor; import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.metadata.exceptions.MetExtractionException; -//JDK imports import java.io.File; +import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; import java.util.List; @@ -42,26 +43,28 @@ import java.util.Vector; import java.util.logging.Level; import java.util.logging.Logger; +//JDK imports + /** * @author mattmann * @author bfoster * @version $Revision$ - * + * * <p> * An implementation of the {@link Ingster} interface that uses the following * pieces of {@link Metadata} information to determine how to ingest a * {@link Product}: - * + * * <ul> * <li>Filename - The name of the Product file to ingest.</li> * <li>ProductType - The type of the Product to ingest.</li> * <li>FileLocation - A full path pointer to directory containing the Product * file to ingest.</li> * </ul> - * + * * The {@link Ingester} supports overriding certain {@link Product} properties, * including: - * + * * <ul> * <li>Specification of <code>ProductStructure</code> parameter that will * tell the {@link Ingester} whether or not the {@link Product} is a directory @@ -90,7 +93,7 @@ public class StdIngester implements Ingester, CoreMetKeys { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.ingest.Ingester#ingest(java.net.URL, * java.io.File, org.apache.oodt.cas.metadata.MetExtractor, * java.io.File) @@ -111,7 +114,7 @@ public class StdIngester implements Ingester, CoreMetKeys { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.ingest.Ingester#ingest(java.net.URL, * java.util.List, org.apache.oodt.cas.metadata.MetExtractor, java.io.File) */ @@ -143,7 +146,7 @@ public class StdIngester implements Ingester, CoreMetKeys { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.ingest.Ingester#ingest(java.net.URL, * java.io.File, org.apache.oodt.cas.metadata.Metadata) */ @@ -224,7 +227,7 @@ public class StdIngester implements Ingester, CoreMetKeys { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.ingest.Ingester#hasProduct(java.net.URL, * java.io.File) */ @@ -293,7 +296,7 @@ public class StdIngester implements Ingester, CoreMetKeys { /* * (non-Javadoc) - * + * * @see org.apache.oodt.cas.filemgr.ingest.Ingester#hasProduct(java.net.URL, * java.lang.String) */ @@ -309,4 +312,10 @@ public class StdIngester implements Ingester, CoreMetKeys { } } + @Override + public void close() throws IOException { + if (fmClient != null) { + fmClient.close(); + } + } } http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java index ca3839d..0b28bff 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java @@ -88,7 +88,7 @@ public class AvroFileManagerClient implements FileManagerClient { //setup the and start the client try { this.fileManagerUrl = url; - InetSocketAddress inetSocketAddress = new InetSocketAddress(url.getHost(),this.fileManagerUrl.getPort()); + InetSocketAddress inetSocketAddress = new InetSocketAddress(url.getHost(), this.fileManagerUrl.getPort()); this.client = new NettyTransceiver(inetSocketAddress, 40000L); proxy = (AvroFileManager) SpecificRequestor.getClient(AvroFileManager.class, client); } catch (IOException e) { @@ -124,8 +124,7 @@ public class AvroFileManagerClient implements FileManagerClient { try { if (proxy != null) { success = proxy.isAlive(); - } - else return false; + } else return false; } catch (AvroRemoteException e) { LOG.log(Level.WARNING, "AvroRemoteException when connecting to filemgr: [" + this.fileManagerUrl + "]"); @@ -155,7 +154,7 @@ public class AvroFileManagerClient implements FileManagerClient { success = proxy.removeProductTransferStatus(AvroTypeFactory.getAvroProduct(product)); } catch (AvroRemoteException e) { throw new DataTransferException(e.getMessage()); - } + } return success; } @@ -584,7 +583,7 @@ public class AvroFileManagerClient implements FileManagerClient { @Override public String ingestProduct(Product product, Metadata metadata, - boolean clientTransfer) throws Exception { + boolean clientTransfer) throws Exception { try { // ingest product String productId = this.proxy.ingestProduct( @@ -594,7 +593,7 @@ public class AvroFileManagerClient implements FileManagerClient { if (clientTransfer) { LOG.log(Level.FINEST, "File Manager Client: clientTransfer enabled: transfering product [" - + product.getProductName() + "]"); + + product.getProductName() + "]"); // we need to transfer the product ourselves // make sure we have the product ID @@ -736,4 +735,10 @@ public class AvroFileManagerClient implements FileManagerClient { this.dataTransfer = dataTransfer; } + @Override + public void close() throws IOException { + if (client != null) { + client.close(); + } + } } http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java index 8b1c54a..c0fa46a 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java @@ -35,6 +35,8 @@ import org.apache.oodt.cas.filemgr.structs.query.QueryResult; import org.apache.oodt.cas.metadata.Metadata; //JDK imports +import java.io.Closeable; +import java.io.IOException; import java.net.URL; import java.util.List; @@ -45,7 +47,7 @@ import java.util.List; * <p>Interface of client for FileManager RPC logic. All methods that are used for * RPC transfer.</p> */ -public interface FileManagerClient { +public interface FileManagerClient extends Closeable { public boolean refreshConfigAndPolicy(); @@ -176,4 +178,5 @@ public interface FileManagerClient { public void setDataTransfer(DataTransfer dataTransfer); + void close() throws IOException; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java index 3bd91fd..06c2726 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java @@ -1404,4 +1404,9 @@ public class XmlRpcFileManagerClient implements FileManagerClient { this.dataTransfer.setFileManagerUrl(this.fileManagerUrl); } + @Override + public void close() throws IOException { + + } + } http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java ---------------------------------------------------------------------- diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java index 6a4c2bb..ebf42f0 100644 --- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java +++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java @@ -285,4 +285,9 @@ public class DummyFileManagerClient implements FileManagerClient { public void setDataTransfer(DataTransfer dataTransfer) { } + + @Override + public void close() throws IOException { + + } }
