Repository: incubator-impala Updated Branches: refs/heads/master bc8c55afc -> 947305741
IMPALA-3779: Disable cache pool reader thread when HDFS isn't running When running Impala on S3 (without hdfs started) the logs every minute produced an error which detailed that the HDFS cache pool couldn't be read from. This change disables the cache pool reader thread if HDFS is not the default filesystem. Change-Id: I2ff3769fff9099aed88a6a587aa721b3487216bd Reviewed-on: http://gerrit.cloudera.org:8080/3577 Reviewed-by: Alex Behm <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/94730574 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/94730574 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/94730574 Branch: refs/heads/master Commit: 94730574192d905c8c4b44143f45335cdddb7071 Parents: bc8c55a Author: Sailesh Mukil <[email protected]> Authored: Wed Jul 6 11:32:05 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Wed Jul 20 02:30:09 2016 -0700 ---------------------------------------------------------------------- .../cloudera/impala/catalog/CatalogServiceCatalog.java | 12 +++++++++++- .../java/com/cloudera/impala/common/FileSystemUtil.java | 7 ++++++- 2 files changed, 17 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/94730574/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java index f8ae864..dddc545 100644 --- a/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java +++ b/fe/src/main/java/com/cloudera/impala/catalog/CatalogServiceCatalog.java @@ -14,6 +14,7 @@ package com.cloudera.impala.catalog; +import java.io.IOException; import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; @@ -160,7 +161,16 @@ public class CatalogServiceCatalog extends Catalog { catalogServiceId_ = catalogServiceId; tableLoadingMgr_ = new TableLoadingMgr(this, numLoadingThreads); loadInBackground_ = loadInBackground; - cachePoolReader_.scheduleAtFixedRate(new CachePoolReader(), 0, 1, TimeUnit.MINUTES); + try { + // We want only 'true' HDFS filesystems to poll the HDFS cache (i.e not S3, + // local, etc.) + if (FileSystemUtil.getDefaultFileSystem() instanceof DistributedFileSystem) { + cachePoolReader_.scheduleAtFixedRate( + new CachePoolReader(), 0, 1, TimeUnit.MINUTES); + } + } catch (IOException e) { + LOG.error("Couldn't identify the default FS. Cache Pool reader will be disabled."); + } if (sentryConfig != null) { sentryProxy_ = new SentryProxy(sentryConfig, this, kerberosPrincipal); } else { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/94730574/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java b/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java index 1f3da2e..e22ba15 100644 --- a/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java +++ b/fe/src/main/java/com/cloudera/impala/common/FileSystemUtil.java @@ -323,9 +323,14 @@ public class FileSystemUtil { return isDistributedFileSystem(path.getFileSystem(CONF)); } - public static DistributedFileSystem getDistributedFileSystem() throws IOException { + public static FileSystem getDefaultFileSystem() throws IOException { Path path = new Path(FileSystem.getDefaultUri(CONF)); FileSystem fs = path.getFileSystem(CONF); + return fs; + } + + public static DistributedFileSystem getDistributedFileSystem() throws IOException { + FileSystem fs = getDefaultFileSystem(); Preconditions.checkState(fs instanceof DistributedFileSystem); return (DistributedFileSystem) fs; }
