Repository: incubator-griffin Updated Branches: refs/heads/master 10afa997e -> 576d3a22e
delete single thread pool for hivemetastoreclient and print full log stack trace in log4j delete single thread pool for hivemetastoreclient and print full log stack trace in log4j Author: jasonliaoxiaoge <181276...@qq.com> Closes #370 from jasonliaoxiaoge/master. Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/576d3a22 Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/576d3a22 Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/576d3a22 Branch: refs/heads/master Commit: 576d3a22e5566b43caf32cae747abadf60ab90dc Parents: 10afa99 Author: jasonliaoxiaoge <181276...@qq.com> Authored: Tue Jul 24 09:11:01 2018 +0800 Committer: William Guo <gu...@apache.org> Committed: Tue Jul 24 09:11:01 2018 +0800 ---------------------------------------------------------------------- .../apache/griffin/core/job/JobServiceImpl.java | 2 +- .../core/job/StreamingJobOperatorImpl.java | 2 -- .../griffin/core/job/entity/JobHealth.java | 6 ++--- .../factory/AutowiringSpringBeanJobFactory.java | 2 +- .../measure/ExternalMeasureOperatorImpl.java | 2 -- .../measure/GriffinMeasureOperatorImpl.java | 4 ---- .../core/metastore/hive/HiveMetaStoreProxy.java | 7 +----- .../hive/HiveMetaStoreServiceImpl.java | 24 +++++++------------- .../org/apache/griffin/core/util/FSUtil.java | 2 +- 9 files changed, 15 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/576d3a22/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java index 29cb249..694b7ab 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java @@ -252,7 +252,7 @@ public class JobServiceImpl implements JobService { try { jobHealth = op.getHealth(jobHealth, job); } catch (SchedulerException e) { - LOGGER.error("Job schedule exception. {}", e.getMessage()); + LOGGER.error("Job schedule exception. {}", e); throw new GriffinException.ServiceException("Fail to Get HealthInfo", e); } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/576d3a22/service/src/main/java/org/apache/griffin/core/job/StreamingJobOperatorImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/StreamingJobOperatorImpl.java b/service/src/main/java/org/apache/griffin/core/job/StreamingJobOperatorImpl.java index 03ae36a..78fe5b9 100644 --- a/service/src/main/java/org/apache/griffin/core/job/StreamingJobOperatorImpl.java +++ b/service/src/main/java/org/apache/griffin/core/job/StreamingJobOperatorImpl.java @@ -32,7 +32,6 @@ import org.quartz.TriggerKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.env.Environment; import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.springframework.stereotype.Service; @@ -44,7 +43,6 @@ import org.springframework.web.client.RestTemplate; import javax.annotation.PostConstruct; import java.util.List; -import java.util.Properties; import static org.apache.griffin.core.exception.GriffinExceptionMessage.INVALID_JOB_NAME; import static org.apache.griffin.core.exception.GriffinExceptionMessage.STREAMING_JOB_IS_RUNNING; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/576d3a22/service/src/main/java/org/apache/griffin/core/job/entity/JobHealth.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/entity/JobHealth.java b/service/src/main/java/org/apache/griffin/core/job/entity/JobHealth.java index 9d2a654..e0ecc90 100644 --- a/service/src/main/java/org/apache/griffin/core/job/entity/JobHealth.java +++ b/service/src/main/java/org/apache/griffin/core/job/entity/JobHealth.java @@ -39,11 +39,11 @@ public class JobHealth { this.jobCount = jobCount; } - public JobHealth() { - } - public JobHealth(int healthyJobCount, int jobCount) { this.healthyJobCount = healthyJobCount; this.jobCount = jobCount; } + + public JobHealth() { + } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/576d3a22/service/src/main/java/org/apache/griffin/core/job/factory/AutowiringSpringBeanJobFactory.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/job/factory/AutowiringSpringBeanJobFactory.java b/service/src/main/java/org/apache/griffin/core/job/factory/AutowiringSpringBeanJobFactory.java index cc2ff15..fe06019 100644 --- a/service/src/main/java/org/apache/griffin/core/job/factory/AutowiringSpringBeanJobFactory.java +++ b/service/src/main/java/org/apache/griffin/core/job/factory/AutowiringSpringBeanJobFactory.java @@ -47,7 +47,7 @@ public final class AutowiringSpringBeanJobFactory extends SpringBeanJobFactory return job; } catch (Exception e) { - LOGGER.error("fail to create job instance. {}", e.getMessage()); + LOGGER.error("fail to create job instance. {}", e); } return null; } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/576d3a22/service/src/main/java/org/apache/griffin/core/measure/ExternalMeasureOperatorImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/ExternalMeasureOperatorImpl.java b/service/src/main/java/org/apache/griffin/core/measure/ExternalMeasureOperatorImpl.java index cbf5504..3ab346e 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/ExternalMeasureOperatorImpl.java +++ b/service/src/main/java/org/apache/griffin/core/measure/ExternalMeasureOperatorImpl.java @@ -24,8 +24,6 @@ import org.apache.griffin.core.job.repo.VirtualJobRepo; import org.apache.griffin.core.measure.entity.ExternalMeasure; import org.apache.griffin.core.measure.entity.Measure; import org.apache.griffin.core.measure.repo.ExternalMeasureRepo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/576d3a22/service/src/main/java/org/apache/griffin/core/measure/GriffinMeasureOperatorImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/measure/GriffinMeasureOperatorImpl.java b/service/src/main/java/org/apache/griffin/core/measure/GriffinMeasureOperatorImpl.java index 9938e0e..b1aa37d 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/GriffinMeasureOperatorImpl.java +++ b/service/src/main/java/org/apache/griffin/core/measure/GriffinMeasureOperatorImpl.java @@ -23,8 +23,6 @@ import org.apache.griffin.core.job.JobServiceImpl; import org.apache.griffin.core.measure.entity.Measure; import org.apache.griffin.core.measure.repo.MeasureRepo; import org.quartz.SchedulerException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -32,8 +30,6 @@ import static org.apache.griffin.core.util.MeasureUtil.validateMeasure; @Component("griffinOperation") public class GriffinMeasureOperatorImpl implements MeasureOperator { - private static final Logger LOGGER = LoggerFactory.getLogger(GriffinMeasureOperatorImpl.class); - private final MeasureRepo<Measure> measureRepo; private final JobServiceImpl jobService; http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/576d3a22/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java index 35274f9..0e4e132 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java @@ -21,7 +21,6 @@ package org.apache.griffin.core.metastore.hive; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.MetaException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -29,10 +28,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import javax.annotation.PreDestroy; -import java.io.File; -import java.net.URI; -import java.net.URL; -import java.security.CodeSource; @Component public class HiveMetaStoreProxy { @@ -66,7 +61,7 @@ public class HiveMetaStoreProxy { try { client = new HiveMetaStoreClient(hiveConf); } catch (Exception e) { - LOGGER.error("Failed to connect hive metastore. {}", e.getMessage()); + LOGGER.error("Failed to connect hive metastore. {}", e); } return client; } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/576d3a22/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java index 6b04488..b1f9f62 100644 --- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java @@ -53,11 +53,8 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { @Value("${hive.metastore.dbname}") private String defaultDbName; - private ThreadPoolExecutor singleThreadExecutor; public HiveMetaStoreServiceImpl() { - singleThreadExecutor = new ThreadPoolExecutor(1, 5, 3, TimeUnit.SECONDS, new ArrayBlockingQueue<>(3), new ThreadPoolExecutor.DiscardPolicy()); - LOGGER.info("HiveMetaStoreServiceImpl single thread pool created."); } @Override @@ -72,7 +69,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { results = client.getAllDatabases(); } catch (Exception e) { reconnect(); - LOGGER.error("Can not get databases : {}", e.getMessage()); + LOGGER.error("Can not get databases : {}", e); } return results; } @@ -90,7 +87,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { results = client.getAllTables(getUseDbName(dbName)); } catch (Exception e) { reconnect(); - LOGGER.error("Exception fetching tables info: {}", e.getMessage()); + LOGGER.error("Exception fetching tables info: {}", e); } return results; } @@ -136,7 +133,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { result = client.getTable(getUseDbName(dbName), tableName); } catch (Exception e) { reconnect(); - LOGGER.error("Exception fetching table info : {}. {}", tableName, e.getMessage()); + LOGGER.error("Exception fetching table info : {}. {}", tableName, e); } return result; } @@ -165,7 +162,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { } } catch (Exception e) { reconnect(); - LOGGER.error("Exception fetching tables info: {}", e.getMessage()); + LOGGER.error("Exception fetching tables info: {}", e); } return allTables; } @@ -179,15 +176,10 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService { } private void reconnect() { - if (singleThreadExecutor.getActiveCount() == 0) { - System.out.println("execute create thread."); - singleThreadExecutor.execute(() -> { - try { - client.reconnect(); - } catch (Exception e) { - LOGGER.error("reconnect to hive failed."); - } - }); + try { + client.reconnect(); + } catch (Exception e) { + LOGGER.error("reconnect to hive failed: {}", e); } } } http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/576d3a22/service/src/main/java/org/apache/griffin/core/util/FSUtil.java ---------------------------------------------------------------------- diff --git a/service/src/main/java/org/apache/griffin/core/util/FSUtil.java b/service/src/main/java/org/apache/griffin/core/util/FSUtil.java index 1aa20e6..c8f27d3 100644 --- a/service/src/main/java/org/apache/griffin/core/util/FSUtil.java +++ b/service/src/main/java/org/apache/griffin/core/util/FSUtil.java @@ -65,7 +65,7 @@ public class FSUtil { try { fileSystem = FileSystem.get(conf); } catch (Exception e) { - LOGGER.error("Can not get hdfs file system. {}", e.getMessage()); + LOGGER.error("Can not get hdfs file system. {}", e); } }
