[EAGLE-605] Fix MR feeder provider.xml and remove useless configure https://issues.apache.org/jira/browse/EAGLE-605
1. add missing configure from application.conf to provider.xml 2. remove useless configure like : pathContainsJobTrackerName/jobTrackerName/dryRun 3. Fix bug: add IsExecutable to ApplicationDesc so that web ui will know some applications can not be started, just static Author: wujinhu <wujinhu...@126.com> Closes #490 from wujinhu/ui-test. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/fa85dc3e Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/fa85dc3e Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/fa85dc3e Branch: refs/heads/master Commit: fa85dc3e20f163ff84f39ca51d16c5d54fad0482 Parents: 97978c6 Author: wujinhu <wujinhu...@126.com> Authored: Tue Oct 11 16:36:24 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Tue Oct 11 16:36:24 2016 +0800 ---------------------------------------------------------------------- .../service/ApplicationOperationContext.java | 1 - .../impl/ApplicationManagementServiceImpl.java | 22 ++++-- .../impl/ApplicationProviderSPILoader.java | 1 + .../eagle/metadata/model/ApplicationDesc.java | 9 +++ .../eagle/metadata/model/ApplicationEntity.java | 11 +-- .../jpm/mr/history/MRHistoryJobConfig.java | 6 -- .../history/crawler/AbstractJobHistoryDAO.java | 42 +---------- .../history/crawler/JHFCrawlerDriverImpl.java | 21 ++---- .../mr/history/crawler/JobHistoryDAOImpl.java | 2 +- ....history.MRHistoryJobApplicationProvider.xml | 76 +++++++++---------- .../src/main/resources/application.conf | 5 -- ....running.MRRunningJobApplicationProvider.xml | 77 ++++++++++++-------- .../src/main/resources/application.conf | 2 - .../jpm/util/jobrecover/RunningJobManager.java | 4 +- eagle-topology-assembly/pom.xml | 2 +- 15 files changed, 117 insertions(+), 164 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java index 33a407c..3561374 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java @@ -90,7 +90,6 @@ public class ApplicationOperationContext implements Serializable, ApplicationLif @Override public void onInstall() { - metadata.setExecutable(application.isExecutable()); if (metadata.getDescriptor().getStreams() != null) { List<StreamDesc> streamDescToInstall = metadata.getDescriptor().getStreams().stream().map((streamDefinition -> { StreamDefinition copied = streamDefinition.copy(); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java index 04a7007..abd9197 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java @@ -21,6 +21,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.typesafe.config.Config; import org.apache.eagle.alert.metadata.IMetadataDao; +import org.apache.eagle.app.Application; import org.apache.eagle.app.service.ApplicationManagementService; import org.apache.eagle.app.service.ApplicationOperations; import org.apache.eagle.app.service.ApplicationOperationContext; @@ -141,12 +142,14 @@ public class ApplicationManagementServiceImpl implements ApplicationManagementSe @Override public ApplicationEntity start(ApplicationOperations.StartOperation operation) throws ApplicationWrongStatusException { ApplicationEntity applicationEntity = applicationEntityService.getByUUIDOrAppId(operation.getUuid(), operation.getAppId()); + Application application = applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(); + Preconditions.checkArgument(application.isExecutable(), "Application is not executable"); + ApplicationEntity.Status currentStatus = applicationEntity.getStatus(); try { if (currentStatus == ApplicationEntity.Status.INITIALIZED || currentStatus == ApplicationEntity.Status.STOPPED) { ApplicationOperationContext applicationOperationContext = new ApplicationOperationContext( - applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(), - applicationEntity, config, alertMetadataService); + application, applicationEntity, config, alertMetadataService); applicationOperationContext.onStart(); //Only when topology submitted successfully can the state change to STARTING @@ -172,9 +175,11 @@ public class ApplicationManagementServiceImpl implements ApplicationManagementSe @Override public ApplicationEntity stop(ApplicationOperations.StopOperation operation) throws ApplicationWrongStatusException { ApplicationEntity applicationEntity = applicationEntityService.getByUUIDOrAppId(operation.getUuid(), operation.getAppId()); + Application application = applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(); + Preconditions.checkArgument(application.isExecutable(), "Application is not executable"); + ApplicationOperationContext applicationOperationContext = new ApplicationOperationContext( - applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(), - applicationEntity, config, alertMetadataService); + application, applicationEntity, config, alertMetadataService); ApplicationEntity.Status currentStatus = applicationEntity.getStatus(); try { if (currentStatus == ApplicationEntity.Status.RUNNING) { @@ -200,12 +205,13 @@ public class ApplicationManagementServiceImpl implements ApplicationManagementSe @Override public ApplicationEntity.Status getStatus(ApplicationOperations.CheckStatusOperation operation) { - ApplicationEntity applicationEntity = null; try { - applicationEntity = applicationEntityService.getByUUIDOrAppId(operation.getUuid(), operation.getAppId()); + ApplicationEntity applicationEntity = applicationEntityService.getByUUIDOrAppId(operation.getUuid(), operation.getAppId()); + Application application = applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(); + Preconditions.checkArgument(application.isExecutable(), "Application is not executable"); + ApplicationOperationContext applicationOperationContext = new ApplicationOperationContext( - applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(), - applicationEntity, config, alertMetadataService); + application, applicationEntity, config, alertMetadataService); ApplicationEntity.Status topologyStatus = applicationOperationContext.getStatus(); return topologyStatus; } catch (IllegalArgumentException e) { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java index bd96e28..9f52c9c 100644 --- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java @@ -80,6 +80,7 @@ public class ApplicationProviderSPILoader extends ApplicationProviderLoader { ApplicationProviderConfig providerConfig = new ApplicationProviderConfig(); providerConfig.setClassName(applicationProvider.getClass().getCanonicalName()); providerConfig.setJarPath(jarFileSupplier.apply(applicationProvider)); + applicationProvider.getApplicationDesc().setExecutable(applicationProvider.getApplication().isExecutable()); applicationProvider.prepare(providerConfig, getConfig()); registerProvider(applicationProvider); LOG.warn("Loaded {}:{} ({}) from {}", http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java index 506bdc6..8bc6ee3 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java +++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java @@ -36,6 +36,15 @@ public class ApplicationDesc implements Serializable { private Configuration configuration; private List<StreamDefinition> streams; private ApplicationDocs docs; + private boolean executable; + + public boolean isExecutable() { + return executable; + } + + public void setExecutable(boolean executable) { + this.executable = executable; + } private List<ApplicationDependency> dependencies; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java index bae6ba7..5297de1 100644 --- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java +++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java @@ -33,7 +33,6 @@ public class ApplicationEntity extends PersistenceEntity { private String appId; private SiteEntity site; private ApplicationDesc descriptor; - private boolean executable = true; private Map<String, Object> configuration = new HashMap<>(); private Map<String, String> context = new HashMap<>(); @@ -153,15 +152,7 @@ public class ApplicationEntity extends PersistenceEntity { this.streams = streams; } - public boolean isExecutable() { - return executable; - } - - public void setExecutable(boolean executable) { - this.executable = executable; - } - - public static enum Status { + public enum Status { INITIALIZED("INITIALIZED"), STARTING("STARTING"), RUNNING("RUNNING"), http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java index 839d2e4..37df5ad 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java @@ -85,13 +85,10 @@ public class MRHistoryJobConfig implements Serializable { public static class JobHistoryEndpointConfig implements Serializable { public String mrHistoryServerUrl; public String basePath; - public boolean pathContainsJobTrackerName; - public String jobTrackerName; public Map<String, String> hdfs; } public static class ControlConfig implements Serializable { - public boolean dryRun; public Class<? extends JobIdPartitioner> partitionerCls; public boolean zeroBasedMonth; public String timeZone; @@ -158,14 +155,11 @@ public class MRHistoryJobConfig implements Serializable { //parse job history endpoint this.jobHistoryEndpointConfig.basePath = config.getString("endpointConfig.basePath"); - this.jobHistoryEndpointConfig.jobTrackerName = config.getString("endpointConfig.jobTrackerName"); this.jobHistoryEndpointConfig.mrHistoryServerUrl = config.getString("endpointConfig.mrHistoryServerUrl"); - this.jobHistoryEndpointConfig.pathContainsJobTrackerName = config.getBoolean("endpointConfig.pathContainsJobTrackerName"); for (Map.Entry<String, ConfigValue> entry : config.getConfig("endpointConfig.hdfs").entrySet()) { this.jobHistoryEndpointConfig.hdfs.put(entry.getKey(), entry.getValue().unwrapped().toString()); } //parse control config - this.controlConfig.dryRun = config.getBoolean("controlConfig.dryRun"); try { this.controlConfig.partitionerCls = (Class<? extends JobIdPartitioner>) Class.forName(config.getString("controlConfig.partitionerCls")); assert this.controlConfig.partitionerCls != null; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java index 74489cd..9ed2ed2 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java @@ -45,35 +45,19 @@ public abstract class AbstractJobHistoryDAO implements JobHistoryLCM { protected static final Pattern JOBID_PATTERN = Pattern.compile("job_\\d+_\\d+"); protected final String basePath; - protected volatile String jobTrackerName; public static final String JOB_CONF_POSTFIX = "_conf.xml"; private static final Timer timer = new Timer(true); private static final long JOB_TRACKER_SYNC_DURATION = 10 * 60 * 1000; // 10 minutes - private boolean pathContainsJobTrackerName; - - public AbstractJobHistoryDAO(String basePath, boolean pathContainsJobTrackerName, String startingJobTrackerName) throws Exception { + public AbstractJobHistoryDAO(String basePath) throws Exception { this.basePath = basePath; - this.pathContainsJobTrackerName = pathContainsJobTrackerName; - jobTrackerName = startingJobTrackerName; - if (this.pathContainsJobTrackerName) { - if (startingJobTrackerName == null || startingJobTrackerName.isEmpty()) { - throw new IllegalStateException("startingJobTrackerName should not be null or empty"); - } - // start background thread to check what is current job tracker - startThread(this.basePath); - } } protected String buildWholePathToYearMonthDay(int year, int month, int day) { StringBuilder sb = new StringBuilder(); sb.append(basePath); - if (!pathContainsJobTrackerName && jobTrackerName != null && !jobTrackerName.isEmpty()) { - sb.append("/"); - sb.append(jobTrackerName); - } sb.append(String.format(YEAR_MONTH_DAY_URL_FORMAT, year, month, day)); return sb.toString(); } @@ -110,30 +94,6 @@ public abstract class AbstractJobHistoryDAO implements JobHistoryLCM { return null; } - private void startThread(final String basePath) throws Exception { - LOG.info("start an every-" + JOB_TRACKER_SYNC_DURATION / (60 * 1000) + "min timer task to check current jobTrackerName in background"); - // Automatically update current job tracker name in background every 30 minutes - timer.scheduleAtFixedRate(new TimerTask() { - @Override - public void run() { - try { - LOG.info("regularly checking current jobTrackerName in background"); - final String _jobTrackerName = calculateJobTrackerName(basePath); - if (_jobTrackerName != null && !_jobTrackerName.equals(jobTrackerName)) { - LOG.info("jobTrackerName changed from " + jobTrackerName + " to " + _jobTrackerName); - jobTrackerName = _jobTrackerName; - } - LOG.info("Current jobTrackerName is: " + jobTrackerName); - } catch (Exception e) { - LOG.error("failed to figure out current job tracker name that is not configured due to: " + e.getMessage(), e); - } catch (Throwable t) { - LOG.error("failed to figure out current job tracker name that is not configured due to: " + t.getMessage(), t); - } - } - }, JOB_TRACKER_SYNC_DURATION, JOB_TRACKER_SYNC_DURATION); - } - - @Override public void readFileContent(int year, int month, int day, int serialNumber, String jobHistoryFileName, JHFInputStreamCallback reader) throws Exception { InputStream downloadIs; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java index ae9df37..a33abeb 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java @@ -52,7 +52,6 @@ public class JHFCrawlerDriverImpl implements JHFCrawlerDriver { private Set<String> processedJobFileNames = new HashSet<>(); private final JobProcessDate processDate = new JobProcessDate(); - private boolean dryRun; private JHFInputStreamCallback reader; protected boolean zeroBasedMonth = true; @@ -65,10 +64,6 @@ public class JHFCrawlerDriverImpl implements JHFCrawlerDriver { public JHFCrawlerDriverImpl(JHFInputStreamCallback reader, JobHistoryLCM historyLCM, JobIdFilter jobFilter, int partitionId) throws Exception { this.zeroBasedMonth = MRHistoryJobConfig.get().getControlConfig().zeroBasedMonth; - this.dryRun = MRHistoryJobConfig.get().getControlConfig().dryRun; - if (this.dryRun) { - LOG.info("this is a dry run"); - } this.reader = reader; jhfLCM = historyLCM;//new JobHistoryDAOImpl(jobHistoryConfig); this.partitionId = partitionId; @@ -169,15 +164,15 @@ public class JHFCrawlerDriverImpl implements JHFCrawlerDriver { LOG.warn("illegal job history file name : " + jobHistoryFile); return -1; } - if (!dryRun) { - jhfLCM.readFileContent( - processDate.year, - getActualMonth(processDate.month), - processDate.day, - Integer.valueOf(serialNumber), - jobHistoryFile, + + jhfLCM.readFileContent( + processDate.year, + getActualMonth(processDate.month), + processDate.day, + Integer.valueOf(serialNumber), + jobHistoryFile, reader); - } + JobHistoryZKStateManager.instance().addProcessedJob(String.format(FORMAT_JOB_PROCESS_DATE, this.processDate.year, this.processDate.month + 1, http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java index a035357..07ca5c3 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java @@ -40,7 +40,7 @@ public class JobHistoryDAOImpl extends AbstractJobHistoryDAO { private FileSystem hdfs; public JobHistoryDAOImpl(JobHistoryEndpointConfig endpointConfig) throws Exception { - super(endpointConfig.basePath, endpointConfig.pathContainsJobTrackerName, endpointConfig.jobTrackerName); + super(endpointConfig.basePath); for (Map.Entry<String, String> entry : endpointConfig.hdfs.entrySet()) { this.conf.set(entry.getKey(), entry.getValue()); LOG.info("conf key {}, conf value {}", entry.getKey(), entry.getValue()); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml index 02e8c05..66111eb 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml @@ -28,73 +28,73 @@ <value>sandbox</value> </property> <property> - <name>jobExtractorConfig.mrVersion</name> - <value>MRVer2</value> + <name>workers</name> + <displayName>storm worker number</displayName> + <value>4</value> </property> <property> - <name>jobExtractorConfig.readTimeOutSeconds</name> - <displayName>zkPort</displayName> - <value>10</value> + <name>envContextConfig.parallelismConfig.mrHistoryJobExecutor</name> + <value>4</value> </property> <property> - <name>dataSourceConfig.zkQuorum</name> - <value>sandbox.hortonworks.com:2181</value> + <name>envContextConfig.tasks.mrHistoryJobExecutor</name> + <value>4</value> </property> <property> - <name>dataSourceConfig.zkPort</name> - <value>2181</value> + <name>jobExtractorConfig.mrVersion</name> + <value>MRVer2</value> </property> <property> - <name>dataSourceConfig.zkSessionTimeoutMs</name> - <value>15000</value> + <name>jobExtractorConfig.readTimeOutSeconds</name> + <value>60</value> </property> <property> - <name>dataSourceConfig.zkRetryTimes</name> - <value>3</value> + <name>zkStateConfig.zkQuorum</name> + <value>sandbox.hortonworks.com:2181</value> </property> <property> - <name>dataSourceConfig.zkRetryInterval</name> - <value>20000</value> + <name>zkStateConfig.zkPort</name> + <value>2181</value> </property> <property> - <name>dataSourceConfig.zkRoot</name> - <value>/test_mrjobhistory</value> + <name>zkStateConfig.zkRoot</name> + <value>/mrjobhistory_sandbox</value> </property> <property> - <name>dataSourceConfig.basePath</name> - <value>/mr-history/done</value> + <name>zkStateConfig.zkSessionTimeoutMs</name> + <value>15000</value> </property> <property> - <name>dataSourceConfig.jobTrackerName</name> - <value></value> + <name>zkStateConfig.zkRetryTimes</name> + <value>3</value> </property> <property> - <name>dataSourceConfig.nnEndpoint</name> - <value>hdfs://sandbox.hortonworks.com:8020</value> + <name>zkStateConfig.zkRetryInterval</name> + <value>20000</value> </property> <property> - <name>dataSourceConfig.pathContainsJobTrackerName</name> + <name>controlConfig.zeroBasedMonth</name> <value>false</value> </property> <property> - <name>dataSourceConfig.principal</name> - <value></value> + <name>controlConfig.partitionerCls</name> + <value>org.apache.eagle.jpm.util.DefaultJobIdPartitioner</value> </property> <property> - <name>dataSourceConfig.keytab</name> - <value></value> + <name>controlConfig.timeZone</name> + <value>Etc/GMT+7</value> </property> <property> - <name>dataSourceConfig.dryRun</name> - <value>false</value> + <name>endpointConfig.mrHistoryServerUrl</name> + <value>http://sandbox.hortonworks.com:19888</value> </property> <property> - <name>dataSourceConfig.partitionerCls</name> - <value>org.apache.eagle.jpm.util.DefaultJobIdPartitioner</value> + <name>endpointConfig.basePath</name> + <value>/mr-history/done</value> </property> <property> - <name>dataSourceConfig.zeroBasedMonth</name> - <value>false</value> + <name>endpointConfig.hdfs.fs.defaultFS</name> + <value>hdfs://sandbox.hortonworks.com:8020</value> </property> <property> <name>MRConfigureKeys.jobConfigKey</name> @@ -115,14 +115,6 @@ <value>eagle.job.name</value> </property> <property> - <name>envContextConfig.parallelismConfig.mrHistoryJobExecutor</name> - <value>6</value> - </property> - <property> - <name>envContextConfig.tasks.mrHistoryJobExecutor</name> - <value>6</value> - </property> - <property> <name>eagleProps.eagleService.host</name> <description>eagleProps.eagleService.host</description> <value>sandbox.hortonworks.com</value> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf index 1440227..fa53dfb 100644 --- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf @@ -40,7 +40,6 @@ "controlConfig" : { "zeroBasedMonth" : false, - "dryRun" : false, "partitionerCls" : "org.apache.eagle.jpm.util.DefaultJobIdPartitioner", "timeZone" : "UTC" }, @@ -48,8 +47,6 @@ "endpointConfig" : { "mrHistoryServerUrl" : "http://sandbox.hortonworks.com:19888", "basePath" : "/mr-history/done", - "pathContainsJobTrackerName" : false, - "jobTrackerName" : "", "hdfs" : { fs.defaultFS : "hdfs://sandbox.hortonworks.com:8020", #if not need, then do not set @@ -60,8 +57,6 @@ }, "eagleProps" : { - "mailHost" : "abc.com", - "mailDebug" : "true", "eagleService": { "host": "sandbox.hortonworks.com", "port": 9099, http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml index 51e9eb8..4b95b36 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml @@ -23,10 +23,30 @@ <configuration> <!-- org.apache.eagle.jpm.spark.running.SparkRunningJobAppConfig --> <property> - <name>envContextConfig.env</name> - <value>local</value> - <displayName>Environment</displayName> - <description>Execution environment</description> + <name>jobExtractorConfig.site</name> + <displayName>Site ID</displayName> + <value>sandbox</value> + </property> + <property> + <name>workers</name> + <displayName>storm worker number</displayName> + <value>4</value> + </property> + <property> + <name>envContextConfig.parallelismConfig.mrRunningJobFetchSpout</name> + <value>1</value> + </property> + <property> + <name>envContextConfig.parallelismConfig.mrRunningJobParseBolt</name> + <value>8</value> + </property> + <property> + <name>envContextConfig.tasks.mrRunningJobFetchSpout</name> + <value>1</value> + </property> + <property> + <name>envContextConfig.tasks.mrRunningJobParseBolt</name> + <value>8</value> </property> <property> <name>zookeeperConfig.zkQuorum</name> @@ -60,7 +80,7 @@ </property> <property> <name>zookeeperConfig.zkRoot</name> - <value>/apps/mr/running</value> + <value>/apps/mr/runningSandbox</value> </property> <property> <name>eagleProps.eagleService.host</name> @@ -85,26 +105,21 @@ <property> <name>eagleProps.eagleService.readTimeOutSeconds</name> <description>eagleProps.eagleService.readTimeOutSeconds</description> - <value>20</value> + <value>60</value> </property> <property> <name>eagleProps.eagleService.maxFlushNum</name> <description>eagleProps.eagleService.maxFlushNum</description> - <value>500</value> - </property> - <property> - <name>jobExtractorConfig.site</name> - <description>jobExtractorConfig.site</description> - <value>sandbox</value> + <value>1000</value> </property> <property> <name>jobExtractorConfig.fetchRunningJobInterval</name> <description>jobExtractorConfig.fetchRunningJobInterval</description> - <value>15</value> + <value>60</value> </property> <property> - <name>jobExtractorConfig.parseThreadPoolSize</name> - <description>jobExtractorConfig.parseThreadPoolSize</description> + <name>jobExtractorConfig.parseJobThreadPoolSize</name> + <description>jobExtractorConfig.parseJobThreadPoolSize</description> <value>5</value> </property> <property> @@ -115,27 +130,25 @@ <property> <name>dataSourceConfig.rmUrls</name> <description>dataSourceConfig.rmUrls</description> - <value>http://sandbox.hortonworks.com:50030</value> - </property> - <property> - <name>dataSourceConfig.nnEndpoint</name> - <description>dataSourceConfig.nnEndpoint</description> - <value>hdfs://sandbox.hortonworks.com:8020</value> - </property> - <property> - <name>dataSourceConfig.keytab</name> - <description>dataSourceConfig.keytab</description> - <value></value> + <value>http://sandbox.hortonworks.com:8088</value> </property> <property> - <name>dataSourceConfig.principal</name> - <description>dataSourceConfig.principal</description> - <value></value> + <name>MRConfigureKeys.jobConfigKey</name> + <value>mapreduce.map.output.compress, + mapreduce.map.output.compress.codec, + mapreduce.output.fileoutputformat.compress, + mapreduce.output.fileoutputformat.compress.type, + mapreduce.output.fileoutputformat.compress.codec, + mapred.output.format.class, + dataplatform.etl.info, + mapreduce.map.memory.mb, + mapreduce.reduce.memory.mb, + mapreduce.map.java.opts, + mapreduce.reduce.java.opts</value> </property> <property> - <name>dataSourceConfig.rmUrls</name> - <description>dataSourceConfig.rmUrls</description> - <value>http://sandbox.hortonworks.com:8088</value> + <name>MRConfigureKeys.jobNameKey</name> + <value>eagle.job.name</value> </property> </configuration> <docs> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf index 830c72b..9c354c7 100644 --- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf +++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf @@ -51,8 +51,6 @@ }, "eagleProps" : { - "mailHost" : "abc.com", - "mailDebug" : "true", "eagleService": { "host": "sandbox.hortonworks.com", "port": 9099, http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java index 253c61a..a2d97bf 100644 --- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java +++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java @@ -159,8 +159,8 @@ public class RunningJobManager implements Serializable { appInfo.put("startedTime", app.getStartedTime() + ""); appInfo.put("finishedTime", app.getFinishedTime() + ""); appInfo.put("elapsedTime", app.getElapsedTime() + ""); - appInfo.put("amContainerLogs", app.getAmContainerLogs()); - appInfo.put("amHostHttpAddress", app.getAmHostHttpAddress()); + appInfo.put("amContainerLogs", app.getAmContainerLogs() == null ? "" : app.getAmContainerLogs()); + appInfo.put("amHostHttpAddress", app.getAmHostHttpAddress() == null ? "" : app.getAmHostHttpAddress()); appInfo.put("allocatedMB", app.getAllocatedMB() + ""); appInfo.put("allocatedVCores", app.getAllocatedVCores() + ""); appInfo.put("runningContainers", app.getRunningContainers() + ""); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-topology-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-topology-assembly/pom.xml b/eagle-topology-assembly/pom.xml index 6a87a98..80ca265 100644 --- a/eagle-topology-assembly/pom.xml +++ b/eagle-topology-assembly/pom.xml @@ -177,4 +177,4 @@ </plugin> </plugins> </build> -</project> +</project> \ No newline at end of file