Repository: incubator-eagle
Updated Branches:
  refs/heads/master 97978c60d -> fa85dc3e2


[EAGLE-605] Fix MR feeder provider.xml and remove useless configure

https://issues.apache.org/jira/browse/EAGLE-605

1. add missing configure from application.conf to provider.xml
2. remove useless configure like : 
pathContainsJobTrackerName/jobTrackerName/dryRun
3. Fix bug: add IsExecutable to ApplicationDesc so that web ui will know some 
applications can not be started, just static

Author: wujinhu <wujinhu...@126.com>

Closes #490 from wujinhu/ui-test.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/fa85dc3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/fa85dc3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/fa85dc3e

Branch: refs/heads/master
Commit: fa85dc3e20f163ff84f39ca51d16c5d54fad0482
Parents: 97978c6
Author: wujinhu <wujinhu...@126.com>
Authored: Tue Oct 11 16:36:24 2016 +0800
Committer: Hao Chen <h...@apache.org>
Committed: Tue Oct 11 16:36:24 2016 +0800

----------------------------------------------------------------------
 .../service/ApplicationOperationContext.java    |  1 -
 .../impl/ApplicationManagementServiceImpl.java  | 22 ++++--
 .../impl/ApplicationProviderSPILoader.java      |  1 +
 .../eagle/metadata/model/ApplicationDesc.java   |  9 +++
 .../eagle/metadata/model/ApplicationEntity.java | 11 +--
 .../jpm/mr/history/MRHistoryJobConfig.java      |  6 --
 .../history/crawler/AbstractJobHistoryDAO.java  | 42 +----------
 .../history/crawler/JHFCrawlerDriverImpl.java   | 21 ++----
 .../mr/history/crawler/JobHistoryDAOImpl.java   |  2 +-
 ....history.MRHistoryJobApplicationProvider.xml | 76 +++++++++----------
 .../src/main/resources/application.conf         |  5 --
 ....running.MRRunningJobApplicationProvider.xml | 77 ++++++++++++--------
 .../src/main/resources/application.conf         |  2 -
 .../jpm/util/jobrecover/RunningJobManager.java  |  4 +-
 eagle-topology-assembly/pom.xml                 |  2 +-
 15 files changed, 117 insertions(+), 164 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
----------------------------------------------------------------------
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
index 33a407c..3561374 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
@@ -90,7 +90,6 @@ public class ApplicationOperationContext implements 
Serializable, ApplicationLif
 
     @Override
     public void onInstall() {
-        metadata.setExecutable(application.isExecutable());
         if (metadata.getDescriptor().getStreams() != null) {
             List<StreamDesc> streamDescToInstall = 
metadata.getDescriptor().getStreams().stream().map((streamDefinition -> {
                 StreamDefinition copied = streamDefinition.copy();

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
----------------------------------------------------------------------
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
index 04a7007..abd9197 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
@@ -21,6 +21,7 @@ import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.typesafe.config.Config;
 import org.apache.eagle.alert.metadata.IMetadataDao;
+import org.apache.eagle.app.Application;
 import org.apache.eagle.app.service.ApplicationManagementService;
 import org.apache.eagle.app.service.ApplicationOperations;
 import org.apache.eagle.app.service.ApplicationOperationContext;
@@ -141,12 +142,14 @@ public class ApplicationManagementServiceImpl implements 
ApplicationManagementSe
     @Override
     public ApplicationEntity start(ApplicationOperations.StartOperation 
operation) throws ApplicationWrongStatusException {
         ApplicationEntity applicationEntity = 
applicationEntityService.getByUUIDOrAppId(operation.getUuid(), 
operation.getAppId());
+        Application application = 
applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication();
+        Preconditions.checkArgument(application.isExecutable(), "Application 
is not executable");
+
         ApplicationEntity.Status currentStatus = applicationEntity.getStatus();
         try {
             if (currentStatus == ApplicationEntity.Status.INITIALIZED || 
currentStatus == ApplicationEntity.Status.STOPPED) {
                 ApplicationOperationContext applicationOperationContext = new 
ApplicationOperationContext(
-                    
applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(),
-                    applicationEntity, config, alertMetadataService);
+                        application, applicationEntity, config, 
alertMetadataService);
 
                 applicationOperationContext.onStart();
                 //Only when topology submitted successfully can the state 
change to STARTING
@@ -172,9 +175,11 @@ public class ApplicationManagementServiceImpl implements 
ApplicationManagementSe
     @Override
     public ApplicationEntity stop(ApplicationOperations.StopOperation 
operation) throws ApplicationWrongStatusException {
         ApplicationEntity applicationEntity = 
applicationEntityService.getByUUIDOrAppId(operation.getUuid(), 
operation.getAppId());
+        Application application = 
applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication();
+        Preconditions.checkArgument(application.isExecutable(), "Application 
is not executable");
+
         ApplicationOperationContext applicationOperationContext = new 
ApplicationOperationContext(
-            
applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(),
-            applicationEntity, config, alertMetadataService);
+                application, applicationEntity, config, alertMetadataService);
         ApplicationEntity.Status currentStatus = applicationEntity.getStatus();
         try {
             if (currentStatus == ApplicationEntity.Status.RUNNING) {
@@ -200,12 +205,13 @@ public class ApplicationManagementServiceImpl implements 
ApplicationManagementSe
 
     @Override
     public ApplicationEntity.Status 
getStatus(ApplicationOperations.CheckStatusOperation operation)  {
-        ApplicationEntity applicationEntity = null;
         try {
-            applicationEntity = 
applicationEntityService.getByUUIDOrAppId(operation.getUuid(), 
operation.getAppId());
+            ApplicationEntity applicationEntity = 
applicationEntityService.getByUUIDOrAppId(operation.getUuid(), 
operation.getAppId());
+            Application application = 
applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication();
+            Preconditions.checkArgument(application.isExecutable(), 
"Application is not executable");
+
             ApplicationOperationContext applicationOperationContext = new 
ApplicationOperationContext(
-                
applicationProviderService.getApplicationProviderByType(applicationEntity.getDescriptor().getType()).getApplication(),
-                applicationEntity, config, alertMetadataService);
+                    application, applicationEntity, config, 
alertMetadataService);
             ApplicationEntity.Status topologyStatus = 
applicationOperationContext.getStatus();
             return topologyStatus;
         } catch (IllegalArgumentException e) {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java
----------------------------------------------------------------------
diff --git 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java
 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java
index bd96e28..9f52c9c 100644
--- 
a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java
+++ 
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationProviderSPILoader.java
@@ -80,6 +80,7 @@ public class ApplicationProviderSPILoader extends 
ApplicationProviderLoader {
                 ApplicationProviderConfig providerConfig = new 
ApplicationProviderConfig();
                 
providerConfig.setClassName(applicationProvider.getClass().getCanonicalName());
                 
providerConfig.setJarPath(jarFileSupplier.apply(applicationProvider));
+                
applicationProvider.getApplicationDesc().setExecutable(applicationProvider.getApplication().isExecutable());
                 applicationProvider.prepare(providerConfig, getConfig());
                 registerProvider(applicationProvider);
                 LOG.warn("Loaded {}:{} ({}) from {}",

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
----------------------------------------------------------------------
diff --git 
a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
 
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
index 506bdc6..8bc6ee3 100644
--- 
a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
+++ 
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
@@ -36,6 +36,15 @@ public class ApplicationDesc implements Serializable {
     private Configuration configuration;
     private List<StreamDefinition> streams;
     private ApplicationDocs docs;
+    private boolean executable;
+
+    public boolean isExecutable() {
+        return executable;
+    }
+
+    public void setExecutable(boolean executable) {
+        this.executable = executable;
+    }
 
     private List<ApplicationDependency> dependencies;
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
----------------------------------------------------------------------
diff --git 
a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
 
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
index bae6ba7..5297de1 100644
--- 
a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
+++ 
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
@@ -33,7 +33,6 @@ public class ApplicationEntity extends PersistenceEntity {
     private String appId;
     private SiteEntity site;
     private ApplicationDesc descriptor;
-    private boolean executable = true;
 
     private Map<String, Object> configuration = new HashMap<>();
     private Map<String, String> context = new HashMap<>();
@@ -153,15 +152,7 @@ public class ApplicationEntity extends PersistenceEntity {
         this.streams = streams;
     }
 
-    public boolean isExecutable() {
-        return executable;
-    }
-
-    public void setExecutable(boolean executable) {
-        this.executable = executable;
-    }
-
-    public static enum Status {
+    public enum Status {
         INITIALIZED("INITIALIZED"),
         STARTING("STARTING"),
         RUNNING("RUNNING"),

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
----------------------------------------------------------------------
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
index 839d2e4..37df5ad 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobConfig.java
@@ -85,13 +85,10 @@ public class MRHistoryJobConfig implements Serializable {
     public static class JobHistoryEndpointConfig implements Serializable {
         public String mrHistoryServerUrl;
         public String basePath;
-        public boolean pathContainsJobTrackerName;
-        public String jobTrackerName;
         public Map<String, String> hdfs;
     }
 
     public static class ControlConfig implements Serializable {
-        public boolean dryRun;
         public Class<? extends JobIdPartitioner> partitionerCls;
         public boolean zeroBasedMonth;
         public String timeZone;
@@ -158,14 +155,11 @@ public class MRHistoryJobConfig implements Serializable {
 
         //parse job history endpoint
         this.jobHistoryEndpointConfig.basePath = 
config.getString("endpointConfig.basePath");
-        this.jobHistoryEndpointConfig.jobTrackerName = 
config.getString("endpointConfig.jobTrackerName");
         this.jobHistoryEndpointConfig.mrHistoryServerUrl = 
config.getString("endpointConfig.mrHistoryServerUrl");
-        this.jobHistoryEndpointConfig.pathContainsJobTrackerName = 
config.getBoolean("endpointConfig.pathContainsJobTrackerName");
         for (Map.Entry<String, ConfigValue> entry : 
config.getConfig("endpointConfig.hdfs").entrySet()) {
             this.jobHistoryEndpointConfig.hdfs.put(entry.getKey(), 
entry.getValue().unwrapped().toString());
         }
         //parse control config
-        this.controlConfig.dryRun = config.getBoolean("controlConfig.dryRun");
         try {
             this.controlConfig.partitionerCls = (Class<? extends 
JobIdPartitioner>) 
Class.forName(config.getString("controlConfig.partitionerCls"));
             assert this.controlConfig.partitionerCls != null;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java
----------------------------------------------------------------------
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java
index 74489cd..9ed2ed2 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/AbstractJobHistoryDAO.java
@@ -45,35 +45,19 @@ public abstract class AbstractJobHistoryDAO implements 
JobHistoryLCM {
     protected static final Pattern JOBID_PATTERN = 
Pattern.compile("job_\\d+_\\d+");
 
     protected final String basePath;
-    protected volatile String jobTrackerName;
 
     public  static final String JOB_CONF_POSTFIX = "_conf.xml";
 
     private static final Timer timer = new Timer(true);
     private static final long JOB_TRACKER_SYNC_DURATION = 10 * 60 * 1000; // 
10 minutes
 
-    private boolean pathContainsJobTrackerName;
-
-    public AbstractJobHistoryDAO(String basePath, boolean 
pathContainsJobTrackerName, String startingJobTrackerName) throws Exception {
+    public AbstractJobHistoryDAO(String basePath) throws Exception {
         this.basePath = basePath;
-        this.pathContainsJobTrackerName = pathContainsJobTrackerName;
-        jobTrackerName = startingJobTrackerName;
-        if (this.pathContainsJobTrackerName) {
-            if (startingJobTrackerName == null || 
startingJobTrackerName.isEmpty()) {
-                throw new IllegalStateException("startingJobTrackerName should 
not be null or empty");
-            }
-            // start background thread to check what is current job tracker
-            startThread(this.basePath);
-        }
     }
 
     protected String buildWholePathToYearMonthDay(int year, int month, int 
day) {
         StringBuilder sb = new StringBuilder();
         sb.append(basePath);
-        if (!pathContainsJobTrackerName && jobTrackerName != null && 
!jobTrackerName.isEmpty()) {
-            sb.append("/");
-            sb.append(jobTrackerName);
-        }
         sb.append(String.format(YEAR_MONTH_DAY_URL_FORMAT, year, month, day));
         return sb.toString();
     }
@@ -110,30 +94,6 @@ public abstract class AbstractJobHistoryDAO implements 
JobHistoryLCM {
         return null;
     }
 
-    private void startThread(final String basePath) throws Exception {
-        LOG.info("start an every-" + JOB_TRACKER_SYNC_DURATION / (60 * 1000) + 
"min timer task to check current jobTrackerName in background");
-        // Automatically update current job tracker name in background every 
30 minutes
-        timer.scheduleAtFixedRate(new TimerTask() {
-            @Override
-            public void run() {
-                try {
-                    LOG.info("regularly checking current jobTrackerName in 
background");
-                    final String _jobTrackerName = 
calculateJobTrackerName(basePath);
-                    if (_jobTrackerName != null && 
!_jobTrackerName.equals(jobTrackerName)) {
-                        LOG.info("jobTrackerName changed from " + 
jobTrackerName + " to " + _jobTrackerName);
-                        jobTrackerName = _jobTrackerName;
-                    }
-                    LOG.info("Current jobTrackerName is: " + jobTrackerName);
-                } catch (Exception e) {
-                    LOG.error("failed to figure out current job tracker name 
that is not configured due to: " + e.getMessage(), e);
-                } catch (Throwable t) {
-                    LOG.error("failed to figure out current job tracker name 
that is not configured due to: " + t.getMessage(), t);
-                }
-            }
-        }, JOB_TRACKER_SYNC_DURATION, JOB_TRACKER_SYNC_DURATION);
-    }
-
-
     @Override
     public void readFileContent(int year, int month, int day, int 
serialNumber, String jobHistoryFileName, JHFInputStreamCallback reader) throws 
Exception {
         InputStream downloadIs;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
----------------------------------------------------------------------
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
index ae9df37..a33abeb 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JHFCrawlerDriverImpl.java
@@ -52,7 +52,6 @@ public class JHFCrawlerDriverImpl implements JHFCrawlerDriver 
{
     private Set<String> processedJobFileNames = new HashSet<>();
 
     private final JobProcessDate processDate = new JobProcessDate();
-    private boolean dryRun;
     private JHFInputStreamCallback reader;
     protected boolean zeroBasedMonth = true;
 
@@ -65,10 +64,6 @@ public class JHFCrawlerDriverImpl implements 
JHFCrawlerDriver {
     public JHFCrawlerDriverImpl(JHFInputStreamCallback reader,
                                 JobHistoryLCM historyLCM, JobIdFilter 
jobFilter, int partitionId) throws Exception {
         this.zeroBasedMonth = 
MRHistoryJobConfig.get().getControlConfig().zeroBasedMonth;
-        this.dryRun = MRHistoryJobConfig.get().getControlConfig().dryRun;
-        if (this.dryRun)  {
-            LOG.info("this is a dry run");
-        }
         this.reader = reader;
         jhfLCM = historyLCM;//new JobHistoryDAOImpl(jobHistoryConfig);
         this.partitionId = partitionId;
@@ -169,15 +164,15 @@ public class JHFCrawlerDriverImpl implements 
JHFCrawlerDriver {
             LOG.warn("illegal job history file name : " + jobHistoryFile);
             return -1;
         }
-        if (!dryRun) {
-            jhfLCM.readFileContent(
-                    processDate.year,
-                    getActualMonth(processDate.month),
-                    processDate.day,
-                    Integer.valueOf(serialNumber),
-                    jobHistoryFile,
+
+        jhfLCM.readFileContent(
+                processDate.year,
+                getActualMonth(processDate.month),
+                processDate.day,
+                Integer.valueOf(serialNumber),
+                jobHistoryFile,
                 reader);
-        }
+
         
JobHistoryZKStateManager.instance().addProcessedJob(String.format(FORMAT_JOB_PROCESS_DATE,
                 this.processDate.year,
                 this.processDate.month + 1,

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java
----------------------------------------------------------------------
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java
index a035357..07ca5c3 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/crawler/JobHistoryDAOImpl.java
@@ -40,7 +40,7 @@ public class JobHistoryDAOImpl extends AbstractJobHistoryDAO {
     private FileSystem hdfs;
 
     public JobHistoryDAOImpl(JobHistoryEndpointConfig endpointConfig) throws 
Exception {
-        super(endpointConfig.basePath, 
endpointConfig.pathContainsJobTrackerName, endpointConfig.jobTrackerName);
+        super(endpointConfig.basePath);
         for (Map.Entry<String, String> entry : endpointConfig.hdfs.entrySet()) 
{
             this.conf.set(entry.getKey(), entry.getValue());
             LOG.info("conf key {}, conf value {}", entry.getKey(), 
entry.getValue());

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
----------------------------------------------------------------------
diff --git 
a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
 
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
index 02e8c05..66111eb 100644
--- 
a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
+++ 
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.history.MRHistoryJobApplicationProvider.xml
@@ -28,73 +28,73 @@
             <value>sandbox</value>
         </property>
         <property>
-            <name>jobExtractorConfig.mrVersion</name>
-            <value>MRVer2</value>
+            <name>workers</name>
+            <displayName>storm worker number</displayName>
+            <value>4</value>
         </property>
         <property>
-            <name>jobExtractorConfig.readTimeOutSeconds</name>
-            <displayName>zkPort</displayName>
-            <value>10</value>
+            
<name>envContextConfig.parallelismConfig.mrHistoryJobExecutor</name>
+            <value>4</value>
         </property>
         <property>
-            <name>dataSourceConfig.zkQuorum</name>
-            <value>sandbox.hortonworks.com:2181</value>
+            <name>envContextConfig.tasks.mrHistoryJobExecutor</name>
+            <value>4</value>
         </property>
         <property>
-            <name>dataSourceConfig.zkPort</name>
-            <value>2181</value>
+            <name>jobExtractorConfig.mrVersion</name>
+            <value>MRVer2</value>
         </property>
         <property>
-            <name>dataSourceConfig.zkSessionTimeoutMs</name>
-            <value>15000</value>
+            <name>jobExtractorConfig.readTimeOutSeconds</name>
+            <value>60</value>
         </property>
         <property>
-            <name>dataSourceConfig.zkRetryTimes</name>
-            <value>3</value>
+            <name>zkStateConfig.zkQuorum</name>
+            <value>sandbox.hortonworks.com:2181</value>
         </property>
         <property>
-            <name>dataSourceConfig.zkRetryInterval</name>
-            <value>20000</value>
+            <name>zkStateConfig.zkPort</name>
+            <value>2181</value>
         </property>
         <property>
-            <name>dataSourceConfig.zkRoot</name>
-            <value>/test_mrjobhistory</value>
+            <name>zkStateConfig.zkRoot</name>
+            <value>/mrjobhistory_sandbox</value>
         </property>
         <property>
-            <name>dataSourceConfig.basePath</name>
-            <value>/mr-history/done</value>
+            <name>zkStateConfig.zkSessionTimeoutMs</name>
+            <value>15000</value>
         </property>
         <property>
-            <name>dataSourceConfig.jobTrackerName</name>
-            <value></value>
+            <name>zkStateConfig.zkRetryTimes</name>
+            <value>3</value>
         </property>
         <property>
-            <name>dataSourceConfig.nnEndpoint</name>
-            <value>hdfs://sandbox.hortonworks.com:8020</value>
+            <name>zkStateConfig.zkRetryInterval</name>
+            <value>20000</value>
         </property>
         <property>
-            <name>dataSourceConfig.pathContainsJobTrackerName</name>
+            <name>controlConfig.zeroBasedMonth</name>
             <value>false</value>
         </property>
         <property>
-            <name>dataSourceConfig.principal</name>
-            <value></value>
+            <name>controlConfig.partitionerCls</name>
+            <value>org.apache.eagle.jpm.util.DefaultJobIdPartitioner</value>
         </property>
         <property>
-            <name>dataSourceConfig.keytab</name>
-            <value></value>
+            <name>controlConfig.timeZone</name>
+            <value>Etc/GMT+7</value>
         </property>
         <property>
-            <name>dataSourceConfig.dryRun</name>
-            <value>false</value>
+            <name>endpointConfig.mrHistoryServerUrl</name>
+            <value>http://sandbox.hortonworks.com:19888</value>
         </property>
         <property>
-            <name>dataSourceConfig.partitionerCls</name>
-            <value>org.apache.eagle.jpm.util.DefaultJobIdPartitioner</value>
+            <name>endpointConfig.basePath</name>
+            <value>/mr-history/done</value>
         </property>
         <property>
-            <name>dataSourceConfig.zeroBasedMonth</name>
-            <value>false</value>
+            <name>endpointConfig.hdfs.fs.defaultFS</name>
+            <value>hdfs://sandbox.hortonworks.com:8020</value>
         </property>
         <property>
             <name>MRConfigureKeys.jobConfigKey</name>
@@ -115,14 +115,6 @@
             <value>eagle.job.name</value>
         </property>
         <property>
-            
<name>envContextConfig.parallelismConfig.mrHistoryJobExecutor</name>
-            <value>6</value>
-        </property>
-        <property>
-            <name>envContextConfig.tasks.mrHistoryJobExecutor</name>
-            <value>6</value>
-        </property>
-        <property>
             <name>eagleProps.eagleService.host</name>
             <description>eagleProps.eagleService.host</description>
             <value>sandbox.hortonworks.com</value>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf 
b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf
index 1440227..fa53dfb 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/application.conf
@@ -40,7 +40,6 @@
 
   "controlConfig" : {
     "zeroBasedMonth" : false,
-    "dryRun" : false,
     "partitionerCls" : "org.apache.eagle.jpm.util.DefaultJobIdPartitioner",
     "timeZone" : "UTC"
   },
@@ -48,8 +47,6 @@
   "endpointConfig" : {
     "mrHistoryServerUrl" : "http://sandbox.hortonworks.com:19888";,
     "basePath" : "/mr-history/done",
-    "pathContainsJobTrackerName" : false,
-    "jobTrackerName" : "",
     "hdfs" : {
       fs.defaultFS : "hdfs://sandbox.hortonworks.com:8020",
       #if not need, then do not set
@@ -60,8 +57,6 @@
   },
 
   "eagleProps" : {
-    "mailHost" : "abc.com",
-    "mailDebug" : "true",
     "eagleService": {
       "host": "sandbox.hortonworks.com",
       "port": 9099,

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
----------------------------------------------------------------------
diff --git 
a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
 
b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
index 51e9eb8..4b95b36 100644
--- 
a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
+++ 
b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
@@ -23,10 +23,30 @@
     <configuration>
         <!-- org.apache.eagle.jpm.spark.running.SparkRunningJobAppConfig -->
         <property>
-            <name>envContextConfig.env</name>
-            <value>local</value>
-            <displayName>Environment</displayName>
-            <description>Execution environment</description>
+            <name>jobExtractorConfig.site</name>
+            <displayName>Site ID</displayName>
+            <value>sandbox</value>
+        </property>
+        <property>
+            <name>workers</name>
+            <displayName>storm worker number</displayName>
+            <value>4</value>
+        </property>
+        <property>
+            
<name>envContextConfig.parallelismConfig.mrRunningJobFetchSpout</name>
+            <value>1</value>
+        </property>
+        <property>
+            
<name>envContextConfig.parallelismConfig.mrRunningJobParseBolt</name>
+            <value>8</value>
+        </property>
+        <property>
+            <name>envContextConfig.tasks.mrRunningJobFetchSpout</name>
+            <value>1</value>
+        </property>
+        <property>
+            <name>envContextConfig.tasks.mrRunningJobParseBolt</name>
+            <value>8</value>
         </property>
         <property>
             <name>zookeeperConfig.zkQuorum</name>
@@ -60,7 +80,7 @@
         </property>
         <property>
             <name>zookeeperConfig.zkRoot</name>
-            <value>/apps/mr/running</value>
+            <value>/apps/mr/runningSandbox</value>
         </property>
         <property>
             <name>eagleProps.eagleService.host</name>
@@ -85,26 +105,21 @@
         <property>
             <name>eagleProps.eagleService.readTimeOutSeconds</name>
             
<description>eagleProps.eagleService.readTimeOutSeconds</description>
-            <value>20</value>
+            <value>60</value>
         </property>
         <property>
             <name>eagleProps.eagleService.maxFlushNum</name>
             <description>eagleProps.eagleService.maxFlushNum</description>
-            <value>500</value>
-        </property>
-        <property>
-            <name>jobExtractorConfig.site</name>
-            <description>jobExtractorConfig.site</description>
-            <value>sandbox</value>
+            <value>1000</value>
         </property>
         <property>
             <name>jobExtractorConfig.fetchRunningJobInterval</name>
             
<description>jobExtractorConfig.fetchRunningJobInterval</description>
-            <value>15</value>
+            <value>60</value>
         </property>
         <property>
-            <name>jobExtractorConfig.parseThreadPoolSize</name>
-            <description>jobExtractorConfig.parseThreadPoolSize</description>
+            <name>jobExtractorConfig.parseJobThreadPoolSize</name>
+            
<description>jobExtractorConfig.parseJobThreadPoolSize</description>
             <value>5</value>
         </property>
         <property>
@@ -115,27 +130,25 @@
         <property>
             <name>dataSourceConfig.rmUrls</name>
             <description>dataSourceConfig.rmUrls</description>
-            <value>http://sandbox.hortonworks.com:50030</value>
-        </property>
-        <property>
-            <name>dataSourceConfig.nnEndpoint</name>
-            <description>dataSourceConfig.nnEndpoint</description>
-            <value>hdfs://sandbox.hortonworks.com:8020</value>
-        </property>
-        <property>
-            <name>dataSourceConfig.keytab</name>
-            <description>dataSourceConfig.keytab</description>
-            <value></value>
+            <value>http://sandbox.hortonworks.com:8088</value>
         </property>
         <property>
-            <name>dataSourceConfig.principal</name>
-            <description>dataSourceConfig.principal</description>
-            <value></value>
+            <name>MRConfigureKeys.jobConfigKey</name>
+            <value>mapreduce.map.output.compress,
+                mapreduce.map.output.compress.codec,
+                mapreduce.output.fileoutputformat.compress,
+                mapreduce.output.fileoutputformat.compress.type,
+                mapreduce.output.fileoutputformat.compress.codec,
+                mapred.output.format.class,
+                dataplatform.etl.info,
+                mapreduce.map.memory.mb,
+                mapreduce.reduce.memory.mb,
+                mapreduce.map.java.opts,
+                mapreduce.reduce.java.opts</value>
         </property>
         <property>
-            <name>dataSourceConfig.rmUrls</name>
-            <description>dataSourceConfig.rmUrls</description>
-            <value>http://sandbox.hortonworks.com:8088</value>
+            <name>MRConfigureKeys.jobNameKey</name>
+            <value>eagle.job.name</value>
         </property>
     </configuration>
     <docs>

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf 
b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf
index 830c72b..9c354c7 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/application.conf
@@ -51,8 +51,6 @@
   },
 
   "eagleProps" : {
-    "mailHost" : "abc.com",
-    "mailDebug" : "true",
     "eagleService": {
       "host": "sandbox.hortonworks.com",
       "port": 9099,

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java
----------------------------------------------------------------------
diff --git 
a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java
 
b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java
index 253c61a..a2d97bf 100644
--- 
a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java
+++ 
b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobrecover/RunningJobManager.java
@@ -159,8 +159,8 @@ public class RunningJobManager implements Serializable {
         appInfo.put("startedTime", app.getStartedTime() + "");
         appInfo.put("finishedTime", app.getFinishedTime() + "");
         appInfo.put("elapsedTime", app.getElapsedTime() + "");
-        appInfo.put("amContainerLogs", app.getAmContainerLogs());
-        appInfo.put("amHostHttpAddress", app.getAmHostHttpAddress());
+        appInfo.put("amContainerLogs", app.getAmContainerLogs() == null ? "" : 
app.getAmContainerLogs());
+        appInfo.put("amHostHttpAddress", app.getAmHostHttpAddress() == null ? 
"" : app.getAmHostHttpAddress());
         appInfo.put("allocatedMB", app.getAllocatedMB() + "");
         appInfo.put("allocatedVCores", app.getAllocatedVCores() + "");
         appInfo.put("runningContainers", app.getRunningContainers() + "");

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/fa85dc3e/eagle-topology-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/pom.xml b/eagle-topology-assembly/pom.xml
index 6a87a98..80ca265 100644
--- a/eagle-topology-assembly/pom.xml
+++ b/eagle-topology-assembly/pom.xml
@@ -177,4 +177,4 @@
             </plugin>
         </plugins>
     </build>
-</project>
+</project>
\ No newline at end of file


Reply via email to