Repository: tez Updated Branches: refs/heads/master 6e15b2f6d -> a44f5c582
TEZ-2210. Record DAG AM CPU usage stats (bikas) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b7b1b69f Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b7b1b69f Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b7b1b69f Branch: refs/heads/master Commit: b7b1b69fd7b5c4883d8ee011496a731f3d0de395 Parents: 6e15b2f Author: Bikas Saha <[email protected]> Authored: Sat Mar 21 00:55:32 2015 -0700 Committer: Bikas Saha <[email protected]> Committed: Sat Mar 21 00:55:32 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/tez/common/counters/DAGCounter.java | 4 +- .../org/apache/tez/common/AsyncDispatcher.java | 6 +- .../org/apache/tez/common/TezUtilsInternal.java | 4 -- .../java/org/apache/tez/dag/app/AppContext.java | 4 ++ .../org/apache/tez/dag/app/DAGAppMaster.java | 49 ++++++++++++- .../apache/tez/dag/app/dag/impl/DAGImpl.java | 72 +++++++------------- .../dag/history/events/DAGFinishedEvent.java | 1 - .../tez/dag/app/TestMockDAGAppMaster.java | 2 + .../tez/util/TezMxBeanResourceCalculator.java | 1 - .../tez/runtime/metrics/GcTimeUpdater.java | 70 ------------------- .../tez/runtime/metrics/TaskCounterUpdater.java | 1 + 12 files changed, 87 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 6045975..a33fe2b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ Release 0.7.0: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2210. Record DAG AM CPU usage stats TEZ-2203. Intern strings in tez counters TEZ-2202. Fix LocalTaskExecutionThread ID to the standard thread numbering. TEZ-2059. Remove TaskEventHandler in TestDAGImpl http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java index b5bdffb..5064c35 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java @@ -36,5 +36,7 @@ public enum DAGCounter { FALLOW_SLOTS_MILLIS_TASKS, TOTAL_LAUNCHED_UBERTASKS, NUM_UBER_SUBTASKS, - NUM_FAILED_UBERTASKS + NUM_FAILED_UBERTASKS, + AM_CPU_MILLISECONDS, + AM_GC_TIME_MILLIS } http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java ---------------------------------------------------------------------- diff --git a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java index 253db23..fad0453 100644 --- a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java +++ b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java @@ -26,8 +26,7 @@ import java.util.concurrent.LinkedBlockingQueue; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.ShutdownHookManager; @@ -45,8 +44,7 @@ import com.google.common.collect.Maps; * class and a thread pool can be used to dispatch the events. */ @SuppressWarnings({"rawtypes", "unchecked"}) -@Public -@Evolving +@Private public class AsyncDispatcher extends CompositeService implements Dispatcher { private static final Log LOG = LogFactory.getLog(AsyncDispatcher.class); http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java ---------------------------------------------------------------------- diff --git a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java index 8d54ead..74d54ff 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java @@ -26,7 +26,6 @@ import java.nio.charset.Charset; import java.util.BitSet; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.DataFormatException; @@ -36,13 +35,11 @@ import java.util.zip.Inflater; import com.google.protobuf.ByteString; import com.google.protobuf.Descriptors; import com.google.protobuf.TextFormat; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.log4j.Appender; import org.apache.log4j.Logger; import org.apache.tez.dag.api.DagTypeConverters; @@ -57,7 +54,6 @@ import com.google.common.base.Stopwatch; public class TezUtilsInternal { private static final Log LOG = LogFactory.getLog(TezUtilsInternal.class); - private static final Random RANDOM = new Random(); public static void addUserSpecifiedTezConfiguration(String baseDir, Configuration conf) throws IOException { http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java index 5564809..4781784 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java @@ -53,6 +53,10 @@ public interface AppContext { ApplicationId getApplicationID(); TezDAGID getCurrentDAGID(); + + long getCumulativeCPUTime(); + + long getCumulativeGCTime(); ApplicationAttemptId getApplicationAttemptId(); http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 67efefc..38a3692 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -86,8 +86,10 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import org.apache.hadoop.yarn.util.SystemClock; import org.apache.tez.common.AsyncDispatcher; +import org.apache.tez.common.GcTimeUpdater; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezConverterUtils; import org.apache.tez.common.TezUtilsInternal; @@ -283,6 +285,9 @@ public class DAGAppMaster extends AbstractService { private String clientVersion; private boolean versionMismatch = false; private String versionMismatchDiagnostics; + + private ResourceCalculatorProcessTree cpuPlugin; + private GcTimeUpdater gcPlugin; // must be LinkedHashMap to preserve order of service addition Map<Service, ServiceWithDependency> services = @@ -319,11 +324,44 @@ public class DAGAppMaster extends AbstractService { LOG.info("Created DAGAppMaster for application " + applicationAttemptId + ", versionInfo=" + dagVersionInfo.toString()); } + + private void initResourceCalculatorPlugins() { + Class<? extends ResourceCalculatorProcessTree> clazz = amConf.getClass( + TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, null, + ResourceCalculatorProcessTree.class); + + // this is set by YARN NM + String pid = System.getenv().get("JVM_PID"); + // for the local debug test cases fallback to JVM hooks that are not portable + if (pid == null || pid.length() == 0) { + String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName(); + pid = processName.split("@")[0]; + } + cpuPlugin = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pid, clazz, amConf); + + gcPlugin = new GcTimeUpdater(null); + } + + private long getAMCPUTime() { + if (cpuPlugin != null) { + cpuPlugin.updateProcessTree(); + return cpuPlugin.getCumulativeCpuTime(); + } + return 0; + } + + private long getAMGCTime() { + if (gcPlugin != null) { + return gcPlugin.getCumulativaGcTime(); + } + return 0; + } @Override public synchronized void serviceInit(final Configuration conf) throws Exception { this.amConf = conf; + initResourceCalculatorPlugins(); this.isLocal = conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT); @@ -1377,6 +1415,16 @@ public class DAGAppMaster extends AbstractService { wLock.unlock(); } } + + @Override + public long getCumulativeCPUTime() { + return getAMCPUTime(); + } + + @Override + public long getCumulativeGCTime() { + return getAMGCTime(); + } } private static class ServiceWithDependency implements ServiceStateChangeListener { @@ -2007,7 +2055,6 @@ public class DAGAppMaster extends AbstractService { private void startDAGExecution(DAG dag, final Map<String, LocalResource> additionalAmResources) throws TezException { currentDAG = dag; - // Try localizing the actual resources. List<URL> additionalUrlsForClasspath; try { http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java index d1dcfb1..c469013 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.Clock; import org.apache.tez.common.ATSConstants; import org.apache.tez.common.ReflectionUtils; +import org.apache.tez.common.counters.DAGCounter; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.DagTypeConverters; import org.apache.tez.dag.api.EdgeProperty; @@ -64,7 +65,6 @@ import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.TezUncheckedException; import org.apache.tez.dag.api.VertexLocationHint; -import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.DAGStatusBuilder; import org.apache.tez.dag.api.client.ProgressBuilder; import org.apache.tez.dag.api.client.StatusGetOpts; @@ -179,6 +179,9 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, private final DAGPlan jobPlan; Map<String, LocalResource> localResources; + + long startDAGCpuTime = 0; + long startDAGGCTime = 0; private final List<String> diagnostics = new ArrayList<String>(); @@ -460,7 +463,10 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, this.aclManager = new ACLManager(appContext.getAMACLManager(), dagUGI.getShortUserName(), this.dagConf); - + // this is only for recovery in case it does not call the init transition + this.startDAGCpuTime = appContext.getCumulativeCPUTime(); + this.startDAGGCTime = appContext.getCumulativeGCTime(); + this.taskSpecificLaunchCmdOption = new TaskSpecificLaunchCmdOption(dagConf); // This "this leak" is okay because the retained pointer is in an // instance variable. @@ -605,6 +611,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, return fullCounters; } + // dag not yet finished. update cpu time counters + updateCpuCounters(); TezCounters counters = new TezCounters(); counters.incrAllCounters(dagCounters); return incrTaskCounters(counters, vertices.values()); @@ -1155,27 +1163,16 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, return dag.getInternalState(); } - private synchronized DAGState finished(DAGState finalState) { - // TODO Metrics - /* - if (getInternalState() == DAGState.RUNNING) { - metrics.endRunningJob(this); - } - */ - // TODO Metrics - /* - switch (finalState) { - case KILLED: - metrics.killedJob(this); - break; - case FAILED: - metrics.failedJob(this); - break; - case SUCCEEDED: - metrics.completedJob(this); - } - */ - + private void updateCpuCounters() { + long stopDAGCpuTime = appContext.getCumulativeCPUTime(); + long totalDAGCpuTime = stopDAGCpuTime - startDAGCpuTime; + long stopDAGGCTime = appContext.getCumulativeGCTime(); + long totalDAGGCTime = stopDAGGCTime - startDAGGCTime; + dagCounters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).setValue(totalDAGCpuTime); + dagCounters.findCounter(DAGCounter.AM_GC_TIME_MILLIS).setValue(totalDAGGCTime); + } + + private DAGState finished(DAGState finalState) { if (finishTime == 0) { setFinishTime(); } @@ -1188,6 +1185,10 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, } boolean recoveryError = false; + + // update cpu time counters before finishing the dag + updateCpuCounters(); + try { if (finalState == DAGState.SUCCEEDED) { logJobHistoryFinishedEvent(); @@ -1211,29 +1212,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, return finalState; } - private DAGStatus.State getDAGStatusFromState(DAGState finalState) { - switch (finalState) { - case NEW: - return DAGStatus.State.INITING; - case INITED: - return DAGStatus.State.INITING; - case RUNNING: - return DAGStatus.State.RUNNING; - case SUCCEEDED: - return DAGStatus.State.SUCCEEDED; - case FAILED: - return DAGStatus.State.FAILED; - case KILLED: - return DAGStatus.State.KILLED; - case ERROR: - return DAGStatus.State.ERROR; - case TERMINATING: - return DAGStatus.State.KILLED; - default: - throw new TezUncheckedException("Unknown DAGState: " + finalState); - } - } - @Override public String getUserName() { return userName; @@ -1596,6 +1574,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, // TODO Metrics //dag.metrics.submittedJob(dag); //dag.metrics.preparingJob(dag); + dag.startDAGCpuTime = dag.appContext.getCumulativeCPUTime(); + dag.startDAGGCTime = dag.appContext.getCumulativeGCTime(); DAGState state = dag.initializeDAG(); if (state != DAGState.INITED) { http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java index e05f043..8d8af0d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java @@ -51,7 +51,6 @@ public class DAGFinishedEvent implements HistoryEvent, SummaryEvent { private TezCounters tezCounters; private String user; private String dagName; - Map<String, Integer> dagTaskStats; public DAGFinishedEvent() { http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java ---------------------------------------------------------------------- diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java index 822fe7f..1150b10 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.URL; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.tez.common.counters.CounterGroup; +import org.apache.tez.common.counters.DAGCounter; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.DAG; import org.apache.tez.dag.api.Edge; @@ -296,6 +297,7 @@ public class TestMockDAGAppMaster { DAGStatus status = dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState()); TezCounters counters = dagImpl.getAllCounters(); + Assert.assertTrue(counters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).getValue() > 0); // verify processor counters Assert.assertEquals(10, counters.findCounter(vAName, procCounterName).getValue()); Assert.assertEquals(1, counters.findCounter(vBName, procCounterName).getValue()); http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java index c48225a..1f8e922 100644 --- a/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java +++ b/tez-plugins/tez-mbeans-resource-calculator/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java @@ -22,7 +22,6 @@ import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import java.lang.management.ManagementFactory; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; /** * Uses sun's MBeans to return process information. http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/GcTimeUpdater.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/GcTimeUpdater.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/GcTimeUpdater.java deleted file mode 100644 index 5e551c5..0000000 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/GcTimeUpdater.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.tez.runtime.metrics; - -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.util.List; - -import org.apache.tez.common.counters.TezCounter; -import org.apache.tez.common.counters.TezCounters; -import org.apache.tez.common.counters.TaskCounter; - -/** - * An updater that tracks the amount of time this task has spent in GC. - */ -class GcTimeUpdater { - private long lastGcMillis = 0; - private List<GarbageCollectorMXBean> gcBeans = null; - TezCounters counters; - - public GcTimeUpdater(TezCounters counters) { - this.gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); - getElapsedGc(); // Initialize 'lastGcMillis' with the current time spent. - this.counters = counters; - } - - /** - * @return the number of milliseconds that the gc has used for CPU since the - * last time this method was called. - */ - protected long getElapsedGc() { - long thisGcMillis = 0; - for (GarbageCollectorMXBean gcBean : gcBeans) { - thisGcMillis += gcBean.getCollectionTime(); - } - - long delta = thisGcMillis - lastGcMillis; - this.lastGcMillis = thisGcMillis; - return delta; - } - - /** - * Increment the gc-elapsed-time counter. - */ - void incrementGcCounter() { - if (null == counters) { - return; // nothing to do. - } - - TezCounter gcCounter = counters.findCounter(TaskCounter.GC_TIME_MILLIS); - if (null != gcCounter) { - gcCounter.increment(getElapsedGc()); - } - } -} http://git-wip-us.apache.org/repos/asf/tez/blob/b7b1b69f/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java index dfdb035..d0ad309 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem.Statistics; import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; +import org.apache.tez.common.GcTimeUpdater; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.TezConfiguration;
