Author: szetszwo
Date: Mon Jul 21 23:37:54 2014
New Revision: 1612432
URL: http://svn.apache.org/r1612432
Log:
Merge r1609845 through r1612431 from trunk.
Added:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/records/
- copied from r1612431,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/records/
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_recovery.proto
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt?rev=1612432&r1=1612431&r2=1612432&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt Mon Jul 21
23:37:54 2014
@@ -54,6 +54,8 @@ Release 2.6.0 - UNRELEASED
YARN-2323. FairShareComparator creates too many Resource objects (Hong
Zhiguo
via Sandy Ryza)
+ YARN-2045. Data persisted in NM should be versioned (Junping Du via jlowe)
+
OPTIMIZATIONS
BUG FIXES
@@ -77,6 +79,9 @@ Release 2.6.0 - UNRELEASED
YARN-2244. FairScheduler missing handling of containers for unknown
application attempts. (Anubhav Dhoot via kasha)
+ YARN-2321. NodeManager web UI can incorrectly report Pmem enforcement
+ (Leitao Guo via jlowe)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java?rev=1612432&r1=1612431&r2=1612432&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
Mon Jul 21 23:37:54 2014
@@ -42,8 +42,11 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import
org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
import
org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto;
+import
org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.NMDBSchemaVersionProto;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
+import
org.apache.hadoop.yarn.server.nodemanager.recovery.records.NMDBSchemaVersion;
+import
org.apache.hadoop.yarn.server.nodemanager.recovery.records.impl.pb.NMDBSchemaVersionPBImpl;
import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.fusesource.leveldbjni.JniDBFactory;
@@ -54,14 +57,18 @@ import org.iq80.leveldb.Logger;
import org.iq80.leveldb.Options;
import org.iq80.leveldb.WriteBatch;
+import com.google.common.annotations.VisibleForTesting;
+
public class NMLeveldbStateStoreService extends NMStateStoreService {
public static final Log LOG =
LogFactory.getLog(NMLeveldbStateStoreService.class);
private static final String DB_NAME = "yarn-nm-state";
- private static final String DB_SCHEMA_VERSION_KEY = "schema-version";
- private static final String DB_SCHEMA_VERSION = "1.0";
+ private static final String DB_SCHEMA_VERSION_KEY = "nm-schema-version";
+
+ private static final NMDBSchemaVersion CURRENT_VERSION_INFO =
NMDBSchemaVersion
+ .newInstance(1, 0);
private static final String DELETION_TASK_KEY_PREFIX =
"DeletionService/deltask_";
@@ -475,22 +482,16 @@ public class NMLeveldbStateStoreService
options.logger(new LeveldbLogger());
LOG.info("Using state database at " + storeRoot + " for recovery");
File dbfile = new File(storeRoot.toString());
- byte[] schemaVersionData = null;
try {
db = JniDBFactory.factory.open(dbfile, options);
- try {
- schemaVersionData = db.get(bytes(DB_SCHEMA_VERSION_KEY));
- } catch (DBException e) {
- throw new IOException(e.getMessage(), e);
- }
} catch (NativeDB.DBException e) {
if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
LOG.info("Creating state database at " + dbfile);
options.createIfMissing(true);
try {
db = JniDBFactory.factory.open(dbfile, options);
- schemaVersionData = bytes(DB_SCHEMA_VERSION);
- db.put(bytes(DB_SCHEMA_VERSION_KEY), schemaVersionData);
+ // store version
+ storeVersion();
} catch (DBException dbErr) {
throw new IOException(dbErr.getMessage(), dbErr);
}
@@ -498,16 +499,7 @@ public class NMLeveldbStateStoreService
throw e;
}
}
- if (schemaVersionData != null) {
- String schemaVersion = asString(schemaVersionData);
- // only support exact schema matches for now
- if (!DB_SCHEMA_VERSION.equals(schemaVersion)) {
- throw new IOException("Incompatible state database schema, found "
- + schemaVersion + " expected " + DB_SCHEMA_VERSION);
- }
- } else {
- throw new IOException("State database schema version not found");
- }
+ checkVersion();
}
private Path createStorageDir(Configuration conf) throws IOException {
@@ -532,4 +524,68 @@ public class NMLeveldbStateStoreService
LOG.info(message);
}
}
+
+
+ NMDBSchemaVersion loadVersion() throws IOException {
+ byte[] data = db.get(bytes(DB_SCHEMA_VERSION_KEY));
+ // if version is not stored previously, treat it as 1.0.
+ if (data == null || data.length == 0) {
+ return NMDBSchemaVersion.newInstance(1, 0);
+ }
+ NMDBSchemaVersion version =
+ new NMDBSchemaVersionPBImpl(NMDBSchemaVersionProto.parseFrom(data));
+ return version;
+ }
+
+ private void storeVersion() throws IOException {
+ dbStoreVersion(CURRENT_VERSION_INFO);
+ }
+
+ // Only used for test
+ @VisibleForTesting
+ void storeVersion(NMDBSchemaVersion state) throws IOException {
+ dbStoreVersion(state);
+ }
+
+ private void dbStoreVersion(NMDBSchemaVersion state) throws IOException {
+ String key = DB_SCHEMA_VERSION_KEY;
+ byte[] data =
+ ((NMDBSchemaVersionPBImpl) state).getProto().toByteArray();
+ try {
+ db.put(bytes(key), data);
+ } catch (DBException e) {
+ throw new IOException(e.getMessage(), e);
+ }
+ }
+
+ NMDBSchemaVersion getCurrentVersion() {
+ return CURRENT_VERSION_INFO;
+ }
+
+ /**
+ * 1) Versioning scheme: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
+ * 2) Any incompatible change of state-store is a major upgrade, and any
+ * compatible change of state-store is a minor upgrade.
+ * 3) Within a minor upgrade, say 1.1 to 1.2:
+ * overwrite the version info and proceed as normal.
+ * 4) Within a major upgrade, say 1.2 to 2.0:
+ * throw exception and indicate user to use a separate upgrade tool to
+ * upgrade NM state or remove incompatible old state.
+ */
+ private void checkVersion() throws IOException {
+ NMDBSchemaVersion loadedVersion = loadVersion();
+ LOG.info("Loaded NM state version info " + loadedVersion);
+ if (loadedVersion != null && loadedVersion.equals(getCurrentVersion())) {
+ return;
+ }
+ if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
+ LOG.info("Storing NM state version info " + getCurrentVersion());
+ storeVersion();
+ } else {
+ throw new IOException(
+ "Incompatible version for NM state: expecting NM state version "
+ + getCurrentVersion() + ", but loading version " + loadedVersion);
+ }
+ }
+
}
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java?rev=1612432&r1=1612431&r2=1612432&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
Mon Jul 21 23:37:54 2014
@@ -72,7 +72,7 @@ public class NodePage extends NMView {
._("Total Pmem allocated for Container",
StringUtils.byteDesc(info.getTotalPmemAllocated() * BYTES_IN_MB))
._("Pmem enforcement enabled",
- info.isVmemCheckEnabled())
+ info.isPmemCheckEnabled())
._("Total VCores allocated for Containers",
String.valueOf(info.getTotalVCoresAllocated()))
._("NodeHealthyStatus",
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_recovery.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_recovery.proto?rev=1612432&r1=1612431&r2=1612432&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_recovery.proto
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_recovery.proto
Mon Jul 21 23:37:54 2014
@@ -38,3 +38,9 @@ message LocalizedResourceProto {
optional string localPath = 2;
optional int64 size = 3;
}
+
+message NMDBSchemaVersionProto {
+ optional int32 majorVersion = 1;
+ optional int32 minorVersion = 2;
+}
+
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java?rev=1612432&r1=1612431&r2=1612432&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
(original)
+++
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
Mon Jul 21 23:37:54 2014
@@ -29,6 +29,7 @@ import java.util.Map;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -45,9 +46,11 @@ import org.apache.hadoop.yarn.server.nod
import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredLocalizationState;
import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredNMTokenState;
import
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredUserResources;
+import
org.apache.hadoop.yarn.server.nodemanager.recovery.records.NMDBSchemaVersion;
import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -102,6 +105,36 @@ public class TestNMLeveldbStateStoreServ
assertTrue(stateStore.canRecover());
verifyEmptyState();
}
+
+ @Test
+ public void testCheckVersion() throws IOException {
+ // default version
+ NMDBSchemaVersion defaultVersion = stateStore.getCurrentVersion();
+ Assert.assertEquals(defaultVersion, stateStore.loadVersion());
+
+ // compatible version
+ NMDBSchemaVersion compatibleVersion =
+ NMDBSchemaVersion.newInstance(defaultVersion.getMajorVersion(),
+ defaultVersion.getMinorVersion() + 2);
+ stateStore.storeVersion(compatibleVersion);
+ Assert.assertEquals(compatibleVersion, stateStore.loadVersion());
+ restartStateStore();
+ // overwrite the compatible version
+ Assert.assertEquals(defaultVersion, stateStore.loadVersion());
+
+ // incompatible version
+ NMDBSchemaVersion incompatibleVersion =
+ NMDBSchemaVersion.newInstance(defaultVersion.getMajorVersion() + 1,
+ defaultVersion.getMinorVersion());
+ stateStore.storeVersion(incompatibleVersion);
+ try {
+ restartStateStore();
+ Assert.fail("Incompatible version, should expect fail here.");
+ } catch (ServiceStateException e) {
+ Assert.assertTrue("Exception message mismatch",
+ e.getMessage().contains("Incompatible version for NM state:"));
+ }
+ }
@Test
public void testStartResourceLocalization() throws IOException {
