Repository: hadoop Updated Branches: refs/heads/branch-3.1 671e407ce -> f955d52f3
YARN-8120. JVM can crash with SIGSEGV when exiting due to custom leveldb logger. Contributed by Jason Lowe. (cherry picked from commit 6bb128dfb893cf0e4aa2d3ecc65440668a1fc8d7) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f955d52f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f955d52f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f955d52f Branch: refs/heads/branch-3.1 Commit: f955d52f39c1ecb4aa53cc4f4d18652b4914d8ee Parents: 671e407 Author: Eric E Payne <er...@oath.com> Authored: Thu Apr 12 16:04:23 2018 +0000 Committer: Eric E Payne <er...@oath.com> Committed: Thu Apr 12 16:11:57 2018 +0000 ---------------------------------------------------------------------- .../v2/hs/HistoryServerLeveldbStateStoreService.java | 11 ----------- .../java/org/apache/hadoop/mapred/ShuffleHandler.java | 12 ------------ .../recovery/NMLeveldbStateStoreService.java | 12 ------------ .../resourcemanager/recovery/LeveldbRMStateStore.java | 12 ------------ 4 files changed, 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f955d52f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java index c8741aa..6d2e407 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerLeveldbStateStoreService.java @@ -74,7 +74,6 @@ public class HistoryServerLeveldbStateStoreService extends Path storeRoot = createStorageDir(getConfig()); Options options = new Options(); options.createIfMissing(false); - options.logger(new LeveldbLogger()); LOG.info("Using state database at " + storeRoot + " for recovery"); File dbfile = new File(storeRoot.toString()); try { @@ -366,14 +365,4 @@ public class HistoryServerLeveldbStateStoreService extends + getCurrentVersion() + ", but loading version " + loadedVersion); } } - - private static class LeveldbLogger implements org.iq80.leveldb.Logger { - private static final Logger LOG = - LoggerFactory.getLogger(LeveldbLogger.class); - - @Override - public void log(String message) { - LOG.info(message); - } - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f955d52f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index ec992fe..aeda9cc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -93,7 +93,6 @@ import org.fusesource.leveldbjni.JniDBFactory; import org.fusesource.leveldbjni.internal.NativeDB; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBException; -import org.iq80.leveldb.Logger; import org.iq80.leveldb.Options; import org.jboss.netty.bootstrap.ServerBootstrap; import org.jboss.netty.buffer.ChannelBuffers; @@ -628,7 +627,6 @@ public class ShuffleHandler extends AuxiliaryService { private void startStore(Path recoveryRoot) throws IOException { Options options = new Options(); options.createIfMissing(false); - options.logger(new LevelDBLogger()); Path dbPath = new Path(recoveryRoot, STATE_DB_NAME); LOG.info("Using state database at " + dbPath + " for recovery"); File dbfile = new File(dbPath.toString()); @@ -774,16 +772,6 @@ public class ShuffleHandler extends AuxiliaryService { } } - private static class LevelDBLogger implements Logger { - private static final org.slf4j.Logger LOG = - LoggerFactory.getLogger(LevelDBLogger.class); - - @Override - public void log(String message) { - LOG.info(message); - } - } - static class TimeoutHandler extends IdleStateAwareChannelHandler { private boolean enabledTimeout; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f955d52f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index 723dd48..6f643b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -1531,7 +1531,6 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { Path storeRoot = createStorageDir(conf); Options options = new Options(); options.createIfMissing(false); - options.logger(new LeveldbLogger()); LOG.info("Using state database at " + storeRoot + " for recovery"); File dbfile = new File(storeRoot.toString()); try { @@ -1596,17 +1595,6 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { } } - private static class LeveldbLogger implements org.iq80.leveldb.Logger { - private static final org.slf4j.Logger LOG = - LoggerFactory.getLogger(LeveldbLogger.class); - - @Override - public void log(String message) { - LOG.info(message); - } - } - - Version loadVersion() throws IOException { byte[] data = db.get(bytes(DB_SCHEMA_VERSION_KEY)); // if version is not stored previously, treat it as CURRENT_VERSION_INFO. http://git-wip-us.apache.org/repos/asf/hadoop/blob/f955d52f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java index a53083f..36a8dfa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java @@ -64,12 +64,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AM import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; import org.apache.hadoop.yarn.server.utils.LeveldbIterator; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.fusesource.leveldbjni.JniDBFactory; import org.fusesource.leveldbjni.internal.NativeDB; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBException; -import org.iq80.leveldb.Logger; import org.iq80.leveldb.Options; import org.iq80.leveldb.WriteBatch; @@ -165,7 +163,6 @@ public class LeveldbRMStateStore extends RMStateStore { Path storeRoot = createStorageDir(); Options options = new Options(); options.createIfMissing(false); - options.logger(new LeveldbLogger()); LOG.info("Using state database at " + storeRoot + " for recovery"); File dbfile = new File(storeRoot.toString()); try { @@ -872,13 +869,4 @@ public class LeveldbRMStateStore extends RMStateStore { LOG.info("Full compaction cycle completed in " + duration + " msec"); } } - - private static class LeveldbLogger implements Logger { - private static final Log LOG = LogFactory.getLog(LeveldbLogger.class); - - @Override - public void log(String message) { - LOG.info(message); - } - } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
