[38/50] [abbrv] hbase git commit: HBASE-19083 Introduce a new log writer which can write to two HDFSes
HBASE-19083 Introduce a new log writer which can write to two HDFSes Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8753f5f6 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8753f5f6 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8753f5f6 Branch: refs/heads/HBASE-19064 Commit: 8753f5f671783827018220898fd864383c66b341 Parents: 125767b Author: zhangduoAuthored: Thu Jan 11 21:08:02 2018 +0800 Committer: zhangduo Committed: Tue Apr 17 14:40:29 2018 +0800 -- .../hbase/regionserver/wal/AsyncFSWAL.java | 21 +-- .../regionserver/wal/CombinedAsyncWriter.java | 134 ++ .../hbase/regionserver/wal/DualAsyncFSWAL.java | 67 + .../wal/AbstractTestProtobufLog.java| 110 +++ .../regionserver/wal/ProtobufLogTestHelper.java | 99 ++ .../regionserver/wal/TestAsyncProtobufLog.java | 32 + .../wal/TestCombinedAsyncWriter.java| 136 +++ .../hbase/regionserver/wal/TestProtobufLog.java | 14 +- .../regionserver/wal/WriterOverAsyncWriter.java | 63 + 9 files changed, 533 insertions(+), 143 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/8753f5f6/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java index e34818f..0bee9d6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java @@ -607,12 +607,16 @@ public class AsyncFSWAL extends AbstractFSWAL { } } - @Override - protected AsyncWriter createWriterInstance(Path path) throws IOException { + protected final AsyncWriter createAsyncWriter(FileSystem fs, Path path) throws IOException { return AsyncFSWALProvider.createAsyncWriter(conf, fs, path, false, eventLoopGroup, channelClass); } + @Override + protected AsyncWriter createWriterInstance(Path path) throws IOException { +return createAsyncWriter(fs, path); + } + private void waitForSafePoint() { consumeLock.lock(); try { @@ -632,13 +636,12 @@ public class AsyncFSWAL extends AbstractFSWAL { } } - private long closeWriter() { -AsyncWriter oldWriter = this.writer; -if (oldWriter != null) { - long fileLength = oldWriter.getLength(); + protected final long closeWriter(AsyncWriter writer) { +if (writer != null) { + long fileLength = writer.getLength(); closeExecutor.execute(() -> { try { - oldWriter.close(); + writer.close(); } catch (IOException e) { LOG.warn("close old writer failed", e); } @@ -654,7 +657,7 @@ public class AsyncFSWAL extends AbstractFSWAL { throws IOException { Preconditions.checkNotNull(nextWriter); waitForSafePoint(); -long oldFileLen = closeWriter(); +long oldFileLen = closeWriter(this.writer); logRollAndSetupWalProps(oldPath, newPath, oldFileLen); this.writer = nextWriter; if (nextWriter instanceof AsyncProtobufLogWriter) { @@ -679,7 +682,7 @@ public class AsyncFSWAL extends AbstractFSWAL { @Override protected void doShutdown() throws IOException { waitForSafePoint(); -closeWriter(); +closeWriter(this.writer); closeExecutor.shutdown(); try { if (!closeExecutor.awaitTermination(waitOnShutdownInSeconds, TimeUnit.SECONDS)) { http://git-wip-us.apache.org/repos/asf/hbase/blob/8753f5f6/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java new file mode 100644 index 000..8ecfede --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + *
[38/50] [abbrv] hbase git commit: HBASE-19083 Introduce a new log writer which can write to two HDFSes
HBASE-19083 Introduce a new log writer which can write to two HDFSes Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/380c54c7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/380c54c7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/380c54c7 Branch: refs/heads/HBASE-19064 Commit: 380c54c7a998c166240b5305e6774e3dccfb01c6 Parents: f663430 Author: zhangduoAuthored: Thu Jan 11 21:08:02 2018 +0800 Committer: zhangduo Committed: Sun Apr 8 17:12:09 2018 +0800 -- .../hbase/regionserver/wal/AsyncFSWAL.java | 21 +-- .../regionserver/wal/CombinedAsyncWriter.java | 134 ++ .../hbase/regionserver/wal/DualAsyncFSWAL.java | 67 + .../wal/AbstractTestProtobufLog.java| 110 +++ .../regionserver/wal/ProtobufLogTestHelper.java | 99 ++ .../regionserver/wal/TestAsyncProtobufLog.java | 32 + .../wal/TestCombinedAsyncWriter.java| 136 +++ .../hbase/regionserver/wal/TestProtobufLog.java | 14 +- .../regionserver/wal/WriterOverAsyncWriter.java | 63 + 9 files changed, 533 insertions(+), 143 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hbase/blob/380c54c7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java index e34818f..0bee9d6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AsyncFSWAL.java @@ -607,12 +607,16 @@ public class AsyncFSWAL extends AbstractFSWAL { } } - @Override - protected AsyncWriter createWriterInstance(Path path) throws IOException { + protected final AsyncWriter createAsyncWriter(FileSystem fs, Path path) throws IOException { return AsyncFSWALProvider.createAsyncWriter(conf, fs, path, false, eventLoopGroup, channelClass); } + @Override + protected AsyncWriter createWriterInstance(Path path) throws IOException { +return createAsyncWriter(fs, path); + } + private void waitForSafePoint() { consumeLock.lock(); try { @@ -632,13 +636,12 @@ public class AsyncFSWAL extends AbstractFSWAL { } } - private long closeWriter() { -AsyncWriter oldWriter = this.writer; -if (oldWriter != null) { - long fileLength = oldWriter.getLength(); + protected final long closeWriter(AsyncWriter writer) { +if (writer != null) { + long fileLength = writer.getLength(); closeExecutor.execute(() -> { try { - oldWriter.close(); + writer.close(); } catch (IOException e) { LOG.warn("close old writer failed", e); } @@ -654,7 +657,7 @@ public class AsyncFSWAL extends AbstractFSWAL { throws IOException { Preconditions.checkNotNull(nextWriter); waitForSafePoint(); -long oldFileLen = closeWriter(); +long oldFileLen = closeWriter(this.writer); logRollAndSetupWalProps(oldPath, newPath, oldFileLen); this.writer = nextWriter; if (nextWriter instanceof AsyncProtobufLogWriter) { @@ -679,7 +682,7 @@ public class AsyncFSWAL extends AbstractFSWAL { @Override protected void doShutdown() throws IOException { waitForSafePoint(); -closeWriter(); +closeWriter(this.writer); closeExecutor.shutdown(); try { if (!closeExecutor.awaitTermination(waitOnShutdownInSeconds, TimeUnit.SECONDS)) { http://git-wip-us.apache.org/repos/asf/hbase/blob/380c54c7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java -- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java new file mode 100644 index 000..8ecfede --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CombinedAsyncWriter.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + *