[hbase] branch branch-2 updated: HBASE-26675 Data race on Compactor.writer (#4035)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 5aa0fd2 HBASE-26675 Data race on Compactor.writer (#4035)
5aa0fd2 is described below
commit 5aa0fd2651b04dc251134ffbebb0fa59c3db01b6
Author: Duo Zhang
AuthorDate: Mon Jan 24 19:45:50 2022 +0800
HBASE-26675 Data race on Compactor.writer (#4035)
Signed-off-by: Xin Sun
---
.../hbase/regionserver/compactions/Compactor.java| 20 +++-
.../regionserver/compactions/DefaultCompactor.java | 12 +---
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index a821a90..d934ecb 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -95,7 +95,10 @@ public abstract class Compactor {
private final boolean dropCacheMajor;
private final boolean dropCacheMinor;
- protected T writer = null;
+ // In compaction process only a single thread will access and write to this
field, and
+ // getCompactionTargets is the only place we will access it other than the
compaction thread, so
+ // make it volatile.
+ protected volatile T writer = null;
//TODO: depending on Store is not good but, realistically, all compactors
currently do.
Compactor(Configuration conf, HStore store) {
@@ -546,17 +549,16 @@ public abstract class Compactor {
dropDeletesFromRow, dropDeletesToRow);
}
- public List getCompactionTargets(){
-if (writer == null){
+ public List getCompactionTargets() {
+T writer = this.writer;
+if (writer == null) {
return Collections.emptyList();
}
-synchronized (writer){
- if (writer instanceof StoreFileWriter){
-return Arrays.asList(((StoreFileWriter)writer).getPath());
- }
- return ((AbstractMultiFileWriter)writer).writers().stream().map(sfw ->
sfw.getPath()).collect(
-Collectors.toList());
+if (writer instanceof StoreFileWriter) {
+ return Arrays.asList(((StoreFileWriter) writer).getPath());
}
+return ((AbstractMultiFileWriter) writer).writers().stream().map(sfw ->
sfw.getPath())
+ .collect(Collectors.toList());
}
/**
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
index ad2384a..03e3a1b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
@@ -74,24 +74,22 @@ public class DefaultCompactor extends
Compactor {
@Override
protected void abortWriter() throws IOException {
abortWriter(writer);
+// this step signals that the target file is no longer written and can be
cleaned up
+writer = null;
}
- protected void abortWriter(StoreFileWriter writer) throws IOException {
+ protected final void abortWriter(StoreFileWriter writer) throws IOException {
Path leftoverFile = writer.getPath();
try {
writer.close();
} catch (IOException e) {
LOG.warn("Failed to close the writer after an unfinished compaction.",
e);
-} finally {
- //this step signals that the target file is no longer writen and can be
cleaned up
- writer = null;
}
try {
store.getFileSystem().delete(leftoverFile, false);
} catch (IOException e) {
- LOG.warn(
-"Failed to delete the leftover file " + leftoverFile + " after an
unfinished compaction.",
-e);
+ LOG.warn("Failed to delete the leftover file {} after an unfinished
compaction.",
+leftoverFile, e);
}
}
}
[hbase] branch master updated: HBASE-26675 Data race on Compactor.writer (#4035)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 97f3c1c HBASE-26675 Data race on Compactor.writer (#4035)
97f3c1c is described below
commit 97f3c1cf7fea1c88651b5a0cea6cb2bcac8dc147
Author: Duo Zhang
AuthorDate: Mon Jan 24 19:45:50 2022 +0800
HBASE-26675 Data race on Compactor.writer (#4035)
Signed-off-by: Xin Sun
---
.../hbase/regionserver/compactions/Compactor.java| 20 +++-
.../regionserver/compactions/DefaultCompactor.java | 12 +---
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index 0ee7d34..93f0555 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -96,7 +96,10 @@ public abstract class Compactor {
private final boolean dropCacheMajor;
private final boolean dropCacheMinor;
- protected T writer = null;
+ // In compaction process only a single thread will access and write to this
field, and
+ // getCompactionTargets is the only place we will access it other than the
compaction thread, so
+ // make it volatile.
+ protected volatile T writer = null;
//TODO: depending on Store is not good but, realistically, all compactors
currently do.
Compactor(Configuration conf, HStore store) {
@@ -547,17 +550,16 @@ public abstract class Compactor {
dropDeletesFromRow, dropDeletesToRow);
}
- public List getCompactionTargets(){
-if (writer == null){
+ public List getCompactionTargets() {
+T writer = this.writer;
+if (writer == null) {
return Collections.emptyList();
}
-synchronized (writer){
- if (writer instanceof StoreFileWriter){
-return Arrays.asList(((StoreFileWriter)writer).getPath());
- }
- return ((AbstractMultiFileWriter)writer).writers().stream().map(sfw ->
sfw.getPath()).collect(
-Collectors.toList());
+if (writer instanceof StoreFileWriter) {
+ return Arrays.asList(((StoreFileWriter) writer).getPath());
}
+return ((AbstractMultiFileWriter) writer).writers().stream().map(sfw ->
sfw.getPath())
+ .collect(Collectors.toList());
}
/**
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
index ad2384a..03e3a1b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
@@ -74,24 +74,22 @@ public class DefaultCompactor extends
Compactor {
@Override
protected void abortWriter() throws IOException {
abortWriter(writer);
+// this step signals that the target file is no longer written and can be
cleaned up
+writer = null;
}
- protected void abortWriter(StoreFileWriter writer) throws IOException {
+ protected final void abortWriter(StoreFileWriter writer) throws IOException {
Path leftoverFile = writer.getPath();
try {
writer.close();
} catch (IOException e) {
LOG.warn("Failed to close the writer after an unfinished compaction.",
e);
-} finally {
- //this step signals that the target file is no longer writen and can be
cleaned up
- writer = null;
}
try {
store.getFileSystem().delete(leftoverFile, false);
} catch (IOException e) {
- LOG.warn(
-"Failed to delete the leftover file " + leftoverFile + " after an
unfinished compaction.",
-e);
+ LOG.warn("Failed to delete the leftover file {} after an unfinished
compaction.",
+leftoverFile, e);
}
}
}
[hbase-site] branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 5b58fe3 INFRA-10751 Empty commit 5b58fe3 is described below commit 5b58fe3de831dd395fe8032d56081df1b452308b Author: jenkins AuthorDate: Mon Jan 24 20:20:39 2022 + INFRA-10751 Empty commit
[hbase] branch branch-2.5 updated: HBASE-26520 Remove use of `db.hbase.namespance` tracing attribute (#4015)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 6d739de HBASE-26520 Remove use of `db.hbase.namespance` tracing
attribute (#4015)
6d739de is described below
commit 6d739de5bead242f85253e40cebd8ecef481a566
Author: Nick Dimiduk
AuthorDate: Mon Jan 10 17:02:50 2022 -0800
HBASE-26520 Remove use of `db.hbase.namespance` tracing attribute (#4015)
The HBase-specific attribute `db.hbase.namespace` has been deprecated in
favor of the generic
`db.name`. See also
https://github.com/open-telemetry/opentelemetry-specification/issues/1760
Signed-off-by: Duo Zhang
Signed-off-by: Tak Lon (Stephen) Wu
---
.../java/org/apache/hadoop/hbase/client/trace/TableSpanBuilder.java | 2 --
.../src/test/java/org/apache/hadoop/hbase/client/TestTracingBase.java | 2 +-
.../org/apache/hadoop/hbase/client/trace/hamcrest/TraceTestUtil.java| 1 -
.../java/org/apache/hadoop/hbase/trace/HBaseSemanticAttributes.java | 1 -
4 files changed, 1 insertion(+), 5 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/trace/TableSpanBuilder.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/trace/TableSpanBuilder.java
index 8973da6..437c70b 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/trace/TableSpanBuilder.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/trace/TableSpanBuilder.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.client.trace;
import static org.apache.hadoop.hbase.trace.HBaseSemanticAttributes.DB_NAME;
-import static
org.apache.hadoop.hbase.trace.HBaseSemanticAttributes.NAMESPACE_KEY;
import static org.apache.hadoop.hbase.trace.HBaseSemanticAttributes.TABLE_KEY;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.trace.Span;
@@ -92,7 +91,6 @@ public class TableSpanBuilder implements Supplier {
final Map, Object> attributes,
final TableName tableName
) {
-attributes.put(NAMESPACE_KEY, tableName.getNamespaceAsString());
attributes.put(DB_NAME, tableName.getNamespaceAsString());
attributes.put(TABLE_KEY, tableName.getNameAsString());
}
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTracingBase.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTracingBase.java
index c2067e7..3463348 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTracingBase.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestTracingBase.java
@@ -89,7 +89,7 @@ public class TestTracingBase {
if (tableName != null) {
assertEquals(tableName.getNamespaceAsString(),
-data.getAttributes().get(HBaseSemanticAttributes.NAMESPACE_KEY));
+data.getAttributes().get(HBaseSemanticAttributes.DB_NAME));
assertEquals(tableName.getNameAsString(),
data.getAttributes().get(HBaseSemanticAttributes.TABLE_KEY));
}
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/trace/hamcrest/TraceTestUtil.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/trace/hamcrest/TraceTestUtil.java
index 21d37e8..71aedbd 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/trace/hamcrest/TraceTestUtil.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/trace/hamcrest/TraceTestUtil.java
@@ -59,7 +59,6 @@ public final class TraceTestUtil {
public static Matcher buildTableAttributesMatcher(TableName
tableName) {
return hasAttributes(allOf(
containsEntry("db.name", tableName.getNamespaceAsString()),
- containsEntry("db.hbase.namespace", tableName.getNamespaceAsString()),
containsEntry("db.hbase.table", tableName.getNameAsString(;
}
}
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/HBaseSemanticAttributes.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/HBaseSemanticAttributes.java
index 59c372f..fd6ab85 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/HBaseSemanticAttributes.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/trace/HBaseSemanticAttributes.java
@@ -34,7 +34,6 @@ public final class HBaseSemanticAttributes {
SemanticAttributes.DB_CONNECTION_STRING;
public static final AttributeKey DB_USER =
SemanticAttributes.DB_USER;
public static final AttributeKey DB_NAME =
SemanticAttributes.DB_NAME;
- public static final AttributeKey NAMESPACE_KEY =
SemanticAttributes.DB_HBASE_NAMESPACE;
public static final AttributeKey DB_OPERATION =
SemanticAttributes.DB_OPERATION;
public static final AttributeKey TABLE_KEY =
AttributeKey.stringKey("db.hbase.table");
public static final AttributeKey> REGION_NAMES_KEY =
[hbase] branch branch-2.5 updated: HBASE-26688 Threads shared EMPTY_RESULT may lead to unexpected client job down. (#4048)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 0771c04 HBASE-26688 Threads shared EMPTY_RESULT may lead to
unexpected client job down. (#4048)
0771c04 is described below
commit 0771c04a32448605def9421750de15fa362de6da
Author: Yutong Xiao
AuthorDate: Tue Jan 25 14:48:56 2022 +0800
HBASE-26688 Threads shared EMPTY_RESULT may lead to unexpected client job
down. (#4048)
Signed-off-by: Duo Zhang
---
.../src/main/java/org/apache/hadoop/hbase/client/Result.java | 9 ++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
index 1ef1633..138432aa 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -913,16 +913,19 @@ public class Result implements CellScannable, CellScanner
{
@Override
public Cell current() {
-if (cells == null
+if (isEmpty()
|| cellScannerIndex == INITIAL_CELLSCANNER_INDEX
-|| cellScannerIndex >= cells.length)
+|| cellScannerIndex >= cells.length) {
return null;
+}
return this.cells[cellScannerIndex];
}
@Override
public boolean advance() {
-if (cells == null) return false;
+if (isEmpty()) {
+ return false;
+}
cellScannerIndex++;
if (cellScannerIndex < this.cells.length) {
return true;
[hbase] branch branch-2.4 updated: HBASE-26688 Threads shared EMPTY_RESULT may lead to unexpected client job down. (#4048)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new fbd22f0 HBASE-26688 Threads shared EMPTY_RESULT may lead to
unexpected client job down. (#4048)
fbd22f0 is described below
commit fbd22f06bff8546e6d7a8757b3cf2f8eaed88f6b
Author: Yutong Xiao
AuthorDate: Tue Jan 25 14:48:56 2022 +0800
HBASE-26688 Threads shared EMPTY_RESULT may lead to unexpected client job
down. (#4048)
Signed-off-by: Duo Zhang
---
.../src/main/java/org/apache/hadoop/hbase/client/Result.java | 9 ++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
index 1ef1633..138432aa 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -913,16 +913,19 @@ public class Result implements CellScannable, CellScanner
{
@Override
public Cell current() {
-if (cells == null
+if (isEmpty()
|| cellScannerIndex == INITIAL_CELLSCANNER_INDEX
-|| cellScannerIndex >= cells.length)
+|| cellScannerIndex >= cells.length) {
return null;
+}
return this.cells[cellScannerIndex];
}
@Override
public boolean advance() {
-if (cells == null) return false;
+if (isEmpty()) {
+ return false;
+}
cellScannerIndex++;
if (cellScannerIndex < this.cells.length) {
return true;
[hbase] branch master updated: HBASE-26700 The way we bypass broken track file is not enough in StoreFileListFile (#4055)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 8251bd5 HBASE-26700 The way we bypass broken track file is not enough
in StoreFileListFile (#4055)
8251bd5 is described below
commit 8251bd566a825b5b3334c25c4e1e585e26d69cf6
Author: Duo Zhang
AuthorDate: Tue Jan 25 14:51:11 2022 +0800
HBASE-26700 The way we bypass broken track file is not enough in
StoreFileListFile (#4055)
Signed-off-by: Wellington Ramos Chevreuil
---
.../storefiletracker/StoreFileListFile.java| 57 +--
.../master/procedure/TestCreateTableProcedure.java | 4 +-
.../regionserver/TestMergesSplitsAddToTracker.java | 9 +-
...leTracker.java => StoreFileTrackerForTest.java} | 6 +-
.../storefiletracker/TestStoreFileListFile.java| 165 +
5 files changed, 216 insertions(+), 25 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
index ffb3647..ced0118 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
@@ -17,8 +17,10 @@
*/
package org.apache.hadoop.hbase.regionserver.storefiletracker;
+import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.zip.CRC32;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -29,9 +31,6 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams;
-import
org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
-
import
org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList;
/**
@@ -42,18 +41,27 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.
* other file.
*
* So in this way, we could avoid listing when we want to load the store file
list file.
+ *
+ * To prevent loading partial file, we use the first 4 bytes as file length,
and also append a 4
+ * bytes crc32 checksum at the end. This is because the protobuf message
parser sometimes can return
+ * without error on partial bytes if you stop at some special points, but the
return message will
+ * have incorrect field value. We should try our best to prevent this happens
because loading an
+ * incorrect store file list file usually leads to data loss.
*/
@InterfaceAudience.Private
class StoreFileListFile {
private static final Logger LOG =
LoggerFactory.getLogger(StoreFileListFile.class);
- private static final String TRACK_FILE_DIR = ".filelist";
+ static final String TRACK_FILE_DIR = ".filelist";
private static final String TRACK_FILE = "f1";
private static final String TRACK_FILE_ROTATE = "f2";
+ // 16 MB, which is big enough for a tracker file
+ private static final int MAX_FILE_SIZE = 16 * 1024 * 1024;
+
private final StoreContext ctx;
private final Path trackFileDir;
@@ -74,16 +82,26 @@ class StoreFileListFile {
private StoreFileList load(Path path) throws IOException {
FileSystem fs = ctx.getRegionFileSystem().getFileSystem();
-byte[] bytes;
+byte[] data;
+int expectedChecksum;
try (FSDataInputStream in = fs.open(path)) {
- bytes = ByteStreams.toByteArray(in);
+ int length = in.readInt();
+ if (length <= 0 || length > MAX_FILE_SIZE) {
+throw new IOException("Invalid file length " + length +
+ ", either less than 0 or greater then max allowed size " +
MAX_FILE_SIZE);
+ }
+ data = new byte[length];
+ in.readFully(data);
+ expectedChecksum = in.readInt();
}
-// Read all the bytes and then parse it, so we will only throw
InvalidProtocolBufferException
-// here. This is very important for upper layer to determine whether this
is the normal case,
-// where the file does not exist or is incomplete. If there is another
type of exception, the
-// upper layer should throw it out instead of just ignoring it, otherwise
it will lead to data
-// loss.
-return StoreFileList.parseFrom(bytes);
+CRC32 crc32 = new CRC32();
+crc32.update(data);
+int calculatedChecksum = (int) crc32.getValue();
+if (expectedChecksum != calculatedChecksum) {
+ throw new IOException(
+"Checksum mismatch, expected " + expectedChecksum + ", actual " +
calculatedChecksum);
+}
+return StoreFileList.parseFrom(data);
}
[hbase] branch master updated (2be3958 -> d77ede4)
This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hbase.git. from 2be3958 HBASE-26474 Implement connection-level attributes (addendum) add d77ede4 HBASE-26688 Threads shared EMPTY_RESULT may lead to unexpected client job down. (#4048) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/hadoop/hbase/client/Result.java | 9 ++--- 1 file changed, 6 insertions(+), 3 deletions(-)
[hbase] 01/02: HBASE-26688 Threads shared EMPTY_RESULT may lead to unexpected client job down. (#4048)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 087bb1342a14b20b0785a33a563563bfc66fcce8
Author: Yutong Xiao
AuthorDate: Tue Jan 25 14:48:56 2022 +0800
HBASE-26688 Threads shared EMPTY_RESULT may lead to unexpected client job
down. (#4048)
Signed-off-by: Duo Zhang
---
.../src/main/java/org/apache/hadoop/hbase/client/Result.java | 9 ++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
index 1ef1633..138432aa 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
@@ -913,16 +913,19 @@ public class Result implements CellScannable, CellScanner
{
@Override
public Cell current() {
-if (cells == null
+if (isEmpty()
|| cellScannerIndex == INITIAL_CELLSCANNER_INDEX
-|| cellScannerIndex >= cells.length)
+|| cellScannerIndex >= cells.length) {
return null;
+}
return this.cells[cellScannerIndex];
}
@Override
public boolean advance() {
-if (cells == null) return false;
+if (isEmpty()) {
+ return false;
+}
cellScannerIndex++;
if (cellScannerIndex < this.cells.length) {
return true;
[hbase] branch branch-2 updated (26d7682 -> 3021c58)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from 26d7682 HBASE-26474 Implement connection-level attributes (addendum)
new 087bb13 HBASE-26688 Threads shared EMPTY_RESULT may lead to
unexpected client job down. (#4048)
new 3021c58 HBASE-26700 The way we bypass broken track file is not enough
in StoreFileListFile (#4055)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/hadoop/hbase/client/Result.java | 9 +-
.../storefiletracker/StoreFileListFile.java| 57 +--
.../master/procedure/TestCreateTableProcedure.java | 4 +-
.../regionserver/TestMergesSplitsAddToTracker.java | 9 +-
...leTracker.java => StoreFileTrackerForTest.java} | 6 +-
.../storefiletracker/TestStoreFileListFile.java| 165 +
6 files changed, 222 insertions(+), 28 deletions(-)
rename
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/{TestStoreFileTracker.java
=> StoreFileTrackerForTest.java} (91%)
create mode 100644
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileListFile.java
[hbase] 02/02: HBASE-26700 The way we bypass broken track file is not enough in StoreFileListFile (#4055)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 3021c585137b1aa4b716cfa7258f07fd08e79545
Author: Duo Zhang
AuthorDate: Tue Jan 25 14:51:11 2022 +0800
HBASE-26700 The way we bypass broken track file is not enough in
StoreFileListFile (#4055)
Signed-off-by: Wellington Ramos Chevreuil
---
.../storefiletracker/StoreFileListFile.java| 57 +--
.../master/procedure/TestCreateTableProcedure.java | 4 +-
.../regionserver/TestMergesSplitsAddToTracker.java | 9 +-
...leTracker.java => StoreFileTrackerForTest.java} | 6 +-
.../storefiletracker/TestStoreFileListFile.java| 165 +
5 files changed, 216 insertions(+), 25 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
index ffb3647..ced0118 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileListFile.java
@@ -17,8 +17,10 @@
*/
package org.apache.hadoop.hbase.regionserver.storefiletracker;
+import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.zip.CRC32;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -29,9 +31,6 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams;
-import
org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
-
import
org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.StoreFileList;
/**
@@ -42,18 +41,27 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.StoreFileTrackerProtos.
* other file.
*
* So in this way, we could avoid listing when we want to load the store file
list file.
+ *
+ * To prevent loading partial file, we use the first 4 bytes as file length,
and also append a 4
+ * bytes crc32 checksum at the end. This is because the protobuf message
parser sometimes can return
+ * without error on partial bytes if you stop at some special points, but the
return message will
+ * have incorrect field value. We should try our best to prevent this happens
because loading an
+ * incorrect store file list file usually leads to data loss.
*/
@InterfaceAudience.Private
class StoreFileListFile {
private static final Logger LOG =
LoggerFactory.getLogger(StoreFileListFile.class);
- private static final String TRACK_FILE_DIR = ".filelist";
+ static final String TRACK_FILE_DIR = ".filelist";
private static final String TRACK_FILE = "f1";
private static final String TRACK_FILE_ROTATE = "f2";
+ // 16 MB, which is big enough for a tracker file
+ private static final int MAX_FILE_SIZE = 16 * 1024 * 1024;
+
private final StoreContext ctx;
private final Path trackFileDir;
@@ -74,16 +82,26 @@ class StoreFileListFile {
private StoreFileList load(Path path) throws IOException {
FileSystem fs = ctx.getRegionFileSystem().getFileSystem();
-byte[] bytes;
+byte[] data;
+int expectedChecksum;
try (FSDataInputStream in = fs.open(path)) {
- bytes = ByteStreams.toByteArray(in);
+ int length = in.readInt();
+ if (length <= 0 || length > MAX_FILE_SIZE) {
+throw new IOException("Invalid file length " + length +
+ ", either less than 0 or greater then max allowed size " +
MAX_FILE_SIZE);
+ }
+ data = new byte[length];
+ in.readFully(data);
+ expectedChecksum = in.readInt();
}
-// Read all the bytes and then parse it, so we will only throw
InvalidProtocolBufferException
-// here. This is very important for upper layer to determine whether this
is the normal case,
-// where the file does not exist or is incomplete. If there is another
type of exception, the
-// upper layer should throw it out instead of just ignoring it, otherwise
it will lead to data
-// loss.
-return StoreFileList.parseFrom(bytes);
+CRC32 crc32 = new CRC32();
+crc32.update(data);
+int calculatedChecksum = (int) crc32.getValue();
+if (expectedChecksum != calculatedChecksum) {
+ throw new IOException(
+"Checksum mismatch, expected " + expectedChecksum + ", actual " +
calculatedChecksum);
+}
+return StoreFileList.parseFrom(data);
}
private int select(StoreFileList[] lists) {
@@ -101,9 +119,9 @@ class StoreFileListFile {
for (int i = 0; i < 2; i++) {
try {
lists[i] = load(trackFiles[i]);
- } catch
[hbase] branch master updated: HBASE-26474 Implement connection-level attributes (addendum)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 2be3958 HBASE-26474 Implement connection-level attributes (addendum)
2be3958 is described below
commit 2be39588be478a65be91be73d9376cffa10bc50c
Author: Nick Dimiduk
AuthorDate: Wed Jan 12 13:34:51 2022 -0800
HBASE-26474 Implement connection-level attributes (addendum)
Addressing additional comments raised in branch-2 backport PR #4014
---
.../hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java | 5 -
.../apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java | 4
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
index 164c1a9..60137d2 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
@@ -272,11 +272,6 @@ abstract class AbstractRpcBasedConnectionRegistry
implements ConnectionRegistry
}
@Override
- public String getConnectionString() {
-return "unimplemented";
- }
-
- @Override
public void close() {
trace(() -> {
if (registryEndpointRefresher != null) {
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
index 2979743..146895a 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
@@ -167,6 +167,10 @@ public class TestRpcBasedRegistryHedgedReads {
protected CompletableFuture> fetchEndpoints() {
return CompletableFuture.completedFuture(BOOTSTRAP_NODES);
}
+
+ @Override public String getConnectionString() {
+return "unimplemented";
+ }
};
}
[hbase-filesystem] branch master updated: HBASE-26236 Simple travis build for hbase-filesystem (#28)
This is an automated email from the ASF dual-hosted git repository.
elserj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-filesystem.git
The following commit(s) were added to refs/heads/master by this push:
new 9d3d800 HBASE-26236 Simple travis build for hbase-filesystem (#28)
9d3d800 is described below
commit 9d3d800b6f8ba4542ca808dfadf9c12bdb9e2766
Author: Peter Somogyi
AuthorDate: Tue Jan 25 02:40:28 2022 +0100
HBASE-26236 Simple travis build for hbase-filesystem (#28)
Co-authored-by: Josh Elser
---
.travis.yml | 36
1 file changed, 36 insertions(+)
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 000..3a86f63
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+language: java
+jdk:
+ - openjdk8
+ - openjdk11
+env:
+ - HADOOP_PROFILE=3.2
+ - HADOOP_PROFILE=3.3
+dist: xenial
+os: linux
+jobs:
+ fast_finish: true
+script:
+ - "mvn clean install -B -V -Dhadoop.profile=${HADOOP_PROFILE}"
+branches:
+ only:
+- master
+git:
+ depth: 1
+cache:
+ directories:
+- $HOME/.m2
[hbase] branch branch-2.5 updated: HBASE-26474 Implement connection-level attributes (addendum)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 4aa31ce HBASE-26474 Implement connection-level attributes (addendum)
4aa31ce is described below
commit 4aa31ce0cd3ccc7c6852883d6fcd29901d9dc69f
Author: Nick Dimiduk
AuthorDate: Wed Jan 12 13:34:51 2022 -0800
HBASE-26474 Implement connection-level attributes (addendum)
Addressing additional comments raised in branch-2 backport PR #4014
---
.../hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java | 5 -
.../apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java | 4
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
index 0ee374d..54138d3 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
@@ -273,11 +273,6 @@ abstract class AbstractRpcBasedConnectionRegistry
implements ConnectionRegistry
}
@Override
- public String getConnectionString() {
-return "unimplemented";
- }
-
- @Override
public void close() {
trace(() -> {
if (registryEndpointRefresher != null) {
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
index 29a9c7f..e534ab0 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
@@ -167,6 +167,10 @@ public class TestRpcBasedRegistryHedgedReads {
protected CompletableFuture> fetchEndpoints() {
return CompletableFuture.completedFuture(BOOTSTRAP_NODES);
}
+
+ @Override public String getConnectionString() {
+return "unimplemented";
+ }
};
}
[hbase] branch branch-2 updated: HBASE-26474 Implement connection-level attributes (addendum)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 26d7682 HBASE-26474 Implement connection-level attributes (addendum)
26d7682 is described below
commit 26d768243bfa22b480f521dde80cf41d5d6ff38e
Author: Nick Dimiduk
AuthorDate: Wed Jan 12 13:34:51 2022 -0800
HBASE-26474 Implement connection-level attributes (addendum)
Addressing additional comments raised in branch-2 backport PR #4014
---
.../hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java | 5 -
.../apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java | 4
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
index 0ee374d..54138d3 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AbstractRpcBasedConnectionRegistry.java
@@ -273,11 +273,6 @@ abstract class AbstractRpcBasedConnectionRegistry
implements ConnectionRegistry
}
@Override
- public String getConnectionString() {
-return "unimplemented";
- }
-
- @Override
public void close() {
trace(() -> {
if (registryEndpointRefresher != null) {
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
index 29a9c7f..e534ab0 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestRpcBasedRegistryHedgedReads.java
@@ -167,6 +167,10 @@ public class TestRpcBasedRegistryHedgedReads {
protected CompletableFuture> fetchEndpoints() {
return CompletableFuture.completedFuture(BOOTSTRAP_NODES);
}
+
+ @Override public String getConnectionString() {
+return "unimplemented";
+ }
};
}
