[04/22] hadoop git commit: HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo Goiri.

2018-03-15 Thread aengineer 
HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo 
Goiri.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76be6cbf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76be6cbf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76be6cbf

Branch: refs/heads/HDFS-7240
Commit: 76be6cbf6c33f866794f27ca2560ca7c7b2fa0e7
Parents: 427fd02
Author: Yiqun Lin 
Authored: Wed Mar 14 11:20:59 2018 +0800
Committer: Yiqun Lin 
Committed: Wed Mar 14 11:20:59 2018 +0800

--
 .../federation/metrics/StateStoreMetrics.java   |   5 +
 .../driver/StateStoreRecordOperations.java  |  15 -
 .../driver/impl/StateStoreFileBaseImpl.java | 433 ++-
 .../store/driver/impl/StateStoreFileImpl.java   | 109 ++---
 .../driver/impl/StateStoreFileSystemImpl.java   | 128 +++---
 .../driver/impl/StateStoreZooKeeperImpl.java|   6 -
 .../store/driver/TestStateStoreDriverBase.java  |   9 +
 .../store/driver/TestStateStoreFile.java|  12 +
 .../store/driver/TestStateStoreFileBase.java|  47 ++
 .../store/driver/TestStateStoreFileSystem.java  |  14 +-
 10 files changed, 428 insertions(+), 350 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/76be6cbf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
index 40dcd40..09253a2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
@@ -140,5 +140,10 @@ public final class StateStoreMetrics implements 
StateStoreMBean {
 writes.resetMinMax();
 removes.resetMinMax();
 failures.resetMinMax();
+
+reads.lastStat().reset();
+writes.lastStat().reset();
+removes.lastStat().reset();
+failures.lastStat().reset();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76be6cbf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
index e76a733..443d46e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -49,21 +49,6 @@ public interface StateStoreRecordOperations {
QueryResult get(Class clazz) throws IOException;
 
   /**
-   * Get all records of the requested record class from the data store. To use
-   * the default implementations in this class, getAll must return new 
instances
-   * of the records on each call. It is recommended to override the default
-   * implementations for better performance.
-   *
-   * @param clazz Class of record to fetch.
-   * @param sub Sub path.
-   * @return List of all records that match the clazz and the sub path.
-   * @throws IOException
-   */
-  @Idempotent
-   QueryResult get(Class clazz, String sub)
-  throws IOException;
-
-  /**
* Get a single record from the store that matches the query.
*
* @param clazz Class of record to fetch.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76be6cbf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
index a0cd878f..6638d1c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
+++

hadoop git commit: HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo Goiri.

2018-03-13 Thread yqlin 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.9 60feb43b7 -> 6e57ca602


HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo 
Goiri.

(cherry picked from commit 76be6cbf6c33f866794f27ca2560ca7c7b2fa0e7)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e57ca60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e57ca60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e57ca60

Branch: refs/heads/branch-2.9
Commit: 6e57ca602904f5a4a9862df377ea8d0f31ebaa21
Parents: 60feb43
Author: Yiqun Lin 
Authored: Wed Mar 14 11:20:59 2018 +0800
Committer: Yiqun Lin 
Committed: Wed Mar 14 11:32:50 2018 +0800

--
 .../federation/metrics/StateStoreMetrics.java   |   5 +
 .../driver/StateStoreRecordOperations.java  |  15 -
 .../driver/impl/StateStoreFileBaseImpl.java | 433 ++-
 .../store/driver/impl/StateStoreFileImpl.java   | 109 ++---
 .../driver/impl/StateStoreFileSystemImpl.java   | 128 +++---
 .../driver/impl/StateStoreZooKeeperImpl.java|   6 -
 .../store/driver/TestStateStoreDriverBase.java  |   9 +
 .../store/driver/TestStateStoreFile.java|  12 +
 .../store/driver/TestStateStoreFileBase.java|  47 ++
 .../store/driver/TestStateStoreFileSystem.java  |  14 +-
 10 files changed, 428 insertions(+), 350 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e57ca60/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
index 40dcd40..09253a2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
@@ -140,5 +140,10 @@ public final class StateStoreMetrics implements 
StateStoreMBean {
 writes.resetMinMax();
 removes.resetMinMax();
 failures.resetMinMax();
+
+reads.lastStat().reset();
+writes.lastStat().reset();
+removes.lastStat().reset();
+failures.lastStat().reset();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e57ca60/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
index e76a733..443d46e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -49,21 +49,6 @@ public interface StateStoreRecordOperations {
QueryResult get(Class clazz) throws IOException;
 
   /**
-   * Get all records of the requested record class from the data store. To use
-   * the default implementations in this class, getAll must return new 
instances
-   * of the records on each call. It is recommended to override the default
-   * implementations for better performance.
-   *
-   * @param clazz Class of record to fetch.
-   * @param sub Sub path.
-   * @return List of all records that match the clazz and the sub path.
-   * @throws IOException
-   */
-  @Idempotent
-   QueryResult get(Class clazz, String sub)
-  throws IOException;
-
-  /**
* Get a single record from the store that matches the query.
*
* @param clazz Class of record to fetch.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e57ca60/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
index a0cd878f..6638d1c 100644
---

hadoop git commit: HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo Goiri.

2018-03-13 Thread yqlin 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 95a4665ad -> a10506972


HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo 
Goiri.

(cherry picked from commit 76be6cbf6c33f866794f27ca2560ca7c7b2fa0e7)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1050697
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1050697
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1050697

Branch: refs/heads/branch-3.0
Commit: a105069729f35f3c8ff7691fa3d51162677734f9
Parents: 95a4665
Author: Yiqun Lin 
Authored: Wed Mar 14 11:20:59 2018 +0800
Committer: Yiqun Lin 
Committed: Wed Mar 14 11:31:35 2018 +0800

--
 .../federation/metrics/StateStoreMetrics.java   |   5 +
 .../driver/StateStoreRecordOperations.java  |  15 -
 .../driver/impl/StateStoreFileBaseImpl.java | 433 ++-
 .../store/driver/impl/StateStoreFileImpl.java   | 109 ++---
 .../driver/impl/StateStoreFileSystemImpl.java   | 128 +++---
 .../driver/impl/StateStoreZooKeeperImpl.java|   6 -
 .../store/driver/TestStateStoreDriverBase.java  |   9 +
 .../store/driver/TestStateStoreFile.java|  12 +
 .../store/driver/TestStateStoreFileBase.java|  47 ++
 .../store/driver/TestStateStoreFileSystem.java  |  14 +-
 10 files changed, 428 insertions(+), 350 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1050697/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
index 40dcd40..09253a2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
@@ -140,5 +140,10 @@ public final class StateStoreMetrics implements 
StateStoreMBean {
 writes.resetMinMax();
 removes.resetMinMax();
 failures.resetMinMax();
+
+reads.lastStat().reset();
+writes.lastStat().reset();
+removes.lastStat().reset();
+failures.lastStat().reset();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1050697/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
index e76a733..443d46e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -49,21 +49,6 @@ public interface StateStoreRecordOperations {
QueryResult get(Class clazz) throws IOException;
 
   /**
-   * Get all records of the requested record class from the data store. To use
-   * the default implementations in this class, getAll must return new 
instances
-   * of the records on each call. It is recommended to override the default
-   * implementations for better performance.
-   *
-   * @param clazz Class of record to fetch.
-   * @param sub Sub path.
-   * @return List of all records that match the clazz and the sub path.
-   * @throws IOException
-   */
-  @Idempotent
-   QueryResult get(Class clazz, String sub)
-  throws IOException;
-
-  /**
* Get a single record from the store that matches the query.
*
* @param clazz Class of record to fetch.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1050697/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
index a0cd878f..6638d1c 100644
---

hadoop git commit: HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo Goiri.

2018-03-13 Thread yqlin 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 36451f2d5 -> b3d56cb83


HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo 
Goiri.

(cherry picked from commit 76be6cbf6c33f866794f27ca2560ca7c7b2fa0e7)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3d56cb8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3d56cb8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3d56cb8

Branch: refs/heads/branch-2
Commit: b3d56cb83558c797403cb4538d0c21bf097263ce
Parents: 36451f2
Author: Yiqun Lin 
Authored: Wed Mar 14 11:20:59 2018 +0800
Committer: Yiqun Lin 
Committed: Wed Mar 14 11:25:08 2018 +0800

--
 .../federation/metrics/StateStoreMetrics.java   |   5 +
 .../driver/StateStoreRecordOperations.java  |  15 -
 .../driver/impl/StateStoreFileBaseImpl.java | 433 ++-
 .../store/driver/impl/StateStoreFileImpl.java   | 109 ++---
 .../driver/impl/StateStoreFileSystemImpl.java   | 128 +++---
 .../driver/impl/StateStoreZooKeeperImpl.java|   6 -
 .../store/driver/TestStateStoreDriverBase.java  |   9 +
 .../store/driver/TestStateStoreFile.java|  12 +
 .../store/driver/TestStateStoreFileBase.java|  47 ++
 .../store/driver/TestStateStoreFileSystem.java  |  14 +-
 10 files changed, 428 insertions(+), 350 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3d56cb8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
index 40dcd40..09253a2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
@@ -140,5 +140,10 @@ public final class StateStoreMetrics implements 
StateStoreMBean {
 writes.resetMinMax();
 removes.resetMinMax();
 failures.resetMinMax();
+
+reads.lastStat().reset();
+writes.lastStat().reset();
+removes.lastStat().reset();
+failures.lastStat().reset();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3d56cb8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
index e76a733..443d46e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -49,21 +49,6 @@ public interface StateStoreRecordOperations {
QueryResult get(Class clazz) throws IOException;
 
   /**
-   * Get all records of the requested record class from the data store. To use
-   * the default implementations in this class, getAll must return new 
instances
-   * of the records on each call. It is recommended to override the default
-   * implementations for better performance.
-   *
-   * @param clazz Class of record to fetch.
-   * @param sub Sub path.
-   * @return List of all records that match the clazz and the sub path.
-   * @throws IOException
-   */
-  @Idempotent
-   QueryResult get(Class clazz, String sub)
-  throws IOException;
-
-  /**
* Get a single record from the store that matches the query.
*
* @param clazz Class of record to fetch.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3d56cb8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
index a0cd878f..6638d1c 100644
---

hadoop git commit: HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo Goiri.

2018-03-13 Thread yqlin 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 d7aa93b2f -> 19521f71d


HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo 
Goiri.

(cherry picked from commit 76be6cbf6c33f866794f27ca2560ca7c7b2fa0e7)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19521f71
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19521f71
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19521f71

Branch: refs/heads/branch-3.1
Commit: 19521f71d445d61434af380840a285e06503eed1
Parents: d7aa93b
Author: Yiqun Lin 
Authored: Wed Mar 14 11:20:59 2018 +0800
Committer: Yiqun Lin 
Committed: Wed Mar 14 11:23:01 2018 +0800

--
 .../federation/metrics/StateStoreMetrics.java   |   5 +
 .../driver/StateStoreRecordOperations.java  |  15 -
 .../driver/impl/StateStoreFileBaseImpl.java | 433 ++-
 .../store/driver/impl/StateStoreFileImpl.java   | 109 ++---
 .../driver/impl/StateStoreFileSystemImpl.java   | 128 +++---
 .../driver/impl/StateStoreZooKeeperImpl.java|   6 -
 .../store/driver/TestStateStoreDriverBase.java  |   9 +
 .../store/driver/TestStateStoreFile.java|  12 +
 .../store/driver/TestStateStoreFileBase.java|  47 ++
 .../store/driver/TestStateStoreFileSystem.java  |  14 +-
 10 files changed, 428 insertions(+), 350 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/19521f71/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
index 40dcd40..09253a2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
@@ -140,5 +140,10 @@ public final class StateStoreMetrics implements 
StateStoreMBean {
 writes.resetMinMax();
 removes.resetMinMax();
 failures.resetMinMax();
+
+reads.lastStat().reset();
+writes.lastStat().reset();
+removes.lastStat().reset();
+failures.lastStat().reset();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/19521f71/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
index e76a733..443d46e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -49,21 +49,6 @@ public interface StateStoreRecordOperations {
QueryResult get(Class clazz) throws IOException;
 
   /**
-   * Get all records of the requested record class from the data store. To use
-   * the default implementations in this class, getAll must return new 
instances
-   * of the records on each call. It is recommended to override the default
-   * implementations for better performance.
-   *
-   * @param clazz Class of record to fetch.
-   * @param sub Sub path.
-   * @return List of all records that match the clazz and the sub path.
-   * @throws IOException
-   */
-  @Idempotent
-   QueryResult get(Class clazz, String sub)
-  throws IOException;
-
-  /**
* Get a single record from the store that matches the query.
*
* @param clazz Class of record to fetch.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/19521f71/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
index a0cd878f..6638d1c 100644
---

hadoop git commit: HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo Goiri.

2018-03-13 Thread yqlin 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 427fd027a -> 76be6cbf6


HDFS-12773. RBF: Improve State Store FS implementation. Contributed by Inigo 
Goiri.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76be6cbf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76be6cbf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76be6cbf

Branch: refs/heads/trunk
Commit: 76be6cbf6c33f866794f27ca2560ca7c7b2fa0e7
Parents: 427fd02
Author: Yiqun Lin 
Authored: Wed Mar 14 11:20:59 2018 +0800
Committer: Yiqun Lin 
Committed: Wed Mar 14 11:20:59 2018 +0800

--
 .../federation/metrics/StateStoreMetrics.java   |   5 +
 .../driver/StateStoreRecordOperations.java  |  15 -
 .../driver/impl/StateStoreFileBaseImpl.java | 433 ++-
 .../store/driver/impl/StateStoreFileImpl.java   | 109 ++---
 .../driver/impl/StateStoreFileSystemImpl.java   | 128 +++---
 .../driver/impl/StateStoreZooKeeperImpl.java|   6 -
 .../store/driver/TestStateStoreDriverBase.java  |   9 +
 .../store/driver/TestStateStoreFile.java|  12 +
 .../store/driver/TestStateStoreFileBase.java|  47 ++
 .../store/driver/TestStateStoreFileSystem.java  |  14 +-
 10 files changed, 428 insertions(+), 350 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/76be6cbf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
index 40dcd40..09253a2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/StateStoreMetrics.java
@@ -140,5 +140,10 @@ public final class StateStoreMetrics implements 
StateStoreMBean {
 writes.resetMinMax();
 removes.resetMinMax();
 failures.resetMinMax();
+
+reads.lastStat().reset();
+writes.lastStat().reset();
+removes.lastStat().reset();
+failures.lastStat().reset();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76be6cbf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
index e76a733..443d46e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/StateStoreRecordOperations.java
@@ -49,21 +49,6 @@ public interface StateStoreRecordOperations {
QueryResult get(Class clazz) throws IOException;
 
   /**
-   * Get all records of the requested record class from the data store. To use
-   * the default implementations in this class, getAll must return new 
instances
-   * of the records on each call. It is recommended to override the default
-   * implementations for better performance.
-   *
-   * @param clazz Class of record to fetch.
-   * @param sub Sub path.
-   * @return List of all records that match the clazz and the sub path.
-   * @throws IOException
-   */
-  @Idempotent
-   QueryResult get(Class clazz, String sub)
-  throws IOException;
-
-  /**
* Get a single record from the store that matches the query.
*
* @param clazz Class of record to fetch.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76be6cbf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
index a0cd878f..6638d1c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/driver/impl/StateStoreFileBaseImpl.java
+++