[2/2] accumulo git commit: Merge branch '1.5' into 1.6
Merge branch '1.5' into 1.6 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0bcbab7d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0bcbab7d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0bcbab7d Branch: refs/heads/1.6 Commit: 0bcbab7d6ab8bd0523b630bda1a1ff254e68f1cf Parents: dd9cc1c 5ac1b52 Author: Keith Turner ktur...@apache.org Authored: Fri Apr 24 19:39:04 2015 -0400 Committer: Keith Turner ktur...@apache.org Committed: Fri Apr 24 19:39:04 2015 -0400 -- .../system/SourceSwitchingIterator.java | 58 +++- 1 file changed, 33 insertions(+), 25 deletions(-) --
[1/4] accumulo git commit: ACCUMULO-3745 simplify locking and added comments
Repository: accumulo
Updated Branches:
refs/heads/master 51f39d292 - 488f441f7
ACCUMULO-3745 simplify locking and added comments
changes from [~elserj] CR on issue
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5ac1b52e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5ac1b52e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5ac1b52e
Branch: refs/heads/master
Commit: 5ac1b52efcd94b939773ef88f9f4f0bfa4fccaa2
Parents: 95e234c
Author: Keith Turner ke...@deenlo.com
Authored: Thu Apr 23 12:00:36 2015 -0400
Committer: Keith Turner ktur...@apache.org
Committed: Fri Apr 24 19:09:37 2015 -0400
--
.../system/SourceSwitchingIterator.java | 58 +++-
1 file changed, 33 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5ac1b52e/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
b/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
index 7684352..ec73c27 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
@@ -19,7 +19,6 @@ package org.apache.accumulo.core.iterators.system;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -63,17 +62,28 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
private boolean onlySwitchAfterRow;
+ // Synchronization on copies synchronizes operations across all deep copies
of this instance.
+ //
+ // This implementation assumes that there is one thread reading data (a
scan) from all deep copies
+ // and that another thread may call switch at any point. A single scan may
have multiple deep
+ // copies of this iterator if other iterators above this one duplicate their
source. For example,
+ // if an IntersectingIterator over two columns was configured, `copies`
would contain two SSIs
+ // instead of just one SSI. The two instances in `copies` would both be at
the same level
+ // in the tree of iterators for the scan. If multiple instances of SSI are
configure in the iterator
+ // tree (e.g. priority 8 and priority 12), each instance would share their
own `copies` e.g.
+ // SSI@priority8:copies1[...], SSI@priority12:copies2[...]
+
private final ListSourceSwitchingIterator copies;
private SourceSwitchingIterator(DataSource source, boolean
onlySwitchAfterRow, ListSourceSwitchingIterator copies) {
this.source = source;
this.onlySwitchAfterRow = onlySwitchAfterRow;
this.copies = copies;
+copies.add(this);
}
public SourceSwitchingIterator(DataSource source, boolean
onlySwitchAfterRow) {
-this(source, onlySwitchAfterRow, Collections.synchronizedList(new
ArrayListSourceSwitchingIterator()));
-copies.add(this);
+this(source, onlySwitchAfterRow, new ArrayListSourceSwitchingIterator());
}
public SourceSwitchingIterator(DataSource source) {
@@ -83,11 +93,7 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
@Override
public SortedKeyValueIteratorKey,Value deepCopy(IteratorEnvironment env) {
synchronized (copies) {
- synchronized(this){
-SourceSwitchingIterator ssi = new
SourceSwitchingIterator(source.getDeepCopyDataSource(env), onlySwitchAfterRow,
copies);
-copies.add(ssi);
-return ssi;
- }
+ return new SourceSwitchingIterator(source.getDeepCopyDataSource(env),
onlySwitchAfterRow, copies);
}
}
@@ -113,10 +119,12 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
@Override
public void next() throws IOException {
-readNext(false);
+synchronized (copies) {
+ readNext(false);
+}
}
- private synchronized void readNext(boolean initialSeek) throws IOException {
+ private void readNext(boolean initialSeek) throws IOException {
// check of initialSeek second is intentional so that it does not short
// circuit the call to switchSource
@@ -162,18 +170,20 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
}
@Override
- public synchronized void seek(Range range, CollectionByteSequence
columnFamilies, boolean inclusive) throws IOException {
-this.range = range;
-this.inclusive = inclusive;
-this.columnFamilies =
[2/3] accumulo git commit: Merge branch '1.5' into 1.6
Merge branch '1.5' into 1.6 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0bcbab7d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0bcbab7d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0bcbab7d Branch: refs/heads/1.7 Commit: 0bcbab7d6ab8bd0523b630bda1a1ff254e68f1cf Parents: dd9cc1c 5ac1b52 Author: Keith Turner ktur...@apache.org Authored: Fri Apr 24 19:39:04 2015 -0400 Committer: Keith Turner ktur...@apache.org Committed: Fri Apr 24 19:39:04 2015 -0400 -- .../system/SourceSwitchingIterator.java | 58 +++- 1 file changed, 33 insertions(+), 25 deletions(-) --
[3/3] accumulo git commit: Merge branch '1.6' into 1.7
Merge branch '1.6' into 1.7 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/18650ac6 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/18650ac6 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/18650ac6 Branch: refs/heads/1.7 Commit: 18650ac66d575659660efe33e8a505bf45765cfd Parents: 55981ad 0bcbab7 Author: Keith Turner ktur...@apache.org Authored: Fri Apr 24 19:39:32 2015 -0400 Committer: Keith Turner ktur...@apache.org Committed: Fri Apr 24 19:39:32 2015 -0400 -- .../system/SourceSwitchingIterator.java | 58 +++- 1 file changed, 33 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/18650ac6/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java --
accumulo git commit: ACCUMULO-3745 simplify locking and added comments
Repository: accumulo
Updated Branches:
refs/heads/1.5 95e234c7e - 5ac1b52ef
ACCUMULO-3745 simplify locking and added comments
changes from [~elserj] CR on issue
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5ac1b52e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5ac1b52e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5ac1b52e
Branch: refs/heads/1.5
Commit: 5ac1b52efcd94b939773ef88f9f4f0bfa4fccaa2
Parents: 95e234c
Author: Keith Turner ke...@deenlo.com
Authored: Thu Apr 23 12:00:36 2015 -0400
Committer: Keith Turner ktur...@apache.org
Committed: Fri Apr 24 19:09:37 2015 -0400
--
.../system/SourceSwitchingIterator.java | 58 +++-
1 file changed, 33 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5ac1b52e/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
b/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
index 7684352..ec73c27 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
@@ -19,7 +19,6 @@ package org.apache.accumulo.core.iterators.system;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -63,17 +62,28 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
private boolean onlySwitchAfterRow;
+ // Synchronization on copies synchronizes operations across all deep copies
of this instance.
+ //
+ // This implementation assumes that there is one thread reading data (a
scan) from all deep copies
+ // and that another thread may call switch at any point. A single scan may
have multiple deep
+ // copies of this iterator if other iterators above this one duplicate their
source. For example,
+ // if an IntersectingIterator over two columns was configured, `copies`
would contain two SSIs
+ // instead of just one SSI. The two instances in `copies` would both be at
the same level
+ // in the tree of iterators for the scan. If multiple instances of SSI are
configure in the iterator
+ // tree (e.g. priority 8 and priority 12), each instance would share their
own `copies` e.g.
+ // SSI@priority8:copies1[...], SSI@priority12:copies2[...]
+
private final ListSourceSwitchingIterator copies;
private SourceSwitchingIterator(DataSource source, boolean
onlySwitchAfterRow, ListSourceSwitchingIterator copies) {
this.source = source;
this.onlySwitchAfterRow = onlySwitchAfterRow;
this.copies = copies;
+copies.add(this);
}
public SourceSwitchingIterator(DataSource source, boolean
onlySwitchAfterRow) {
-this(source, onlySwitchAfterRow, Collections.synchronizedList(new
ArrayListSourceSwitchingIterator()));
-copies.add(this);
+this(source, onlySwitchAfterRow, new ArrayListSourceSwitchingIterator());
}
public SourceSwitchingIterator(DataSource source) {
@@ -83,11 +93,7 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
@Override
public SortedKeyValueIteratorKey,Value deepCopy(IteratorEnvironment env) {
synchronized (copies) {
- synchronized(this){
-SourceSwitchingIterator ssi = new
SourceSwitchingIterator(source.getDeepCopyDataSource(env), onlySwitchAfterRow,
copies);
-copies.add(ssi);
-return ssi;
- }
+ return new SourceSwitchingIterator(source.getDeepCopyDataSource(env),
onlySwitchAfterRow, copies);
}
}
@@ -113,10 +119,12 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
@Override
public void next() throws IOException {
-readNext(false);
+synchronized (copies) {
+ readNext(false);
+}
}
- private synchronized void readNext(boolean initialSeek) throws IOException {
+ private void readNext(boolean initialSeek) throws IOException {
// check of initialSeek second is intentional so that it does not short
// circuit the call to switchSource
@@ -162,18 +170,20 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
}
@Override
- public synchronized void seek(Range range, CollectionByteSequence
columnFamilies, boolean inclusive) throws IOException {
-this.range = range;
-this.inclusive = inclusive;
-this.columnFamilies = columnFamilies;
+
[1/2] accumulo git commit: ACCUMULO-3745 simplify locking and added comments
Repository: accumulo
Updated Branches:
refs/heads/1.6 dd9cc1cfb - 0bcbab7d6
ACCUMULO-3745 simplify locking and added comments
changes from [~elserj] CR on issue
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5ac1b52e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5ac1b52e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5ac1b52e
Branch: refs/heads/1.6
Commit: 5ac1b52efcd94b939773ef88f9f4f0bfa4fccaa2
Parents: 95e234c
Author: Keith Turner ke...@deenlo.com
Authored: Thu Apr 23 12:00:36 2015 -0400
Committer: Keith Turner ktur...@apache.org
Committed: Fri Apr 24 19:09:37 2015 -0400
--
.../system/SourceSwitchingIterator.java | 58 +++-
1 file changed, 33 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5ac1b52e/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
b/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
index 7684352..ec73c27 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
@@ -19,7 +19,6 @@ package org.apache.accumulo.core.iterators.system;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -63,17 +62,28 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
private boolean onlySwitchAfterRow;
+ // Synchronization on copies synchronizes operations across all deep copies
of this instance.
+ //
+ // This implementation assumes that there is one thread reading data (a
scan) from all deep copies
+ // and that another thread may call switch at any point. A single scan may
have multiple deep
+ // copies of this iterator if other iterators above this one duplicate their
source. For example,
+ // if an IntersectingIterator over two columns was configured, `copies`
would contain two SSIs
+ // instead of just one SSI. The two instances in `copies` would both be at
the same level
+ // in the tree of iterators for the scan. If multiple instances of SSI are
configure in the iterator
+ // tree (e.g. priority 8 and priority 12), each instance would share their
own `copies` e.g.
+ // SSI@priority8:copies1[...], SSI@priority12:copies2[...]
+
private final ListSourceSwitchingIterator copies;
private SourceSwitchingIterator(DataSource source, boolean
onlySwitchAfterRow, ListSourceSwitchingIterator copies) {
this.source = source;
this.onlySwitchAfterRow = onlySwitchAfterRow;
this.copies = copies;
+copies.add(this);
}
public SourceSwitchingIterator(DataSource source, boolean
onlySwitchAfterRow) {
-this(source, onlySwitchAfterRow, Collections.synchronizedList(new
ArrayListSourceSwitchingIterator()));
-copies.add(this);
+this(source, onlySwitchAfterRow, new ArrayListSourceSwitchingIterator());
}
public SourceSwitchingIterator(DataSource source) {
@@ -83,11 +93,7 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
@Override
public SortedKeyValueIteratorKey,Value deepCopy(IteratorEnvironment env) {
synchronized (copies) {
- synchronized(this){
-SourceSwitchingIterator ssi = new
SourceSwitchingIterator(source.getDeepCopyDataSource(env), onlySwitchAfterRow,
copies);
-copies.add(ssi);
-return ssi;
- }
+ return new SourceSwitchingIterator(source.getDeepCopyDataSource(env),
onlySwitchAfterRow, copies);
}
}
@@ -113,10 +119,12 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
@Override
public void next() throws IOException {
-readNext(false);
+synchronized (copies) {
+ readNext(false);
+}
}
- private synchronized void readNext(boolean initialSeek) throws IOException {
+ private void readNext(boolean initialSeek) throws IOException {
// check of initialSeek second is intentional so that it does not short
// circuit the call to switchSource
@@ -162,18 +170,20 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
}
@Override
- public synchronized void seek(Range range, CollectionByteSequence
columnFamilies, boolean inclusive) throws IOException {
-this.range = range;
-this.inclusive = inclusive;
-this.columnFamilies = columnFamilies;
+
[1/3] accumulo git commit: ACCUMULO-3745 simplify locking and added comments
Repository: accumulo
Updated Branches:
refs/heads/1.7 55981ad88 - 18650ac66
ACCUMULO-3745 simplify locking and added comments
changes from [~elserj] CR on issue
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5ac1b52e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5ac1b52e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5ac1b52e
Branch: refs/heads/1.7
Commit: 5ac1b52efcd94b939773ef88f9f4f0bfa4fccaa2
Parents: 95e234c
Author: Keith Turner ke...@deenlo.com
Authored: Thu Apr 23 12:00:36 2015 -0400
Committer: Keith Turner ktur...@apache.org
Committed: Fri Apr 24 19:09:37 2015 -0400
--
.../system/SourceSwitchingIterator.java | 58 +++-
1 file changed, 33 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5ac1b52e/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
b/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
index 7684352..ec73c27 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java
@@ -19,7 +19,6 @@ package org.apache.accumulo.core.iterators.system;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -63,17 +62,28 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
private boolean onlySwitchAfterRow;
+ // Synchronization on copies synchronizes operations across all deep copies
of this instance.
+ //
+ // This implementation assumes that there is one thread reading data (a
scan) from all deep copies
+ // and that another thread may call switch at any point. A single scan may
have multiple deep
+ // copies of this iterator if other iterators above this one duplicate their
source. For example,
+ // if an IntersectingIterator over two columns was configured, `copies`
would contain two SSIs
+ // instead of just one SSI. The two instances in `copies` would both be at
the same level
+ // in the tree of iterators for the scan. If multiple instances of SSI are
configure in the iterator
+ // tree (e.g. priority 8 and priority 12), each instance would share their
own `copies` e.g.
+ // SSI@priority8:copies1[...], SSI@priority12:copies2[...]
+
private final ListSourceSwitchingIterator copies;
private SourceSwitchingIterator(DataSource source, boolean
onlySwitchAfterRow, ListSourceSwitchingIterator copies) {
this.source = source;
this.onlySwitchAfterRow = onlySwitchAfterRow;
this.copies = copies;
+copies.add(this);
}
public SourceSwitchingIterator(DataSource source, boolean
onlySwitchAfterRow) {
-this(source, onlySwitchAfterRow, Collections.synchronizedList(new
ArrayListSourceSwitchingIterator()));
-copies.add(this);
+this(source, onlySwitchAfterRow, new ArrayListSourceSwitchingIterator());
}
public SourceSwitchingIterator(DataSource source) {
@@ -83,11 +93,7 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
@Override
public SortedKeyValueIteratorKey,Value deepCopy(IteratorEnvironment env) {
synchronized (copies) {
- synchronized(this){
-SourceSwitchingIterator ssi = new
SourceSwitchingIterator(source.getDeepCopyDataSource(env), onlySwitchAfterRow,
copies);
-copies.add(ssi);
-return ssi;
- }
+ return new SourceSwitchingIterator(source.getDeepCopyDataSource(env),
onlySwitchAfterRow, copies);
}
}
@@ -113,10 +119,12 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
@Override
public void next() throws IOException {
-readNext(false);
+synchronized (copies) {
+ readNext(false);
+}
}
- private synchronized void readNext(boolean initialSeek) throws IOException {
+ private void readNext(boolean initialSeek) throws IOException {
// check of initialSeek second is intentional so that it does not short
// circuit the call to switchSource
@@ -162,18 +170,20 @@ public class SourceSwitchingIterator implements
SortedKeyValueIteratorKey,Value
}
@Override
- public synchronized void seek(Range range, CollectionByteSequence
columnFamilies, boolean inclusive) throws IOException {
-this.range = range;
-this.inclusive = inclusive;
-this.columnFamilies = columnFamilies;
+
[2/4] accumulo git commit: Merge branch '1.5' into 1.6
Merge branch '1.5' into 1.6 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0bcbab7d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0bcbab7d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0bcbab7d Branch: refs/heads/master Commit: 0bcbab7d6ab8bd0523b630bda1a1ff254e68f1cf Parents: dd9cc1c 5ac1b52 Author: Keith Turner ktur...@apache.org Authored: Fri Apr 24 19:39:04 2015 -0400 Committer: Keith Turner ktur...@apache.org Committed: Fri Apr 24 19:39:04 2015 -0400 -- .../system/SourceSwitchingIterator.java | 58 +++- 1 file changed, 33 insertions(+), 25 deletions(-) --
[3/4] accumulo git commit: Merge branch '1.6' into 1.7
Merge branch '1.6' into 1.7 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/18650ac6 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/18650ac6 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/18650ac6 Branch: refs/heads/master Commit: 18650ac66d575659660efe33e8a505bf45765cfd Parents: 55981ad 0bcbab7 Author: Keith Turner ktur...@apache.org Authored: Fri Apr 24 19:39:32 2015 -0400 Committer: Keith Turner ktur...@apache.org Committed: Fri Apr 24 19:39:32 2015 -0400 -- .../system/SourceSwitchingIterator.java | 58 +++- 1 file changed, 33 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/18650ac6/core/src/main/java/org/apache/accumulo/core/iterators/system/SourceSwitchingIterator.java --
[08/34] accumulo git commit: ACCUMULO-3638 merge master branch
ACCUMULO-3638 merge master branch Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4635de86 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4635de86 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4635de86 Branch: refs/heads/master Commit: 4635de8671a62f0a412da775db05519f6831daaa Parents: 98c3cef 9f108c0 Author: Eric C. Newton eric.new...@gmail.com Authored: Tue Mar 10 16:16:53 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Tue Mar 10 16:16:53 2015 -0400 -- core/pom.xml| 7 -- core/src/main/findbugs/exclude-filter.xml | 67 +- .../client/admin/CompactionStrategyConfig.java | 5 ++ .../client/impl/MultiTableBatchWriterImpl.java | 13 ++-- .../core/client/impl/TabletLocatorImpl.java | 41 +++ .../client/impl/TabletServerBatchWriter.java| 45 .../core/client/mock/MockNamespace.java | 2 +- .../client/security/tokens/DelegationToken.java | 16 ++--- .../client/security/tokens/PasswordToken.java | 13 ++-- .../apache/accumulo/core/conf/PropertyType.java | 2 +- .../org/apache/accumulo/core/data/Value.java| 1 + .../apache/accumulo/core/volume/VolumeImpl.java | 6 ++ .../accumulo/core/cli/TestClientOpts.java | 2 +- .../core/file/rfile/RFileMetricsTest.java | 73 +--- .../iterators/user/ColumnSliceFilterTest.java | 2 +- .../core/iterators/user/RowFilterTest.java | 2 +- .../simple/src/main/findbugs/exclude-filter.xml | 18 + .../examples/simple/mapreduce/RowHash.java | 2 +- fate/src/main/findbugs/exclude-filter.xml | 18 + .../accumulo/fate/zookeeper/ZooReader.java | 14 ++-- .../zookeeper/DistributedReadWriteLockTest.java | 17 +++-- .../src/main/findbugs/exclude-filter.xml| 18 + .../src/main/findbugs/exclude-filter.xml| 30 pom.xml | 3 +- proxy/pom.xml | 13 proxy/src/main/findbugs/exclude-filter.xml | 7 +- .../base/src/main/findbugs/exclude-filter.xml | 26 +++ .../server/master/balancer/GroupBalancer.java | 6 ++ .../master/state/TabletLocationState.java | 3 +- .../server/metrics/MetricsConfiguration.java| 7 +- .../TCredentialsUpdatingInvocationHandler.java | 5 +- .../server/security/SecurityOperation.java | 7 -- .../security/delegation/AuthenticationKey.java | 11 +-- .../security/handler/KerberosAuthenticator.java | 4 +- .../server/watcher/Log4jConfiguration.java | 5 +- .../accumulo/server/ServerConstantsTest.java| 12 +++- .../AuthenticationTokenKeyManagerTest.java | 34 + server/gc/src/main/findbugs/exclude-filter.xml | 18 + .../gc/GarbageCollectWriteAheadLogs.java| 2 +- .../master/src/main/findbugs/exclude-filter.xml | 23 ++ .../java/org/apache/accumulo/master/Master.java | 24 --- .../RemoveCompleteReplicationRecords.java | 3 + .../src/main/findbugs/exclude-filter.xml| 18 + .../monitor/servlets/OperationServlet.java | 53 +- server/tracer/pom.xml | 13 .../tracer/src/main/findbugs/exclude-filter.xml | 7 +- .../src/main/findbugs/exclude-filter.xml| 24 +++ .../accumulo/server/logger/LogFileKey.java | 25 --- .../accumulo/server/logger/LogFileValue.java| 25 --- .../accumulo/tserver/log/LocalWALRecovery.java | 4 +- .../replication/AccumuloReplicaSystem.java | 19 - .../apache/accumulo/tserver/scan/ScanTask.java | 30 ++-- .../apache/accumulo/tserver/tablet/Tablet.java | 14 ++-- .../tserver/tablet/TabletCommitter.java | 3 + shell/src/main/findbugs/exclude-filter.xml | 18 + .../apache/accumulo/shell/ShellConfigTest.java | 29 ++-- .../org/apache/accumulo/shell/ShellTest.java| 11 ++- .../shell/commands/FormatterCommandTest.java| 6 +- start/src/main/findbugs/exclude-filter.xml | 18 + .../start/classloader/vfs/MiniDFSUtil.java | 4 ++ test/src/main/findbugs/exclude-filter.xml | 30 .../server/security/SystemCredentialsIT.java| 2 + .../test/MasterRepairsDualAssignmentIT.java | 2 +- .../accumulo/test/MultiTableRecoveryIT.java | 3 +- .../java/org/apache/accumulo/test/UsersIT.java | 60 .../apache/accumulo/test/functional/BulkIT.java | 13 ++-- .../test/functional/ConfigurableMacIT.java | 4 +- .../accumulo/test/functional/MonitorSslIT.java | 40 ++- .../accumulo/test/functional/PermissionsIT.java | 43 .../apache/accumulo/test/functional/SslIT.java | 5 +- trace/src/main/findbugs/exclude-filter.xml | 26 +++ 71 files changed, 810 insertions(+), 366 deletions(-)
[27/34] accumulo git commit: ACCUMULO-3423 optimize WAL metadata table updates
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3fdd29f5/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
--
diff --git
a/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
b/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
new file mode 100644
index 000..490bd7c
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.apache.accumulo.core.conf.Property.GC_CYCLE_DELAY;
+import static org.apache.accumulo.core.conf.Property.GC_CYCLE_START;
+import static org.apache.accumulo.core.conf.Property.INSTANCE_ZK_TIMEOUT;
+import static org.apache.accumulo.core.conf.Property.TSERV_WALOG_MAX_SIZE;
+import static org.apache.accumulo.core.conf.Property.TSERV_WAL_REPLICATION;
+import static org.apache.accumulo.core.security.Authorizations.EMPTY;
+import static org.apache.accumulo.minicluster.ServerType.GARBAGE_COLLECTOR;
+import static org.apache.accumulo.minicluster.ServerType.TABLET_SERVER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.CurrentLogsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.master.state.SetGoalState;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterControl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class WALSunnyDayIT extends ConfigurableMacIT {
+
+ private static final Text CF = new Text(new byte[0]);
+
+ @Override
+ protected void configure(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
+cfg.setProperty(GC_CYCLE_DELAY, 1s);
+cfg.setProperty(GC_CYCLE_START, 0s);
+cfg.setProperty(TSERV_WALOG_MAX_SIZE, 1M);
+cfg.setProperty(TSERV_WAL_REPLICATION, 1);
+cfg.setProperty(INSTANCE_ZK_TIMEOUT, 3s);
+cfg.setNumTservers(1);
+hadoopCoreSite.set(fs.file.impl, RawLocalFileSystem.class.getName());
+ }
+
+ int countTrue(CollectionBoolean bools) {
+int result = 0;
+for (Boolean b : bools) {
+ if (b.booleanValue())
+result ++;
+}
+return result;
+ }
+
+ @Test
+ public void test() throws Exception {
+MiniAccumuloClusterImpl mac = getCluster();
+MiniAccumuloClusterControl control = mac.getClusterControl();
+control.stop(GARBAGE_COLLECTOR);
+Connector c = getConnector();
+ZooKeeper zoo = new ZooKeeper(c.getInstance().getZooKeepers(),
c.getInstance().getZooKeepersSessionTimeOut(), new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+log.info(event.toString());
+ }
+});
+String tableName = getUniqueNames(1)[0];
[15/34] accumulo git commit: ACCUMULO-3423 updates based on review by [~elserj] and [~kturner]
ACCUMULO-3423 updates based on review by [~elserj] and [~kturner]
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/afa887b6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/afa887b6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/afa887b6
Branch: refs/heads/master
Commit: afa887b6f5f131a06497eaf1d04ba8c55b0d2877
Parents: daa38ce
Author: Eric C. Newton eric.new...@gmail.com
Authored: Mon Mar 30 11:25:19 2015 -0400
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Mon Mar 30 11:25:19 2015 -0400
--
.../core/metadata/schema/MetadataSchema.java| 2 +-
.../core/metadata/MetadataTableSchemaTest.java | 10 +-
.../server/master/state/MetaDataStateStore.java | 18 +--
.../master/state/TabletLocationState.java | 2 -
.../accumulo/server/util/ListVolumesUsed.java | 15 ++
.../server/util/MasterMetadataUtil.java | 2 +-
.../accumulo/server/util/MetadataTableUtil.java | 63 +---
.../gc/GarbageCollectWriteAheadLogs.java| 33 +++--
.../CloseWriteAheadLogReferences.java | 2 -
.../accumulo/master/TabletGroupWatcher.java | 2 +
.../server/GarbageCollectionLogger.java | 2 +-
.../apache/accumulo/tserver/TabletServer.java | 3 +
.../tserver/log/TabletServerLogger.java | 22 +--
.../apache/accumulo/tserver/tablet/Tablet.java | 4 +-
.../accumulo/tserver/log/LogEntryTest.java | 56
.../org/apache/accumulo/test/UnusedWALIT.java | 144 +++
.../java/org/apache/accumulo/test/VolumeIT.java | 17 +++
17 files changed, 337 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/afa887b6/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
index 88e11f4..d2f7d07 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
@@ -322,7 +322,7 @@ public class MetadataSchema {
Text row = new Text();
k.getRow(row);
- if (row.getLength() section.getRowPrefix().length()) {
+ if (!row.toString().startsWith(section.getRowPrefix())) {
throw new IllegalArgumentException(Bad key + k.toString());
}
for (int sessionStart = section.getRowPrefix().length(); sessionStart
row.getLength() - 1; sessionStart++) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/afa887b6/core/src/test/java/org/apache/accumulo/core/metadata/MetadataTableSchemaTest.java
--
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/MetadataTableSchemaTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/MetadataTableSchemaTest.java
index dfc74cf..cfe59f2 100644
---
a/core/src/test/java/org/apache/accumulo/core/metadata/MetadataTableSchemaTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/MetadataTableSchemaTest.java
@@ -18,6 +18,7 @@
package org.apache.accumulo.core.metadata;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import org.apache.accumulo.core.data.Key;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.CurrentLogsSection;
@@ -28,12 +29,19 @@ public class MetadataTableSchemaTest {
@Test
public void testGetTabletServer() throws Exception {
-Key key = new Key(~wal:host:43861[14a7df0e6420003], log,
hdfs://localhost:50514/accumulo/wal/host:43861/70c27ab3-6662-40ab-80fb-01c1f1a59df3);
+Key key = new Key(~wal+host:43861[14a7df0e6420003], log,
hdfs://localhost:50514/accumulo/wal/host:43861/70c27ab3-6662-40ab-80fb-01c1f1a59df3);
Text hostPort = new Text();
Text session = new Text();
CurrentLogsSection.getTabletServer(key, hostPort, session);
assertEquals(host:43861, hostPort.toString());
assertEquals(14a7df0e6420003, session.toString());
+try {
+ Key bogus = new Key(~wal/host:43861[14a7df0e6420003], log,
hdfs://localhost:50514/accumulo/wal/host:43861/70c27ab3-6662-40ab-80fb-01c1f1a59df3);
+ CurrentLogsSection.getTabletServer(bogus, hostPort, session);
+ fail(bad argument not thrown);
+} catch (IllegalArgumentException ex) {
+
+}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/afa887b6/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
--
diff --git
[07/34] accumulo git commit: ACCUMULO-3638 mostly updated file references to WALs to be Path objects
ACCUMULO-3638 mostly updated file references to WALs to be Path objects
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/98c3cef8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/98c3cef8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/98c3cef8
Branch: refs/heads/master
Commit: 98c3cef8ccfccbe84a5c35ce7576a9225ee03051
Parents: 902ee7d
Author: Eric C. Newton eric.new...@gmail.com
Authored: Tue Mar 10 14:07:43 2015 -0400
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Tue Mar 10 14:07:43 2015 -0400
--
.../server/master/state/MetaDataStateStore.java | 17 +-
.../master/state/TabletLocationState.java | 3 +
.../server/master/state/TabletStateStore.java | 7 +-
.../master/state/ZooTabletStateStore.java | 11 +-
.../accumulo/server/util/MetadataTableUtil.java | 31 ++-
.../gc/GarbageCollectWriteAheadLogs.java| 79 +++---
.../accumulo/master/TabletGroupWatcher.java | 15 +-
.../apache/accumulo/tserver/TabletServer.java | 6 +-
.../apache/accumulo/tserver/log/DfsLogger.java | 6 +-
.../accumulo/test/functional/WALSunnyDayIT.java | 240 +++
10 files changed, 334 insertions(+), 81 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/98c3cef8/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
index 1749904..decc8c7 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.accumulo.server.AccumuloServerContext;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
@@ -123,7 +124,7 @@ public class MetaDataStateStore extends TabletStateStore {
}
@Override
- public void unassign(CollectionTabletLocationState tablets,
MapTServerInstance, ListString logsForDeadServers) throws
DistributedStoreException {
+ public void unassign(CollectionTabletLocationState tablets,
MapTServerInstance, ListPath logsForDeadServers) throws
DistributedStoreException {
BatchWriter writer = createBatchWriter();
try {
@@ -136,10 +137,10 @@ public class MetaDataStateStore extends TabletStateStore {
tls.future.clearFutureLocation(m);
}
if (logsForDeadServers != null) {
- ListString logs = logsForDeadServers.get(tls.futureOrCurrent());
+ ListPath logs = logsForDeadServers.get(tls.futureOrCurrent());
if (logs != null) {
-for (String log : logs) {
- LogEntry entry = new LogEntry(tls.extent, 0,
tls.futureOrCurrent().hostPort(), log);
+for (Path log : logs) {
+ LogEntry entry = new LogEntry(tls.extent, 0,
tls.futureOrCurrent().hostPort(), log.toString());
m.put(entry.getColumnFamily(), entry.getColumnQualifier(),
entry.getValue());
}
}
@@ -163,13 +164,13 @@ public class MetaDataStateStore extends TabletStateStore {
}
@Override
- public void markLogsAsUnused(AccumuloServerContext context,
MapTServerInstance,ListString logs) throws DistributedStoreException {
+ public void markLogsAsUnused(AccumuloServerContext context,
MapTServerInstance,ListPath logs) throws DistributedStoreException {
BatchWriter writer = createBatchWriter();
try {
- for (EntryTServerInstance,ListString entry : logs.entrySet()) {
+ for (EntryTServerInstance,ListPath entry : logs.entrySet()) {
Mutation m = new
Mutation(MetadataSchema.CurrentLogsSection.getRowPrefix() +
entry.getKey().toString());
-for (String log : entry.getValue()) {
- m.put(MetadataSchema.CurrentLogsSection.COLF, new Text(log),
MetadataSchema.CurrentLogsSection.UNUSED);
+for (Path log : entry.getValue()) {
+ m.put(MetadataSchema.CurrentLogsSection.COLF, new
Text(log.toString()), MetadataSchema.CurrentLogsSection.UNUSED);
}
writer.addMutation(m);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/98c3cef8/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
--
diff --git
[21/34] accumulo git commit: Merge branch 'master' of http://git-wip-us.apache.org/repos/asf/accumulo
Merge branch 'master' of http://git-wip-us.apache.org/repos/asf/accumulo Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bd6dbba5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bd6dbba5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bd6dbba5 Branch: refs/heads/master Commit: bd6dbba5b871e89aacdfb4b06b223047e9a0a159 Parents: 65d0145 cf69821 Author: Eric Newton eric.new...@gmail.com Authored: Thu Apr 16 12:05:44 2015 -0400 Committer: Eric Newton eric.new...@gmail.com Committed: Thu Apr 16 12:05:44 2015 -0400 -- .../core/client/NewTableConfiguration.java | 108 .../client/admin/CompactionStrategyConfig.java | 8 +- .../client/admin/NewTableConfiguration.java | 107 .../core/client/admin/TableOperations.java | 1 - .../core/client/impl/ConditionalWriterImpl.java | 2 +- .../core/client/impl/TableOperationsImpl.java | 2 +- .../core/client/mapreduce/RangeInputSplit.java | 5 + .../core/client/mock/MockTableOperations.java | 2 +- .../accumulo/core/file/BloomFilterLayer.java| 2 +- .../accumulo/core/util/LoggingRunnable.java | 50 .../accumulo/core/util/NamingThreadFactory.java | 1 + .../client/impl/TableOperationsHelperTest.java | 2 +- .../client/mock/MockTableOperationsTest.java| 2 +- .../minicluster/MiniAccumuloInstance.java | 3 +- .../minicluster/MiniAccumuloClusterTest.java| 2 +- .../org/apache/accumulo/proxy/ProxyServer.java | 2 +- .../accumulo/server/client/BulkImporter.java| 2 +- .../server/problems/ProblemReports.java | 2 +- .../accumulo/server/rpc/TServerUtils.java | 2 +- .../org/apache/accumulo/monitor/Monitor.java| 2 +- .../apache/accumulo/tserver/TabletServer.java | 4 +- .../tserver/TabletServerResourceManager.java| 5 +- .../EverythingCompactionStrategy.java | 2 +- .../apache/accumulo/tserver/log/DfsLogger.java | 2 +- .../tserver/tablet/MinorCompactionTask.java | 8 +- .../apache/accumulo/tserver/tablet/Tablet.java | 63 - .../shell/commands/CreateTableCommand.java | 2 +- .../apache/accumulo/test/HardListIterator.java | 115 + .../accumulo/test/ConditionalWriterIT.java | 2 +- .../test/CreateTableWithNewTableConfigIT.java | 2 +- .../org/apache/accumulo/test/NamespacesIT.java | 2 +- .../apache/accumulo/test/TableOperationsIT.java | 127 +++ .../java/org/apache/accumulo/test/VolumeIT.java | 2 +- .../accumulo/test/functional/LogicalTimeIT.java | 2 +- .../accumulo/test/functional/MergeIT.java | 2 +- .../test/replication/CyclicReplicationIT.java | 2 +- 36 files changed, 423 insertions(+), 226 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/bd6dbba5/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/bd6dbba5/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/bd6dbba5/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/bd6dbba5/test/src/test/java/org/apache/accumulo/test/ConditionalWriterIT.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/bd6dbba5/test/src/test/java/org/apache/accumulo/test/VolumeIT.java --
[06/34] accumulo git commit: ACCUMULO-3638 merge master branch
ACCUMULO-3638 merge master branch Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/902ee7dd Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/902ee7dd Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/902ee7dd Branch: refs/heads/master Commit: 902ee7ddaef13cfd2064fb3c06495955819a3bff Parents: c8f3b7d 87208e5 Author: Eric C. Newton eric.new...@gmail.com Authored: Wed Mar 4 10:54:07 2015 -0500 Committer: Eric C. Newton eric.new...@gmail.com Committed: Wed Mar 4 10:54:07 2015 -0500 -- .../apache/accumulo/core/cli/ClientOpts.java| 36 ++- .../client/impl/SecurityOperationsImpl.java | 10 +- .../core/client/lexicoder/AbstractEncoder.java | 60 .../client/lexicoder/BigIntegerLexicoder.java | 21 +- .../core/client/lexicoder/BytesLexicoder.java | 20 +- .../core/client/lexicoder/DateLexicoder.java| 8 +- .../core/client/lexicoder/DoubleLexicoder.java | 8 +- .../core/client/lexicoder/IntegerLexicoder.java | 8 +- .../core/client/lexicoder/ListLexicoder.java| 14 +- .../core/client/lexicoder/LongLexicoder.java| 4 +- .../core/client/lexicoder/PairLexicoder.java| 11 +- .../core/client/lexicoder/ReverseLexicoder.java | 15 +- .../core/client/lexicoder/StringLexicoder.java | 8 +- .../core/client/lexicoder/TextLexicoder.java| 9 +- .../client/lexicoder/UIntegerLexicoder.java | 16 +- .../core/client/lexicoder/ULongLexicoder.java | 16 +- .../core/client/lexicoder/UUIDLexicoder.java| 13 +- .../lexicoder/impl/AbstractLexicoder.java | 23 ++ .../core/client/lexicoder/impl/ByteUtils.java | 13 +- .../core/client/mapred/AbstractInputFormat.java | 14 +- .../client/mapreduce/AbstractInputFormat.java | 14 +- .../mapreduce/lib/impl/ConfiguratorBase.java| 38 ++- .../client/security/tokens/KerberosToken.java | 27 ++ .../org/apache/accumulo/core/conf/Property.java | 2 + .../accumulo/core/iterators/LongCombiner.java | 27 +- .../core/iterators/user/BigDecimalCombiner.java | 7 +- .../iterators/user/SummingArrayCombiner.java| 14 +- .../lexicoder/BigIntegerLexicoderTest.java | 12 +- .../client/lexicoder/BytesLexicoderTest.java| 28 ++ .../client/lexicoder/DateLexicoderTest.java | 30 ++ .../client/lexicoder/DoubleLexicoderTest.java | 14 +- .../client/lexicoder/IntegerLexicoderTest.java | 12 +- .../core/client/lexicoder/LexicoderTest.java| 31 +- .../client/lexicoder/ListLexicoderTest.java | 33 ++- .../client/lexicoder/LongLexicoderTest.java | 13 +- .../client/lexicoder/PairLexicoderTest.java | 12 +- .../client/lexicoder/ReverseLexicoderTest.java | 15 +- .../client/lexicoder/StringLexicoderTest.java | 28 ++ .../client/lexicoder/TextLexicoderTest.java | 28 ++ .../client/lexicoder/UIntegerLexicoderTest.java | 12 +- .../client/lexicoder/ULongLexicoderTest.java| 13 +- .../client/lexicoder/UUIDLexicoderTest.java | 9 +- .../lexicoder/impl/AbstractLexicoderTest.java | 94 ++ .../client/lexicoder/impl/ByteUtilsTest.java| 71 + .../core/iterators/user/CombinerTest.java | 4 + .../main/asciidoc/chapters/administration.txt | 23 +- docs/src/main/resources/distributedTracing.html | 6 +- .../accumulo/cluster/AccumuloCluster.java | 6 + .../apache/accumulo/cluster/ClusterUser.java| 141 + .../apache/accumulo/cluster/ClusterUsers.java | 39 +++ .../standalone/StandaloneAccumuloCluster.java | 37 ++- .../impl/MiniAccumuloClusterImpl.java | 11 + pom.xml | 4 +- .../server/client/ClientServiceHandler.java | 14 +- .../server/replication/StatusCombiner.java | 24 +- .../server/security/SecurityOperation.java | 2 + .../delegation/ZooAuthenticationKeyWatcher.java | 14 +- .../ZooAuthenticationKeyWatcherTest.java| 24 ++ .../accumulo/monitor/EmbeddedWebServer.java | 2 + .../monitor/servlets/DefaultServlet.java| 4 +- .../accumulo/monitor/servlets/trace/Basic.java | 20 +- .../java/org/apache/accumulo/shell/Shell.java | 3 + .../shell/commands/CreateUserCommand.java | 36 ++- .../accumulo/shell/ShellSetInstanceTest.java| 4 + .../start/classloader/AccumuloClassLoader.java | 22 +- .../org/apache/accumulo/test/TestIngest.java| 5 +- .../accumulo/test/TestMultiTableIngest.java | 2 +- .../accumulo/harness/AccumuloClusterIT.java | 180 .../org/apache/accumulo/harness/AccumuloIT.java | 5 +- .../accumulo/harness/MiniClusterHarness.java| 12 +- .../accumulo/harness/SharedMiniClusterIT.java | 68 - .../org/apache/accumulo/harness/TestingKdc.java | 126 +--- .../conf/AccumuloClusterConfiguration.java | 6 +- .../AccumuloClusterPropertyConfiguration.java | 19 +-
[20/34] accumulo git commit: ACCUMULO-3423 reduce the number of calls to add a flag to the metadata table
ACCUMULO-3423 reduce the number of calls to add a flag to the metadata table
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/65d01458
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/65d01458
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/65d01458
Branch: refs/heads/master
Commit: 65d01458b69c20ec5ae3f83a4beb266481e8c840
Parents: 8f5e002
Author: Eric Newton eric.new...@gmail.com
Authored: Thu Apr 16 12:03:12 2015 -0400
Committer: Eric Newton eric.new...@gmail.com
Committed: Thu Apr 16 12:03:12 2015 -0400
--
.../org/apache/accumulo/server/TabletLevel.java | 34
.../accumulo/server/util/MetadataTableUtil.java | 7 ++--
.../apache/accumulo/tserver/TabletLevel.java| 34
.../apache/accumulo/tserver/TabletServer.java | 16 -
.../tserver/log/TabletServerLogger.java | 10 --
5 files changed, 52 insertions(+), 49 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/65d01458/server/base/src/main/java/org/apache/accumulo/server/TabletLevel.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/TabletLevel.java
b/server/base/src/main/java/org/apache/accumulo/server/TabletLevel.java
new file mode 100644
index 000..91e5ee9
--- /dev/null
+++ b/server/base/src/main/java/org/apache/accumulo/server/TabletLevel.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server;
+
+import org.apache.accumulo.core.data.KeyExtent;
+
+public enum TabletLevel {
+ ROOT,
+ META,
+ NORMAL;
+
+ public static TabletLevel getLevel(KeyExtent extent) {
+if (!extent.isMeta())
+ return NORMAL;
+if (extent.isRootTablet())
+ return ROOT;
+return META;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/65d01458/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 6d6253d..09a5b40 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -81,6 +81,7 @@ import
org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.TabletLevel;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -1057,9 +1058,9 @@ public class MetadataTableUtil {
return tabletEntries;
}
- public static void addNewLogMarker(ClientContext context, ZooLock zooLock,
final TServerInstance tabletSession, final Path filename, KeyExtent extent) {
+ public static void addNewLogMarker(ClientContext context, ZooLock zooLock,
final TServerInstance tabletSession, final Path filename, TabletLevel level) {
log.debug(Adding log entry + filename);
-if (extent.isRootTablet()) {
+if (level == TabletLevel.ROOT) {
retryZooKeeperUpdate(context, zooLock, new ZooOperation() {
@Override
public void run(IZooReaderWriter rw) throws KeeperException,
InterruptedException, IOException {
@@ -1077,7 +1078,7 @@ public class MetadataTableUtil {
Mutation m = new Mutation(CurrentLogsSection.getRowPrefix() +
tabletSession.toString());
m.put(CurrentLogsSection.COLF, new Text(filename.toString()), new
Value(EMPTY_BYTES));
String tableName = MetadataTable.NAME;
- if (extent.isMeta()) {
+ if (level == TabletLevel.META) {
tableName = RootTable.NAME;
[10/34] accumulo git commit: ACCUMULO-3423 merge master
ACCUMULO-3423 merge master
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f4f28c7b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f4f28c7b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f4f28c7b
Branch: refs/heads/master
Commit: f4f28c7bad0df92fade2e88da217f60bfb826783
Parents: 31ee26b 01558a4
Author: Eric C. Newton eric.new...@gmail.com
Authored: Wed Mar 11 14:41:15 2015 -0400
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Wed Mar 11 14:41:15 2015 -0400
--
assemble/bin/tool.sh| 7 +-
.../client/security/tokens/KerberosToken.java | 3 +-
.../accumulo/core/file/rfile/PrintInfo.java | 15 +++-
.../simple/client/TraceDumpExample.java | 28 ++-
.../standalone/StandaloneAccumuloCluster.java | 14 ++--
.../standalone/StandaloneClusterControl.java| 5 +-
.../org/apache/accumulo/proxy/ProxyServer.java | 18 -
.../accumulo/test/SizeCompactionStrategy.java | 59 ++
.../accumulo/test/TestCompactionStrategy.java | 72 +
.../accumulo/harness/AccumuloClusterIT.java | 2 +-
.../StandaloneAccumuloClusterConfiguration.java | 14 +++-
.../accumulo/proxy/ProxyDurabilityIT.java | 8 +-
.../apache/accumulo/proxy/SimpleProxyBase.java | 20 -
.../accumulo/test/BulkImportVolumeIT.java | 1 +
.../accumulo/test/ConditionalWriterIT.java | 21 +
.../org/apache/accumulo/test/MetaSplitIT.java | 39 -
.../org/apache/accumulo/test/NamespacesIT.java | 3 +
.../org/apache/accumulo/test/ShellServerIT.java | 1 -
.../accumulo/test/UserCompactionStrategyIT.java | 85
.../accumulo/test/functional/CleanUpIT.java | 4 +-
.../accumulo/test/functional/CloneTestIT.java | 4 +-
.../accumulo/test/functional/CredentialsIT.java | 15 ++--
.../accumulo/test/functional/ExamplesIT.java| 45 ---
.../accumulo/test/functional/ReadWriteIT.java | 16 +++-
.../accumulo/test/functional/ScanIdIT.java | 36 +
test/src/test/resources/log4j.properties| 1 +
26 files changed, 388 insertions(+), 148 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f4f28c7b/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
--
diff --cc test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
index de8ebc8,404a8fd..654d27d
--- a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
@@@ -58,14 -60,9 +60,14 @@@ import com.google.common.net.HostAndPor
public class ProxyDurabilityIT extends ConfigurableMacIT {
@Override
+ protected int defaultTimeoutSeconds() {
+return 60;
+ }
+
+ @Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
hadoopCoreSite.set(fs.file.impl, RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, 5s);
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, 10s);
cfg.setNumTservers(1);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f4f28c7b/test/src/test/java/org/apache/accumulo/test/ConditionalWriterIT.java
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f4f28c7b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
--
[09/34] accumulo git commit: ACCUMULO-3423 fixed replication bugs with recent refactorings in StatusUtil
ACCUMULO-3423 fixed replication bugs with recent refactorings in StatusUtil
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/31ee26b8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/31ee26b8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/31ee26b8
Branch: refs/heads/master
Commit: 31ee26b8ac41844f2a647a5d1484f47da731872a
Parents: 4635de8
Author: Eric C. Newton eric.new...@gmail.com
Authored: Wed Mar 11 14:37:39 2015 -0400
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Wed Mar 11 14:37:39 2015 -0400
--
.../main/java/org/apache/accumulo/core/replication/StatusUtil.java | 2 +-
.../java/org/apache/accumulo/tserver/log/TabletServerLogger.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31ee26b8/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
b/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
index d8ec403..cdb6963 100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
@@ -155,7 +155,7 @@ public class StatusUtil {
/**
* @return A {@link Status} for an open file of unspecified length, all of
which needs replicating.
*/
- public static Status openWithUnknownLength(long timeCreated) {
+ public static synchronized Status openWithUnknownLength(long timeCreated) {
return
INF_END_REPLICATION_STATUS_BUILDER.setCreatedTime(timeCreated).build();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31ee26b8/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
--
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
index 46101c1..498cbdd 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
@@ -319,7 +319,7 @@ public class TabletServerLogger {
// Need to release
KeyExtent extent = commitSession.getExtent();
if (ReplicationConfigurationUtil.isEnabled(extent,
tserver.getTableConfiguration(extent))) {
-Status status =
StatusUtil.fileCreated(System.currentTimeMillis());
+Status status =
StatusUtil.openWithUnknownLength(System.currentTimeMillis());
log.debug(Writing + ProtobufUtil.toString(status) + to
metadata table for + copy.getFileName());
// Got some new WALs, note this in the metadata table
ReplicationTableUtil.updateFiles(tserver,
commitSession.getExtent(), copy.getFileName(), status);
[13/34] accumulo git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/accumulo
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/accumulo Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/34af43fd Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/34af43fd Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/34af43fd Branch: refs/heads/master Commit: 34af43fd02626eca3c42bc3a39ccf4ab8b927806 Parents: ac176e1 9df45b2 Author: Eric C. Newton eric.new...@gmail.com Authored: Fri Mar 20 10:51:21 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Fri Mar 20 10:51:21 2015 -0400 -- TESTING.md | 1 + assemble/bin/accumulo | 3 +- .../core/client/impl/ThriftScanner.java | 17 +- .../org/apache/accumulo/core/data/Mutation.java | 60 +- .../apache/accumulo/core/data/MutationTest.java | 42 + .../mapreduce/bulk/BulkIngestExample.java | 4 + .../standalone/StandaloneAccumuloCluster.java | 10 +- .../standalone/StandaloneClusterControl.java| 25 ++- .../accumulo/harness/AccumuloClusterIT.java | 2 +- .../StandaloneAccumuloClusterConfiguration.java | 16 ++ .../apache/accumulo/proxy/SimpleProxyBase.java | 185 --- .../apache/accumulo/proxy/TBinaryProxyIT.java | 6 - .../apache/accumulo/proxy/TCompactProxyIT.java | 6 - .../accumulo/proxy/TJsonProtocolProxyIT.java| 6 - .../apache/accumulo/proxy/TTupleProxyIT.java| 6 - .../accumulo/test/BulkImportVolumeIT.java | 3 + .../apache/accumulo/test/ScanIteratorIT.java| 7 + .../accumulo/test/UserCompactionStrategyIT.java | 4 + .../apache/accumulo/test/functional/BulkIT.java | 22 +-- .../functional/BulkSplitOptimizationIT.java | 1 + .../accumulo/test/functional/CompactionIT.java | 1 + .../accumulo/test/functional/DeleteIT.java | 5 - .../test/functional/FunctionalTestUtils.java| 7 + 23 files changed, 269 insertions(+), 170 deletions(-) --
[28/34] accumulo git commit: ACCUMULO-3423 optimize WAL metadata table updates
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3fdd29f5/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
--
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index afd3454..aeb73b4 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -19,7 +19,6 @@ package org.apache.accumulo.tserver;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.net.UnknownHostException;
@@ -30,6 +29,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -45,6 +45,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CancellationException;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
@@ -147,8 +148,8 @@ import
org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.GarbageCollectionLogger;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.TabletLevel;
import org.apache.accumulo.server.client.ClientServiceHandler;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfigurationFactory;
@@ -1440,6 +1441,7 @@ public class TabletServer extends AccumuloServerContext
implements Runnable {
}
}
+
@Override
public void loadTablet(TInfo tinfo, TCredentials credentials, String lock,
final TKeyExtent textent) {
@@ -1500,6 +1502,7 @@ public class TabletServer extends AccumuloServerContext
implements Runnable {
final AssignmentHandler ah = new AssignmentHandler(extent);
// final Runnable ah = new LoggingRunnable(log, );
// Root tablet assignment must take place immediately
+
if (extent.isRootTablet()) {
new Daemon(Root Tablet Assignment) {
@Override
@@ -1692,66 +1695,6 @@ public class TabletServer extends AccumuloServerContext
implements Runnable {
}
@Override
-public void removeLogs(TInfo tinfo, TCredentials credentials, ListString
filenames) throws TException {
- String myname = getClientAddressString();
- myname = myname.replace(':', '+');
- SetString loggers = new HashSetString();
- logger.getLogFiles(loggers);
- SetString loggerUUIDs = new HashSetString();
- for (String logger : loggers)
-loggerUUIDs.add(new Path(logger).getName());
-
- nextFile: for (String filename : filenames) {
-String uuid = new Path(filename).getName();
-// skip any log we're currently using
-if (loggerUUIDs.contains(uuid))
- continue nextFile;
-
-ListTablet onlineTabletsCopy = new ArrayListTablet();
-synchronized (onlineTablets) {
- onlineTabletsCopy.addAll(onlineTablets.values());
-}
-for (Tablet tablet : onlineTabletsCopy) {
- for (String current : tablet.getCurrentLogFiles()) {
-if (current.contains(uuid)) {
- log.info(Attempted to delete + filename + from tablet +
tablet.getExtent());
- continue nextFile;
-}
- }
-}
-
-try {
- Path source = new Path(filename);
- if
(TabletServer.this.getConfiguration().getBoolean(Property.TSERV_ARCHIVE_WALOGS))
{
-Path walogArchive = fs.matchingFileSystem(source,
ServerConstants.getWalogArchives());
-fs.mkdirs(walogArchive);
-Path dest = new Path(walogArchive, source.getName());
-log.info(Archiving walog + source + to + dest);
-if (!fs.rename(source, dest))
- log.error(rename is unsuccessful);
- } else {
-log.info(Deleting walog + filename);
-Path sourcePath = new Path(filename);
-if
(!(!TabletServer.this.getConfiguration().getBoolean(Property.GC_TRASH_IGNORE)
fs.moveToTrash(sourcePath))
- !fs.deleteRecursively(sourcePath))
- log.warn(Failed to delete walog + source);
-for (String recovery : ServerConstants.getRecoveryDirs()) {
-
[26/34] accumulo git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/accumulo
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/accumulo Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fb25f3b1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fb25f3b1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fb25f3b1 Branch: refs/heads/master Commit: fb25f3b15ac0ca4262ac5c8664cea1ff81246bb6 Parents: f43b216 33ef1e0 Author: Eric C. Newton eric.new...@gmail.com Authored: Wed Apr 22 11:36:52 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Wed Apr 22 11:36:52 2015 -0400 -- README.md | 30 +- core/pom.xml| 57 ++ core/src/main/findbugs/exclude-filter.xml | 1 + .../apache/accumulo/core/cli/ClientOpts.java| 5 +- .../core/client/ClientConfiguration.java| 50 +- .../core/client/ClientSideIteratorScanner.java | 27 +- .../accumulo/core/client/IsolatedScanner.java | 4 +- .../core/client/MutationsRejectedException.java | 68 +- .../accumulo/core/client/ZooKeeperInstance.java | 21 +- .../core/client/admin/ActiveCompaction.java | 12 +- .../accumulo/core/client/admin/ActiveScan.java | 12 +- .../client/admin/DelegationTokenConfig.java | 4 +- .../core/client/admin/SecurityOperations.java | 3 +- .../core/client/impl/ActiveCompactionImpl.java | 19 +- .../core/client/impl/ActiveScanImpl.java| 14 +- .../impl/AuthenticationTokenIdentifier.java | 210 + .../core/client/impl/ClientContext.java | 1 - .../core/client/impl/ConditionalWriterImpl.java | 2 +- .../accumulo/core/client/impl/Credentials.java | 157 .../core/client/impl/DelegationTokenImpl.java | 144 .../client/impl/InstanceOperationsImpl.java | 18 + .../client/impl/NamespaceOperationsImpl.java| 1 - .../core/client/impl/OfflineScanner.java| 3 +- .../core/client/impl/RootTabletLocator.java | 2 +- .../client/impl/SecurityOperationsImpl.java | 4 +- .../core/client/impl/TableOperationsImpl.java | 2 +- .../core/client/impl/TabletLocator.java | 2 +- .../core/client/impl/TabletLocatorImpl.java | 2 +- .../impl/TabletServerBatchReaderIterator.java | 2 +- .../client/impl/TabletServerBatchWriter.java| 8 +- .../accumulo/core/client/impl/TabletType.java | 2 +- .../core/client/impl/ThriftScanner.java | 2 +- .../core/client/impl/TimeoutTabletLocator.java | 2 +- .../accumulo/core/client/impl/Translator.java | 2 +- .../accumulo/core/client/impl/Writer.java | 2 +- .../core/client/mapred/AbstractInputFormat.java | 78 +- .../client/mapred/AccumuloFileOutputFormat.java | 5 +- .../client/mapred/AccumuloOutputFormat.java | 23 +- .../client/mapreduce/AbstractInputFormat.java | 82 +- .../mapreduce/AccumuloFileOutputFormat.java | 5 +- .../client/mapreduce/AccumuloOutputFormat.java | 21 +- .../core/client/mapreduce/RangeInputSplit.java | 24 +- .../mapreduce/impl/AccumuloInputSplit.java | 73 +- .../client/mapreduce/impl/BatchInputSplit.java | 21 +- .../core/client/mapreduce/impl/SplitUtils.java | 59 ++ .../mapreduce/lib/impl/ConfiguratorBase.java| 16 +- .../mapreduce/lib/impl/InputConfigurator.java | 8 +- .../core/client/mock/MockConnector.java | 2 +- .../accumulo/core/client/mock/MockInstance.java | 2 +- .../client/mock/impl/MockTabletLocator.java | 2 +- .../client/security/tokens/DelegationToken.java | 134 +--- .../accumulo/core/constraints/Constraint.java | 2 +- .../core/constraints/VisibilityConstraint.java | 93 +++ .../accumulo/core/data/ComparableBytes.java | 5 + .../apache/accumulo/core/data/KeyExtent.java| 717 +++-- .../org/apache/accumulo/core/data/TabletID.java | 29 + .../core/data/impl/ComparableBytes.java | 53 ++ .../accumulo/core/data/impl/KeyExtent.java | 768 +++ .../accumulo/core/data/impl/TabletIDImpl.java | 100 +++ .../accumulo/core/iterators/IteratorUtil.java | 40 +- .../core/iterators/system/MultiIterator.java| 2 +- .../core/metadata/MetadataLocationObtainer.java | 2 +- .../core/metadata/MetadataServicer.java | 2 +- .../accumulo/core/metadata/RootTable.java | 2 +- .../core/metadata/ServicerForRootTable.java | 2 +- .../core/metadata/TableMetadataServicer.java| 2 +- .../ReplicationConfigurationUtil.java | 2 +- .../rpc/SaslClientDigestCallbackHandler.java| 4 +- .../accumulo/core/rpc/SaslConnectionParams.java | 6 +- .../security/AuthenticationTokenIdentifier.java | 210 - .../accumulo/core/security/Credentials.java | 157 .../core/security/VisibilityConstraint.java | 76 +- .../core/security/VisibilityEvaluator.java | 10 +-
[31/34] accumulo git commit: Merge branch '1.7'
Merge branch '1.7' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8b3358cb Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8b3358cb Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8b3358cb Branch: refs/heads/master Commit: 8b3358cb0f5f0e127ce9f66f2a39ebd9aa88f28a Parents: 6df7169 3fdd29f Author: Eric C. Newton eric.new...@gmail.com Authored: Fri Apr 24 18:22:34 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Fri Apr 24 18:22:34 2015 -0400 -- .../client/impl/ReplicationOperationsImpl.java | 4 +- .../org/apache/accumulo/core/conf/Property.java | 4 +- .../accumulo/core/metadata/RootTable.java | 1 + .../core/metadata/schema/MetadataSchema.java| 48 ++ .../core/tabletserver/log/LogEntry.java | 78 ++- .../core/metadata/MetadataTableSchemaTest.java | 47 ++ .../org/apache/accumulo/server/TabletLevel.java | 34 ++ .../apache/accumulo/server/fs/VolumeUtil.java | 22 +- .../apache/accumulo/server/init/Initialize.java | 1 + .../server/master/state/MetaDataStateStore.java | 47 +- .../master/state/MetaDataTableScanner.java | 6 +- .../master/state/TabletLocationState.java | 7 + .../server/master/state/TabletStateStore.java | 16 +- .../master/state/ZooTabletStateStore.java | 35 +- .../accumulo/server/replication/StatusUtil.java | 13 + .../accumulo/server/util/ListVolumesUsed.java | 18 +- .../server/util/MasterMetadataUtil.java | 18 +- .../accumulo/server/util/MetadataTableUtil.java | 239 +--- .../server/util/ReplicationTableUtil.java | 13 +- .../server/util/ReplicationTableUtilTest.java | 2 +- .../gc/GarbageCollectWriteAheadLogs.java| 499 +++- .../accumulo/gc/SimpleGarbageCollector.java | 1 - .../CloseWriteAheadLogReferences.java | 23 +- .../gc/GarbageCollectWriteAheadLogsTest.java| 567 --- .../CloseWriteAheadLogReferencesTest.java | 151 + .../java/org/apache/accumulo/master/Master.java | 3 + .../master/MasterClientServiceHandler.java | 3 +- .../accumulo/master/TabletGroupWatcher.java | 37 +- .../accumulo/master/replication/WorkMaker.java | 1 + .../accumulo/master/state/MergeStats.java | 3 +- .../master/ReplicationOperationsImplTest.java | 9 +- .../apache/accumulo/master/TestMergeState.java | 2 +- .../master/state/RootTabletStateStoreTest.java | 4 +- .../src/main/findbugs/exclude-filter.xml| 2 +- .../server/GarbageCollectionLogger.java | 3 +- .../apache/accumulo/tserver/TabletServer.java | 182 +++--- .../apache/accumulo/tserver/log/DfsLogger.java | 14 +- .../accumulo/tserver/log/SortedLogRecovery.java | 8 +- .../tserver/log/TabletServerLogger.java | 187 +++--- .../accumulo/tserver/tablet/CommitSession.java | 3 +- .../tserver/tablet/DatafileManager.java | 4 +- .../apache/accumulo/tserver/tablet/Tablet.java | 59 +- .../tserver/tablet/TabletCommitter.java | 3 +- .../accumulo/tserver/log/LogEntryTest.java | 56 ++ .../test/performance/thrift/NullTserver.java| 6 +- .../accumulo/proxy/ProxyDurabilityIT.java | 9 +- .../test/BadDeleteMarkersCreatedIT.java | 2 +- .../org/apache/accumulo/test/BalanceIT.java | 20 +- .../org/apache/accumulo/test/CleanWalIT.java| 1 + .../accumulo/test/ConditionalWriterIT.java | 1 + .../accumulo/test/GarbageCollectWALIT.java | 81 +++ .../MissingWalHeaderCompletesRecoveryIT.java| 14 +- .../accumulo/test/NoMutationRecoveryIT.java | 178 -- .../org/apache/accumulo/test/ShellServerIT.java | 2 +- .../org/apache/accumulo/test/UnusedWALIT.java | 144 + .../java/org/apache/accumulo/test/VolumeIT.java | 17 + .../accumulo/test/functional/ReadWriteIT.java | 8 + .../accumulo/test/functional/WALSunnyDayIT.java | 250 .../test/functional/WatchTheWatchCountIT.java | 2 +- .../test/performance/RollWALPerformanceIT.java | 126 + ...bageCollectorCommunicatesWithTServersIT.java | 35 +- .../replication/MultiInstanceReplicationIT.java | 2 +- .../test/replication/ReplicationIT.java | 370 63 files changed, 1857 insertions(+), 1888 deletions(-) --
[25/34] accumulo git commit: ACCUMULO-3423 finished up testing, went back to lazy metadata table references
ACCUMULO-3423 finished up testing, went back to lazy metadata table references
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f43b2160
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f43b2160
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f43b2160
Branch: refs/heads/master
Commit: f43b216038157e471612eac4679696ce7d6f7835
Parents: ed7c4f6
Author: Eric C. Newton eric.new...@gmail.com
Authored: Wed Apr 22 10:34:02 2015 -0400
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Wed Apr 22 10:34:02 2015 -0400
--
.../tserver/log/TabletServerLogger.java | 24 +++-
.../accumulo/proxy/ProxyDurabilityIT.java | 4 ++--
.../org/apache/accumulo/test/UnusedWALIT.java | 2 +-
.../accumulo/test/functional/WALSunnyDayIT.java | 20
4 files changed, 31 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f43b2160/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
--
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
index cdee51b..7d16aae 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
@@ -78,7 +78,7 @@ public class TabletServerLogger {
// The current logger
private DfsLogger currentLog = null;
- private final SynchronousQueueDfsLogger nextLog = new SynchronousQueue();
+ private final SynchronousQueueObject nextLog = new SynchronousQueue();
private ThreadPoolExecutor nextLogMaker;
// The current generation of logs.
@@ -200,11 +200,18 @@ public class TabletServerLogger {
try {
startLogMaker();
- DfsLogger next = nextLog.take();
- log.info(Using next log + next.getFileName());
- currentLog = next;
- logId.incrementAndGet();
- return;
+ Object next = nextLog.take();
+ if (next instanceof Exception) {
+throw (Exception)next;
+ }
+ if (next instanceof DfsLogger) {
+currentLog = (DfsLogger)next;
+logId.incrementAndGet();
+log.info(Using next log + currentLog.getFileName());
+return;
+ } else {
+throw new RuntimeException(Error: unexpected type seen: + next);
+ }
} catch (Exception t) {
walErrors.put(System.currentTimeMillis(), );
if (walErrors.size() = HALT_AFTER_ERROR_COUNT) {
@@ -233,6 +240,11 @@ public class TabletServerLogger {
}
} catch (Exception t) {
log.error({}, t.getMessage(), t);
+try {
+ nextLog.offer(t, 12, TimeUnit.HOURS);
+} catch (InterruptedException ex) {
+ // ignore
+}
}
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f43b2160/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
--
diff --git
a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
b/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
index 654d27d..81e25cc 100644
--- a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
@@ -116,7 +116,7 @@ public class ProxyDurabilityIT extends ConfigurableMacIT {
assertEquals(0, count(tableName));
ConditionalWriterOptions cfg = new ConditionalWriterOptions();
-cfg.setDurability(Durability.LOG);
+cfg.setDurability(Durability.SYNC);
String cwriter = client.createConditionalWriter(login, tableName, cfg);
ConditionalUpdates updates = new ConditionalUpdates();
updates.addToConditions(new Condition(new Column(bytes(cf), bytes(cq),
bytes(;
@@ -125,7 +125,7 @@ public class ProxyDurabilityIT extends ConfigurableMacIT {
assertEquals(ConditionalStatus.ACCEPTED, status.get(bytes(row)));
assertEquals(1, count(tableName));
restartTServer();
-assertEquals(0, count(tableName));
+assertEquals(1, count(tableName));
proxyServer.stop();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f43b2160/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
--
diff --git a/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
b/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
index 3684ee1..03d783c 100644
--- a/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
+++
[33/34] accumulo git commit: Merge branch 'master' of https://github.com/ericnewton/accumulo-3423
Merge branch 'master' of https://github.com/ericnewton/accumulo-3423 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d0a0ac05 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d0a0ac05 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d0a0ac05 Branch: refs/heads/master Commit: d0a0ac0598b23046de0599c2c061adb173ce6840 Parents: 8b3358c fb25f3b Author: Eric C. Newton eric.new...@gmail.com Authored: Fri Apr 24 19:05:58 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Fri Apr 24 19:05:58 2015 -0400 -- --
[19/34] accumulo git commit: ACCUMULO-3423 merge apache master
ACCUMULO-3423 merge apache master Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8f5e0021 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8f5e0021 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8f5e0021 Branch: refs/heads/master Commit: 8f5e0021a11d507fa00853ce62bc36b4d6654a53 Parents: 9c2ca7a 85d254e Author: Eric C. Newton eric.new...@gmail.com Authored: Tue Apr 14 15:14:08 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Tue Apr 14 15:14:08 2015 -0400 -- assemble/bin/accumulo_watcher.sh| 133 +++ assemble/bin/start-server.sh| 9 +- assemble/conf/templates/accumulo-env.sh | 14 + assemble/pom.xml| 32 ++ assemble/src/main/assemblies/component.xml | 11 + .../main/scripts/generate-versions-listing.sh | 27 ++ core/src/main/findbugs/exclude-filter.xml | 5 + .../accumulo/core/bloomfilter/Filter.java | 2 + .../apache/accumulo/core/cli/ClientOpts.java| 45 +-- .../accumulo/core/client/BatchDeleter.java | 4 +- .../accumulo/core/client/BatchScanner.java | 16 +- .../accumulo/core/client/IteratorSetting.java | 8 +- .../accumulo/core/client/ScannerBase.java | 30 +- .../core/client/admin/TableOperations.java | 2 +- .../core/client/impl/ClientContext.java | 10 +- .../client/impl/MultiTableBatchWriterImpl.java | 4 +- .../accumulo/core/client/impl/ScannerImpl.java | 7 +- .../core/client/impl/ScannerIterator.java | 6 +- .../core/client/impl/ScannerOptions.java| 22 +- .../core/client/impl/TableOperationsImpl.java | 218 -- .../client/impl/TabletServerBatchReader.java| 4 +- .../client/impl/TabletServerBatchWriter.java| 6 +- .../core/client/impl/TimeoutTabletLocator.java | 4 - .../core/client/lexicoder/AbstractEncoder.java | 24 +- .../core/client/lexicoder/BytesLexicoder.java | 7 +- .../core/client/lexicoder/ListLexicoder.java| 12 +- .../core/client/lexicoder/UUIDLexicoder.java| 10 +- .../core/client/mapred/AbstractInputFormat.java | 11 +- .../client/mapred/AccumuloOutputFormat.java | 32 +- .../client/mapreduce/AbstractInputFormat.java | 12 +- .../client/mapreduce/AccumuloOutputFormat.java | 32 +- .../core/client/mapreduce/InputFormatBase.java | 9 - .../core/client/mapreduce/InputTableConfig.java | 12 - .../core/client/mapreduce/RangeInputSplit.java | 10 +- .../mapreduce/lib/impl/InputConfigurator.java | 3 +- .../core/client/mock/MockConfiguration.java | 23 +- .../client/security/tokens/DelegationToken.java | 4 +- .../client/security/tokens/KerberosToken.java | 13 +- .../client/security/tokens/PasswordToken.java | 6 +- .../core/conf/AccumuloConfiguration.java| 56 ++- .../accumulo/core/conf/ConfigurationCopy.java | 6 +- .../accumulo/core/conf/ConfigurationDocGen.java | 2 +- .../core/conf/DefaultConfiguration.java | 6 +- .../org/apache/accumulo/core/conf/Property.java | 14 +- .../accumulo/core/conf/SiteConfiguration.java | 7 +- .../accumulo/core/data/ArrayByteSequence.java | 1 + .../apache/accumulo/core/data/ByteSequence.java | 9 +- .../org/apache/accumulo/core/data/Column.java | 7 +- .../accumulo/core/data/ComparableBytes.java | 1 + .../java/org/apache/accumulo/core/data/Key.java | 13 - .../apache/accumulo/core/data/KeyExtent.java| 12 - .../org/apache/accumulo/core/data/Mutation.java | 43 +- .../org/apache/accumulo/core/data/Value.java| 2 + .../accumulo/core/file/BloomFilterLayer.java| 2 +- .../file/blockfile/cache/CachedBlockQueue.java | 1 + .../file/blockfile/cache/LruBlockCache.java | 30 +- .../file/blockfile/impl/CachableBlockFile.java | 9 +- .../core/file/rfile/bcfile/CompareUtils.java| 1 + .../core/file/rfile/bcfile/Compression.java | 1 + .../accumulo/core/file/rfile/bcfile/Utils.java | 6 - .../accumulo/core/iterators/DebugIterator.java | 1 + .../core/iterators/SortedKeyValueIterator.java | 3 +- .../core/iterators/TypedValueCombiner.java | 2 +- .../iterators/aggregation/LongSummation.java| 4 +- .../aggregation/NumArraySummation.java | 3 + .../iterators/aggregation/NumSummation.java | 3 + .../core/iterators/aggregation/StringMax.java | 3 + .../core/iterators/aggregation/StringMin.java | 3 + .../iterators/aggregation/StringSummation.java | 3 + .../core/iterators/conf/ColumnUtil.java | 4 + .../iterators/system/ColumnQualifierFilter.java | 1 + .../core/iterators/system/CountingIterator.java | 1 + .../core/iterators/system/DeletingIterator.java | 1 + .../core/iterators/system/MultiIterator.java| 1 + .../iterators/system/SequenceFileIterator.java | 6 +
[01/34] accumulo git commit: ACCUMULO-3625 use log markers against tservers, not tablets
Repository: accumulo
Updated Branches:
refs/heads/master 6df71693e - 51f39d292
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b2539fb1/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
--
diff --git
a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
index 185a33a..1e508e8 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
@@ -16,11 +16,12 @@
*/
package org.apache.accumulo.test.replication;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Iterator;
@@ -46,6 +47,7 @@ import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.client.replication.ReplicaSystemFactory;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
@@ -67,6 +69,7 @@ import
org.apache.accumulo.core.replication.proto.Replication.Status;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
@@ -75,11 +78,10 @@ import org.apache.accumulo.fate.zookeeper.ZooLock;
import org.apache.accumulo.gc.SimpleGarbageCollector;
import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.replication.StatusCombiner;
import org.apache.accumulo.server.util.ReplicationTableUtil;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
-import org.apache.accumulo.tserver.TabletServer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -123,25 +125,38 @@ public class ReplicationIT extends ConfigurableMacIT {
cfg.setProperty(Property.REPLICATION_WORK_PROCESSOR_DELAY, 1s);
cfg.setProperty(Property.REPLICATION_WORK_PROCESSOR_PERIOD, 1s);
cfg.setProperty(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, 1M);
+cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, 5s);
cfg.setNumTservers(1);
hadoopCoreSite.set(fs.file.impl, RawLocalFileSystem.class.getName());
}
private MultimapString,String getLogs(Connector conn) throws
TableNotFoundException {
-MultimapString,String logs = HashMultimap.create();
+// Map of server to tableId
+MultimapTServerInstance, String serverToTableID = HashMultimap.create();
Scanner scanner = conn.createScanner(MetadataTable.NAME,
Authorizations.EMPTY);
-scanner.fetchColumnFamily(LogColumnFamily.NAME);
-scanner.setRange(new Range());
+scanner.setRange(MetadataSchema.TabletsSection.getRange());
+
scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+for (EntryKey,Value entry : scanner) {
+ TServerInstance key = new TServerInstance(entry.getValue(),
entry.getKey().getColumnQualifier());
+ byte[] tableId = KeyExtent.tableOfMetadataRow(entry.getKey().getRow());
+ serverToTableID.put(key, new String(tableId, UTF_8));
+}
+// Map of logs to tableId
+MultimapString,String logs = HashMultimap.create();
+scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+scanner.setRange(MetadataSchema.CurrentLogsSection.getRange());
for (EntryKey,Value entry : scanner) {
if (Thread.interrupted()) {
return logs;
}
-
- LogEntry logEntry = LogEntry.fromKeyValue(entry.getKey(),
entry.getValue());
-
- for (String log : logEntry.logSet) {
-// Need to normalize the log file from LogEntry
-logs.put(new Path(log).toString(),
logEntry.extent.getTableId().toString());
+ Text path = new Text();
+ MetadataSchema.CurrentLogsSection.getPath(entry.getKey(), path);
+ Text session = new Text();
+ Text hostPort = new Text();
+ MetadataSchema.CurrentLogsSection.getTabletServer(entry.getKey(),
hostPort , session);
+ TServerInstance server = new
TServerInstance(AddressUtil.parseAddress(hostPort.toString()),
[17/34] accumulo git commit: ACCUMULO-3423 more updates based on [~kturner]'s review
ACCUMULO-3423 more updates based on [~kturner]'s review
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a422
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a422
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a422
Branch: refs/heads/master
Commit: a422ddd91cd96c8ca84dc824d931a1b72f70
Parents: c71fc12
Author: Eric C. Newton eric.new...@gmail.com
Authored: Mon Mar 30 16:05:59 2015 -0400
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Mon Mar 30 16:05:59 2015 -0400
--
.../accumulo/server/util/MetadataTableUtil.java | 1 +
.../gc/GarbageCollectWriteAheadLogs.java| 34
.../apache/accumulo/tserver/TabletServer.java | 24 --
.../tserver/log/TabletServerLogger.java | 40 --
.../accumulo/test/GarbageCollectWALIT.java | 81
.../accumulo/test/functional/WALSunnyDayIT.java | 10 +++
6 files changed, 159 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a422/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index f5326bf..edc6189 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -1112,6 +1112,7 @@ public class MetadataTableUtil {
}
public static void markLogUnused(ClientContext context, ZooLock lock,
TServerInstance tabletSession, SetPath all) throws AccumuloException {
+// There could be a marker at the meta and/or root level, mark them both
as unused
try {
BatchWriter root = null;
BatchWriter meta = null;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a422/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
--
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index d523706..59612ab 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -50,9 +50,12 @@ import
org.apache.accumulo.core.replication.ReplicationTableOfflineException;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.accumulo.core.trace.Trace;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.AccumuloServerContext;
+import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.accumulo.server.master.LiveTServerSet;
import org.apache.accumulo.server.master.LiveTServerSet.Listener;
import org.apache.accumulo.server.master.state.MetaDataStateStore;
@@ -63,7 +66,9 @@ import org.apache.accumulo.server.master.state.TabletState;
import org.apache.accumulo.server.replication.StatusUtil;
import org.apache.accumulo.server.replication.proto.Replication.Status;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.Text;
import org.apache.htrace.Span;
import org.apache.zookeeper.KeeperException;
@@ -406,6 +411,35 @@ public class GarbageCollectWriteAheadLogs {
}
}
+// scan HDFS for logs for dead servers
+for (Volume volume : VolumeManagerImpl.get().getVolumes()) {
+ RemoteIteratorLocatedFileStatus iter =
volume.getFileSystem().listFiles(volume.prefixChild(ServerConstants.WAL_DIR),
true);
+ while (iter.hasNext()) {
+LocatedFileStatus next = iter.next();
+// recursive listing returns directories, too
+if (next.isDirectory()) {
+ continue;
+}
+// make sure we've waited long enough for zookeeper propagation
+if (System.currentTimeMillis() - next.getModificationTime()
context.getConnector().getInstance().getZooKeepersSessionTimeOut()) {
+ continue;
+}
+Path path = next.getPath();
+String hostPlusPort = path.getParent().getName();
+// server is still alive, or has a replacement
+TServerInstance instance =
[30/34] accumulo git commit: ACCUMULO-3423 optimize WAL metadata table updates
ACCUMULO-3423 optimize WAL metadata table updates Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3fdd29f5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3fdd29f5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3fdd29f5 Branch: refs/heads/master Commit: 3fdd29f5222f9d1d32ca28b5ecf1d740a8d20f87 Parents: ea25e98 Author: Eric C. Newton eric.new...@gmail.com Authored: Fri Apr 24 18:15:05 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Fri Apr 24 18:18:56 2015 -0400 -- .../client/impl/ReplicationOperationsImpl.java | 4 +- .../org/apache/accumulo/core/conf/Property.java | 4 +- .../accumulo/core/metadata/RootTable.java | 1 + .../core/metadata/schema/MetadataSchema.java| 48 ++ .../core/tabletserver/log/LogEntry.java | 78 ++- .../core/metadata/MetadataTableSchemaTest.java | 47 ++ .../org/apache/accumulo/server/TabletLevel.java | 34 ++ .../apache/accumulo/server/fs/VolumeUtil.java | 22 +- .../apache/accumulo/server/init/Initialize.java | 1 + .../server/master/state/MetaDataStateStore.java | 47 +- .../master/state/MetaDataTableScanner.java | 6 +- .../master/state/TabletLocationState.java | 7 + .../server/master/state/TabletStateStore.java | 16 +- .../master/state/ZooTabletStateStore.java | 35 +- .../accumulo/server/replication/StatusUtil.java | 13 + .../accumulo/server/util/ListVolumesUsed.java | 18 +- .../server/util/MasterMetadataUtil.java | 18 +- .../accumulo/server/util/MetadataTableUtil.java | 239 +--- .../server/util/ReplicationTableUtil.java | 13 +- .../server/util/ReplicationTableUtilTest.java | 2 +- .../gc/GarbageCollectWriteAheadLogs.java| 499 +++- .../accumulo/gc/SimpleGarbageCollector.java | 1 - .../CloseWriteAheadLogReferences.java | 23 +- .../gc/GarbageCollectWriteAheadLogsTest.java| 567 --- .../CloseWriteAheadLogReferencesTest.java | 151 + .../java/org/apache/accumulo/master/Master.java | 3 + .../master/MasterClientServiceHandler.java | 3 +- .../accumulo/master/TabletGroupWatcher.java | 37 +- .../accumulo/master/replication/WorkMaker.java | 1 + .../accumulo/master/state/MergeStats.java | 3 +- .../master/ReplicationOperationsImplTest.java | 9 +- .../apache/accumulo/master/TestMergeState.java | 2 +- .../master/state/RootTabletStateStoreTest.java | 4 +- .../src/main/findbugs/exclude-filter.xml| 2 +- .../server/GarbageCollectionLogger.java | 3 +- .../apache/accumulo/tserver/TabletServer.java | 182 +++--- .../apache/accumulo/tserver/log/DfsLogger.java | 14 +- .../accumulo/tserver/log/SortedLogRecovery.java | 8 +- .../tserver/log/TabletServerLogger.java | 187 +++--- .../accumulo/tserver/tablet/CommitSession.java | 3 +- .../tserver/tablet/DatafileManager.java | 4 +- .../apache/accumulo/tserver/tablet/Tablet.java | 59 +- .../tserver/tablet/TabletCommitter.java | 3 +- .../accumulo/tserver/log/LogEntryTest.java | 56 ++ .../test/performance/thrift/NullTserver.java| 6 +- .../accumulo/proxy/ProxyDurabilityIT.java | 9 +- .../test/BadDeleteMarkersCreatedIT.java | 2 +- .../org/apache/accumulo/test/BalanceIT.java | 20 +- .../org/apache/accumulo/test/CleanWalIT.java| 1 + .../accumulo/test/ConditionalWriterIT.java | 1 + .../accumulo/test/GarbageCollectWALIT.java | 81 +++ .../MissingWalHeaderCompletesRecoveryIT.java| 14 +- .../accumulo/test/NoMutationRecoveryIT.java | 178 -- .../org/apache/accumulo/test/ShellServerIT.java | 2 +- .../org/apache/accumulo/test/UnusedWALIT.java | 144 + .../java/org/apache/accumulo/test/VolumeIT.java | 17 + .../accumulo/test/functional/ReadWriteIT.java | 8 + .../accumulo/test/functional/WALSunnyDayIT.java | 250 .../test/functional/WatchTheWatchCountIT.java | 2 +- .../test/performance/RollWALPerformanceIT.java | 126 + ...bageCollectorCommunicatesWithTServersIT.java | 35 +- .../replication/MultiInstanceReplicationIT.java | 2 +- .../test/replication/ReplicationIT.java | 370 63 files changed, 1857 insertions(+), 1888 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3fdd29f5/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java index 6a5c74a..925877d 100644 ---
[16/34] accumulo git commit: ACCUMULO-3423 merge in master from apache
ACCUMULO-3423 merge in master from apache Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c71fc121 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c71fc121 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c71fc121 Branch: refs/heads/master Commit: c71fc121e70848170f3a3a69c1bb4a0b1b77b24a Parents: afa887b 94bd393 Author: Eric C. Newton eric.new...@gmail.com Authored: Mon Mar 30 11:38:07 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Mon Mar 30 11:38:07 2015 -0400 -- .../accumulo/core/trace/DistributedTrace.java | 1 + .../impl/MiniAccumuloClusterImpl.java | 10 +- .../server/security/SecurityOperation.java | 15 +- .../accumulo/master/TabletGroupWatcher.java | 6 + .../master/tserverOps/ShutdownTServer.java | 51 --- .../accumulo/tracer/AsyncSpanReceiver.java | 13 ++ .../accumulo/tracer/SendSpansViaThrift.java | 5 + .../server/GarbageCollectionLogger.java | 2 +- .../apache/accumulo/tserver/TabletServer.java | 2 +- .../org/apache/accumulo/test/UnusedWALIT.java | 144 --- .../apache/accumulo/proxy/SimpleProxyBase.java | 8 +- .../accumulo/test/AssignmentThreadsIT.java | 10 +- .../test/BalanceWithOfflineTableIT.java | 90 .../org/apache/accumulo/test/UnusedWALIT.java | 144 +++ .../accumulo/test/functional/ClassLoaderIT.java | 4 +- .../accumulo/test/functional/DurabilityIT.java | 2 - .../test/functional/KerberosProxyIT.java| 14 +- .../test/functional/WatchTheWatchCountIT.java | 4 +- .../test/replication/ReplicationIT.java | 4 +- 19 files changed, 346 insertions(+), 183 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c71fc121/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c71fc121/server/tserver/src/main/java/org/apache/accumulo/server/GarbageCollectionLogger.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c71fc121/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c71fc121/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java -- diff --cc test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java index 000,000..3684ee1 new file mode 100644 --- /dev/null +++ b/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java @@@ -1,0 -1,0 +1,144 @@@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the License); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an AS IS BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.accumulo.test; ++ ++import static org.junit.Assert.assertEquals; ++ ++import java.util.Map.Entry; ++ ++import org.apache.accumulo.core.client.BatchWriter; ++import org.apache.accumulo.core.client.BatchWriterConfig; ++import org.apache.accumulo.core.client.Connector; ++import org.apache.accumulo.core.client.Scanner; ++import org.apache.accumulo.core.conf.Property; ++import org.apache.accumulo.core.data.Key; ++import org.apache.accumulo.core.data.Mutation; ++import org.apache.accumulo.core.data.Range; ++import org.apache.accumulo.core.data.Value; ++import org.apache.accumulo.core.metadata.MetadataTable; ++import org.apache.accumulo.core.metadata.schema.MetadataSchema.CurrentLogsSection; ++import org.apache.accumulo.core.security.Authorizations; ++import org.apache.accumulo.minicluster.ServerType; ++import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; ++import org.apache.accumulo.test.functional.ConfigurableMacIT; ++import org.apache.hadoop.conf.Configuration; ++import org.apache.hadoop.fs.RawLocalFileSystem; ++import org.junit.Test; ++ ++import com.google.common.collect.Iterators; ++ ++// When reviewing the
[4/5] accumulo git commit: ACCUMULO-3423 optimize WAL metadata table updates
ACCUMULO-3423 optimize WAL metadata table updates Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3fdd29f5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3fdd29f5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3fdd29f5 Branch: refs/heads/1.7 Commit: 3fdd29f5222f9d1d32ca28b5ecf1d740a8d20f87 Parents: ea25e98 Author: Eric C. Newton eric.new...@gmail.com Authored: Fri Apr 24 18:15:05 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Fri Apr 24 18:18:56 2015 -0400 -- .../client/impl/ReplicationOperationsImpl.java | 4 +- .../org/apache/accumulo/core/conf/Property.java | 4 +- .../accumulo/core/metadata/RootTable.java | 1 + .../core/metadata/schema/MetadataSchema.java| 48 ++ .../core/tabletserver/log/LogEntry.java | 78 ++- .../core/metadata/MetadataTableSchemaTest.java | 47 ++ .../org/apache/accumulo/server/TabletLevel.java | 34 ++ .../apache/accumulo/server/fs/VolumeUtil.java | 22 +- .../apache/accumulo/server/init/Initialize.java | 1 + .../server/master/state/MetaDataStateStore.java | 47 +- .../master/state/MetaDataTableScanner.java | 6 +- .../master/state/TabletLocationState.java | 7 + .../server/master/state/TabletStateStore.java | 16 +- .../master/state/ZooTabletStateStore.java | 35 +- .../accumulo/server/replication/StatusUtil.java | 13 + .../accumulo/server/util/ListVolumesUsed.java | 18 +- .../server/util/MasterMetadataUtil.java | 18 +- .../accumulo/server/util/MetadataTableUtil.java | 239 +--- .../server/util/ReplicationTableUtil.java | 13 +- .../server/util/ReplicationTableUtilTest.java | 2 +- .../gc/GarbageCollectWriteAheadLogs.java| 499 +++- .../accumulo/gc/SimpleGarbageCollector.java | 1 - .../CloseWriteAheadLogReferences.java | 23 +- .../gc/GarbageCollectWriteAheadLogsTest.java| 567 --- .../CloseWriteAheadLogReferencesTest.java | 151 + .../java/org/apache/accumulo/master/Master.java | 3 + .../master/MasterClientServiceHandler.java | 3 +- .../accumulo/master/TabletGroupWatcher.java | 37 +- .../accumulo/master/replication/WorkMaker.java | 1 + .../accumulo/master/state/MergeStats.java | 3 +- .../master/ReplicationOperationsImplTest.java | 9 +- .../apache/accumulo/master/TestMergeState.java | 2 +- .../master/state/RootTabletStateStoreTest.java | 4 +- .../src/main/findbugs/exclude-filter.xml| 2 +- .../server/GarbageCollectionLogger.java | 3 +- .../apache/accumulo/tserver/TabletServer.java | 182 +++--- .../apache/accumulo/tserver/log/DfsLogger.java | 14 +- .../accumulo/tserver/log/SortedLogRecovery.java | 8 +- .../tserver/log/TabletServerLogger.java | 187 +++--- .../accumulo/tserver/tablet/CommitSession.java | 3 +- .../tserver/tablet/DatafileManager.java | 4 +- .../apache/accumulo/tserver/tablet/Tablet.java | 59 +- .../tserver/tablet/TabletCommitter.java | 3 +- .../accumulo/tserver/log/LogEntryTest.java | 56 ++ .../test/performance/thrift/NullTserver.java| 6 +- .../accumulo/proxy/ProxyDurabilityIT.java | 9 +- .../test/BadDeleteMarkersCreatedIT.java | 2 +- .../org/apache/accumulo/test/BalanceIT.java | 20 +- .../org/apache/accumulo/test/CleanWalIT.java| 1 + .../accumulo/test/ConditionalWriterIT.java | 1 + .../accumulo/test/GarbageCollectWALIT.java | 81 +++ .../MissingWalHeaderCompletesRecoveryIT.java| 14 +- .../accumulo/test/NoMutationRecoveryIT.java | 178 -- .../org/apache/accumulo/test/ShellServerIT.java | 2 +- .../org/apache/accumulo/test/UnusedWALIT.java | 144 + .../java/org/apache/accumulo/test/VolumeIT.java | 17 + .../accumulo/test/functional/ReadWriteIT.java | 8 + .../accumulo/test/functional/WALSunnyDayIT.java | 250 .../test/functional/WatchTheWatchCountIT.java | 2 +- .../test/performance/RollWALPerformanceIT.java | 126 + ...bageCollectorCommunicatesWithTServersIT.java | 35 +- .../replication/MultiInstanceReplicationIT.java | 2 +- .../test/replication/ReplicationIT.java | 370 63 files changed, 1857 insertions(+), 1888 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3fdd29f5/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java index 6a5c74a..925877d 100644 ---
[24/34] accumulo git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/accumulo
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/accumulo Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ed7c4f6c Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ed7c4f6c Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ed7c4f6c Branch: refs/heads/master Commit: ed7c4f6c521af124171c757fb9b81dd7dbfb046d Parents: 35e3f12 321f573 Author: Eric C. Newton eric.new...@gmail.com Authored: Tue Apr 21 13:26:38 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Tue Apr 21 13:26:38 2015 -0400 -- assemble/pom.xml| 2 +- core/pom.xml| 2 +- .../mapreduce/lib/impl/InputConfigurator.java | 5 +- docs/pom.xml| 2 +- examples/simple/pom.xml | 2 +- fate/pom.xml| 2 +- maven-plugin/pom.xml| 2 +- minicluster/pom.xml | 2 +- pom.xml | 2 +- proxy/pom.xml | 2 +- server/base/pom.xml | 2 +- server/gc/pom.xml | 2 +- server/master/pom.xml | 2 +- server/monitor/pom.xml | 2 +- server/native/pom.xml | 2 +- server/tracer/pom.xml | 2 +- server/tserver/pom.xml | 2 +- .../tserver/log/LocalWALRecoveryTest.java | 2 +- shell/pom.xml | 2 +- start/pom.xml | 2 +- test/pom.xml| 2 +- .../accumulo/test/AccumuloOutputFormatIT.java | 125 +++ .../accumulo/test/AccumuloOutputFormatTest.java | 114 - .../test/functional/AccumuloInputFormatIT.java | 16 ++- trace/pom.xml | 2 +- 25 files changed, 159 insertions(+), 143 deletions(-) --
[04/34] accumulo git commit: ACCUMULO-3625 use log markers against tservers, not tablets
ACCUMULO-3625 use log markers against tservers, not tablets
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b2539fb1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b2539fb1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b2539fb1
Branch: refs/heads/master
Commit: b2539fb1640e3a3967d4cdb21ca2d8b3fb4f0c38
Parents: aac52a8
Author: Eric C. Newton eric.new...@gmail.com
Authored: Wed Feb 25 16:42:55 2015 -0500
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Wed Feb 25 16:42:55 2015 -0500
--
.../client/impl/ReplicationOperationsImpl.java | 4 +-
.../org/apache/accumulo/core/conf/Property.java | 4 +-
.../accumulo/core/metadata/RootTable.java | 1 +
.../core/metadata/schema/MetadataSchema.java| 48 ++
.../core/tabletserver/log/LogEntry.java | 75 +-
.../thrift/TabletClientService.java | 749 +--
core/src/main/thrift/tabletserver.thrift| 1 -
.../core/metadata/MetadataTableSchemaTest.java | 39 +
.../ReplicationOperationsImplTest.java | 9 +-
.../apache/accumulo/server/fs/VolumeUtil.java | 22 +-
.../apache/accumulo/server/init/Initialize.java | 1 +
.../server/master/state/MetaDataStateStore.java | 40 +-
.../master/state/MetaDataTableScanner.java | 7 +-
.../master/state/TabletLocationState.java | 8 +
.../server/master/state/TabletStateStore.java | 12 +-
.../master/state/ZooTabletStateStore.java | 33 +-
.../accumulo/server/util/ListVolumesUsed.java | 3 -
.../server/util/MasterMetadataUtil.java | 18 +-
.../accumulo/server/util/MetadataTableUtil.java | 218 --
.../server/util/ReplicationTableUtil.java | 13 +-
.../server/util/ReplicationTableUtilTest.java | 2 +-
.../gc/GarbageCollectWriteAheadLogs.java| 447 ---
.../accumulo/gc/SimpleGarbageCollector.java | 1 -
.../CloseWriteAheadLogReferences.java | 25 +-
.../gc/GarbageCollectWriteAheadLogsTest.java| 568 --
.../CloseWriteAheadLogReferencesTest.java | 152 +---
.../java/org/apache/accumulo/master/Master.java | 3 +
.../master/MasterClientServiceHandler.java | 4 +-
.../accumulo/master/TabletGroupWatcher.java | 24 +-
.../accumulo/master/replication/WorkMaker.java | 1 +
.../accumulo/master/state/MergeStats.java | 3 +-
.../apache/accumulo/master/TestMergeState.java | 2 +-
.../master/state/RootTabletStateStoreTest.java | 4 +-
.../server/GarbageCollectionLogger.java | 5 +-
.../apache/accumulo/tserver/TabletLevel.java| 34 +
.../apache/accumulo/tserver/TabletServer.java | 166 ++--
.../apache/accumulo/tserver/log/DfsLogger.java | 7 +-
.../accumulo/tserver/log/SortedLogRecovery.java | 8 +-
.../tserver/log/TabletServerLogger.java | 183 +++--
.../accumulo/tserver/tablet/CommitSession.java | 3 +-
.../tserver/tablet/DatafileManager.java | 4 +-
.../apache/accumulo/tserver/tablet/Tablet.java | 58 +-
.../tserver/tablet/TabletCommitter.java | 3 +-
.../test/performance/thrift/NullTserver.java| 3 -
.../accumulo/proxy/ProxyDurabilityIT.java | 5 +
.../test/BadDeleteMarkersCreatedIT.java | 2 +-
.../org/apache/accumulo/test/BalanceIT.java | 20 +-
.../org/apache/accumulo/test/CleanWalIT.java| 1 +
.../accumulo/test/ConditionalWriterIT.java | 1 +
.../MissingWalHeaderCompletesRecoveryIT.java| 14 +-
.../accumulo/test/NoMutationRecoveryIT.java | 132
.../org/apache/accumulo/test/ShellServerIT.java | 2 +-
.../accumulo/test/functional/ExamplesIT.java| 6 +-
.../test/functional/WatchTheWatchCountIT.java | 4 +-
.../test/performance/RollWALPerformanceIT.java | 126
...bageCollectorCommunicatesWithTServersIT.java | 35 +-
.../replication/MultiInstanceReplicationIT.java | 2 +-
.../test/replication/ReplicationIT.java | 370 +++--
58 files changed, 1149 insertions(+), 2586 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b2539fb1/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
index 6fdf4db..bcdadff 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
@@ -228,9 +228,7 @@ public class ReplicationOperationsImpl implements
ReplicationOperations {
try {
for (EntryKey,Value entry : metaBs) {
[12/34] accumulo git commit: ACCUMULO-3423 merging master
ACCUMULO-3423 merging master Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ac176e19 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ac176e19 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ac176e19 Branch: refs/heads/master Commit: ac176e193769b37dd3d21894d1a084426d9e7e1d Parents: f1591f0 bddb007 Author: Eric C. Newton eric.new...@gmail.com Authored: Wed Mar 18 15:41:32 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Wed Mar 18 15:41:32 2015 -0400 -- .../accumulo/core/bloomfilter/BloomFilter.java | 5 +- .../accumulo/core/client/impl/MasterClient.java | 5 +- .../client/impl/MultiTableBatchWriterImpl.java | 5 +- .../core/client/impl/ScannerIterator.java | 28 +- .../accumulo/core/client/impl/ServerClient.java | 5 +- .../accumulo/core/client/impl/Tables.java | 5 +- .../client/impl/TabletServerBatchReader.java| 6 +- .../client/impl/TabletServerBatchWriter.java| 5 +- .../accumulo/core/client/impl/Writer.java | 5 +- .../client/mock/MockInstanceOperations.java | 5 +- .../client/mock/MockNamespaceOperations.java| 5 +- .../core/client/mock/MockTableOperations.java | 5 +- .../core/conf/AccumuloConfiguration.java| 6 +- .../accumulo/core/conf/ConfigSanityCheck.java | 8 +- .../accumulo/core/conf/ConfigurationDocGen.java | 7 +- .../org/apache/accumulo/core/conf/Property.java | 12 +- .../accumulo/core/conf/SiteConfiguration.java | 6 +- .../file/blockfile/impl/CachableBlockFile.java | 6 +- .../accumulo/core/file/rfile/CreateEmpty.java | 5 +- .../accumulo/core/file/rfile/PrintInfo.java | 5 +- .../apache/accumulo/core/file/rfile/RFile.java | 6 +- .../core/iterators/AggregatingIterator.java | 6 +- .../accumulo/core/iterators/DebugIterator.java | 5 +- .../accumulo/core/iterators/IteratorUtil.java | 5 +- .../iterators/aggregation/LongSummation.java| 6 +- .../core/iterators/system/MapFileIterator.java | 6 +- .../core/iterators/system/VisibilityFilter.java | 6 +- .../core/iterators/user/RegExFilter.java| 6 +- .../iterators/user/TransformingIterator.java| 6 +- .../core/replication/ReplicationTable.java | 5 +- .../accumulo/core/rpc/SslConnectionParams.java | 5 +- .../CachingHDFSSecretKeyEncryptionStrategy.java | 19 +- .../security/crypto/CryptoModuleFactory.java| 13 +- .../security/crypto/DefaultCryptoModule.java| 5 +- .../crypto/DefaultCryptoModuleUtils.java| 5 +- .../crypto/DiscardCloseOutputStream.java| 5 +- .../NonCachingSecretKeyEncryptionStrategy.java | 19 +- .../accumulo/core/trace/DistributedTrace.java | 5 +- .../org/apache/accumulo/core/util/CleanUp.java | 5 +- .../org/apache/accumulo/core/util/Merge.java| 5 +- .../accumulo/core/util/UtilWaitThread.java | 6 +- .../core/util/format/FormatterFactory.java | 6 +- .../apache/accumulo/core/volume/VolumeImpl.java | 6 +- .../apache/accumulo/core/zookeeper/ZooUtil.java | 6 +- .../core/file/BloomFilterLayerLookupTest.java | 5 +- .../iterators/aggregation/NumSummationTest.java | 8 +- .../iterators/user/VersioningIteratorTest.java | 13 +- .../main/asciidoc/accumulo_user_manual.asciidoc | 6 +- .../main/asciidoc/chapters/iterator_design.txt | 386 +++ docs/src/main/resources/isolation.html | 2 +- .../simple/client/RandomBatchScanner.java | 7 +- .../examples/simple/client/RowOperations.java | 6 +- .../examples/simple/client/TracingExample.java | 5 +- .../examples/simple/dirlist/Viewer.java | 5 +- .../simple/filedata/ChunkInputStream.java | 6 +- .../examples/simple/helloworld/ReadData.java| 5 +- .../simple/isolation/InterferenceTest.java | 7 +- .../examples/simple/mapreduce/NGramIngest.java | 5 +- .../simple/mapreduce/TokenFileWordCount.java| 6 +- .../examples/simple/mapreduce/WordCount.java| 5 +- .../simple/mapreduce/bulk/VerifyIngest.java | 5 +- .../examples/simple/reservations/ARS.java | 6 +- .../examples/simple/dirlist/CountTest.java | 6 +- .../simple/filedata/ChunkInputStreamTest.java | 15 +- .../org/apache/accumulo/fate/AdminUtil.java | 7 +- .../org/apache/accumulo/fate/AgeOffStore.java | 6 +- .../java/org/apache/accumulo/fate/Fate.java | 6 +- .../apache/accumulo/fate/util/AddressUtil.java | 6 +- .../accumulo/fate/util/LoggingRunnable.java | 3 +- .../accumulo/fate/util/UtilWaitThread.java | 6 +- .../zookeeper/DistributedReadWriteLock.java | 6 +- .../apache/accumulo/fate/zookeeper/Retry.java | 6 +- .../fate/zookeeper/TransactionWatcher.java | 6 +- .../accumulo/fate/zookeeper/ZooCache.java | 7 +-
[14/34] accumulo git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/accumulo
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/accumulo Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/daa38ce9 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/daa38ce9 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/daa38ce9 Branch: refs/heads/master Commit: daa38ce90f41d12aa28607963e8c512f1f049522 Parents: 34af43f 4ca3143 Author: Eric C. Newton eric.new...@gmail.com Authored: Mon Mar 23 12:53:50 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Mon Mar 23 12:53:50 2015 -0400 -- .../accumulo/monitor/servlets/trace/Basic.java | 85 +++- .../monitor/servlets/trace/ListType.java| 26 +- .../monitor/servlets/trace/ShowTrace.java | 30 +-- .../monitor/servlets/trace/Summary.java | 34 +--- .../org/apache/accumulo/tracer/TraceServer.java | 31 ++- 5 files changed, 164 insertions(+), 42 deletions(-) --
[03/34] accumulo git commit: ACCUMULO-3625 use log markers against tservers, not tablets
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b2539fb1/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index ed7626e..a95cffa 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -23,8 +23,6 @@ import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -60,6 +58,7 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.CurrentLogsSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
@@ -86,6 +85,7 @@ import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.fs.FileRef;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.tablets.TabletTime;
import org.apache.accumulo.server.zookeeper.ZooLock;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
@@ -120,7 +120,7 @@ public class MetadataTableUtil {
return metadataTable;
}
- private synchronized static Writer getRootTable(ClientContext context) {
+ public synchronized static Writer getRootTable(ClientContext context) {
Credentials credentials = context.getCredentials();
Writer rootTable = root_tables.get(credentials);
if (rootTable == null) {
@@ -227,7 +227,7 @@ public class MetadataTableUtil {
// add before removing in case of process death
for (LogEntry logEntry : logsToAdd)
-addLogEntry(context, logEntry, zooLock);
+addRootLogEntry(context, zooLock, logEntry);
removeUnusedWALEntries(context, extent, logsToRemove, zooLock);
} else {
@@ -252,6 +252,39 @@ public class MetadataTableUtil {
}
}
+ private static interface ZooOperation {
+void run(IZooReaderWriter rw) throws KeeperException,
InterruptedException, IOException;
+ }
+
+ private static void retryZooKeeperUpdate(ClientContext context, ZooLock
zooLock, ZooOperation op) {
+while (true) {
+ try {
+IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+if (zoo.isLockHeld(zooLock.getLockID())) {
+ op.run(zoo);
+}
+break;
+ } catch (KeeperException e) {
+log.error(e, e);
+ } catch (InterruptedException e) {
+log.error(e, e);
+ } catch (IOException e) {
+log.error(e, e);
+ }
+ UtilWaitThread.sleep(1000);
+}
+ }
+
+ private static void addRootLogEntry(AccumuloServerContext context, ZooLock
zooLock, final LogEntry entry) {
+retryZooKeeperUpdate(context, zooLock, new ZooOperation() {
+ @Override
+ public void run(IZooReaderWriter rw) throws KeeperException,
InterruptedException, IOException {
+String root = getZookeeperLogLocation();
+rw.putPersistentData(root + / + entry.getUniqueID(),
entry.toBytes(), NodeExistsPolicy.OVERWRITE);
+ }
+});
+ }
+
public static SortedMapFileRef,DataFileValue getDataFileSizes(KeyExtent
extent, ClientContext context) throws IOException {
TreeMapFileRef,DataFileValue sizes = new
TreeMapFileRef,DataFileValue();
@@ -451,34 +484,6 @@ public class MetadataTableUtil {
return ZooUtil.getRoot(HdfsZooInstance.getInstance()) +
RootTable.ZROOT_TABLET_WALOGS;
}
- public static void addLogEntry(ClientContext context, LogEntry entry,
ZooLock zooLock) {
-if (entry.extent.isRootTablet()) {
- String root = getZookeeperLogLocation();
- while (true) {
-try {
- IZooReaderWriter zoo = ZooReaderWriter.getInstance();
- if (zoo.isLockHeld(zooLock.getLockID())) {
-String[] parts = entry.filename.split(/);
-String uniqueId = parts[parts.length - 1];
-zoo.putPersistentData(root + / + uniqueId, entry.toBytes(),
NodeExistsPolicy.OVERWRITE);
- }
- break;
-} catch (KeeperException e) {
- log.error(e, e);
-} catch
[11/34] accumulo git commit: ACCUMULO-3423 merging master
ACCUMULO-3423 merging master Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f1591f03 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f1591f03 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f1591f03 Branch: refs/heads/master Commit: f1591f033ed010f490686b836fe6d6d52cb98238 Parents: f4f28c7 5c670fa Author: Eric C. Newton eric.new...@gmail.com Authored: Tue Mar 17 08:49:11 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Tue Mar 17 08:49:11 2015 -0400 -- TESTING.md | 28 +- core/pom.xml| 27 - core/src/main/findbugs/exclude-filter.xml | 46 +- .../accumulo/core/bloomfilter/BloomFilter.java |3 - .../client/admin/ReplicationOperations.java | 13 +- .../core/client/impl/ClientContext.java |4 +- .../core/client/impl/ConditionalWriterImpl.java |9 +- .../client/impl/NamespaceOperationsImpl.java|8 +- .../client/impl/ReplicationOperationsImpl.java | 125 +- .../core/client/impl/TableOperationsImpl.java | 17 + .../core/client/impl/TabletLocator.java |1 - .../client/mapreduce/AbstractInputFormat.java | 16 +- .../core/client/replication/ReplicaSystem.java | 50 - .../replication/ReplicaSystemFactory.java | 82 -- .../tokens/CredentialProviderToken.java |2 +- .../client/security/tokens/DelegationToken.java |2 +- .../client/security/tokens/KerberosToken.java |7 +- .../core/client/security/tokens/NullToken.java |6 +- .../accumulo/core/conf/SiteConfiguration.java |4 +- .../core/file/blockfile/cache/CachedBlock.java | 14 + .../core/file/blockfile/cache/ClassSize.java| 47 +- .../file/blockfile/cache/LruBlockCache.java |6 + .../accumulo/core/iterators/OrIterator.java | 12 + .../core/iterators/user/AgeOffFilter.java |1 - .../core/iterators/user/TimestampFilter.java|4 +- .../user/WholeColumnFamilyIterator.java |6 +- .../core/master/thrift/MasterClientService.java | 1213 ++ .../replication/PrintReplicationRecords.java| 97 -- .../core/replication/ReplicaSystemHelper.java | 72 -- .../core/replication/ReplicationTable.java |1 - .../core/replication/StatusFormatter.java | 187 --- .../accumulo/core/replication/StatusUtil.java | 225 .../core/replication/proto/Replication.java | 949 -- .../apache/accumulo/core/rpc/ThriftUtil.java|8 +- .../CachingHDFSSecretKeyEncryptionStrategy.java |2 +- .../apache/accumulo/core/util/CreateToken.java |6 +- .../org/apache/accumulo/core/util/Version.java |6 +- .../core/util/format/DefaultFormatter.java | 35 +- core/src/main/protobuf/replication.proto| 26 - core/src/main/scripts/generate-protobuf.sh | 98 -- core/src/main/thrift/master.thrift |3 + .../accumulo/core/cli/TestClientOpts.java |6 +- .../client/impl/TableOperationsHelperTest.java |4 +- .../mapred/AccumuloFileOutputFormatTest.java| 15 +- .../mapred/AccumuloRowInputFormatTest.java |8 +- .../mapreduce/AccumuloFileOutputFormatTest.java | 11 +- .../mapreduce/AccumuloRowInputFormatTest.java |8 +- .../conf/CredentialProviderFactoryShimTest.java |8 +- .../iterators/aggregation/NumSummationTest.java | 17 +- .../ReplicationOperationsImplTest.java | 409 -- .../core/replication/StatusUtilTest.java| 57 - .../core/replication/proto/StatusTest.java | 36 - .../core/util/LocalityGroupUtilTest.java|4 +- .../core/util/format/HexFormatterTest.java |2 +- .../simple/src/main/findbugs/exclude-filter.xml |5 + .../simple/client/RandomBatchScanner.java |3 +- .../examples/simple/client/RowOperations.java | 30 +- .../simple/filedata/ChunkInputFormatTest.java | 16 +- .../simple/filedata/ChunkInputStreamTest.java | 40 +- fate/src/main/findbugs/exclude-filter.xml |5 + .../apache/accumulo/maven/plugin/StartMojo.java |4 +- .../impl/MiniAccumuloClusterImpl.java | 28 +- .../MiniAccumuloClusterStartStopTest.java |6 +- .../minicluster/MiniAccumuloClusterTest.java|5 +- .../impl/MiniAccumuloClusterImplTest.java |6 +- pom.xml |2 +- proxy/src/main/findbugs/exclude-filter.xml |7 +- .../org/apache/accumulo/proxy/ProxyServer.java | 30 +- server/base/pom.xml | 29 + .../base/src/main/findbugs/exclude-filter.xml | 12 + .../server/constraints/MetadataConstraints.java |2 +- .../accumulo/server/data/ServerMutation.java| 13 +- .../apache/accumulo/server/fs/VolumeUtil.java |4 +-
[05/34] accumulo git commit: ACCUMULO-3423 respond to elserj's review
ACCUMULO-3423 respond to elserj's review
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c8f3b7d3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c8f3b7d3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c8f3b7d3
Branch: refs/heads/master
Commit: c8f3b7d3b8591ea2330437549b9578ae9feebaaf
Parents: b2539fb
Author: Eric C. Newton eric.new...@gmail.com
Authored: Thu Feb 26 15:17:04 2015 -0500
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Thu Feb 26 15:17:04 2015 -0500
--
.../accumulo/core/replication/StatusUtil.java | 9 +
.../thrift/TabletClientService.java | 749 ++-
core/src/main/thrift/tabletserver.thrift| 1 +
.../server/master/state/MetaDataStateStore.java | 3 +
.../master/state/MetaDataTableScanner.java | 7 +-
.../server/master/state/TabletStateStore.java | 3 +
.../master/state/ZooTabletStateStore.java | 1 +
.../accumulo/server/util/MetadataTableUtil.java | 7 +-
.../gc/GarbageCollectWriteAheadLogs.java| 4 +-
.../accumulo/master/replication/WorkMaker.java | 2 +-
.../apache/accumulo/tserver/TabletServer.java | 5 +
.../tserver/log/TabletServerLogger.java | 42 +-
.../apache/accumulo/tserver/tablet/Tablet.java | 1 +
.../test/performance/thrift/NullTserver.java| 3 +
14 files changed, 793 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c8f3b7d3/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
b/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
index a7cd3f5..d8ec403 100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
@@ -32,6 +32,7 @@ public class StatusUtil {
private static final Value INF_END_REPLICATION_STATUS_VALUE,
CLOSED_STATUS_VALUE;
private static final Status.Builder CREATED_STATUS_BUILDER;
+ private static final Status.Builder INF_END_REPLICATION_STATUS_BUILDER;
static {
CREATED_STATUS_BUILDER = Status.newBuilder();
@@ -45,6 +46,7 @@ public class StatusUtil {
builder.setEnd(0);
builder.setInfiniteEnd(true);
builder.setClosed(false);
+INF_END_REPLICATION_STATUS_BUILDER = builder;
INF_END_REPLICATION_STATUS = builder.build();
INF_END_REPLICATION_STATUS_VALUE =
ProtobufUtil.toValue(INF_END_REPLICATION_STATUS);
@@ -153,6 +155,13 @@ public class StatusUtil {
/**
* @return A {@link Status} for an open file of unspecified length, all of
which needs replicating.
*/
+ public static Status openWithUnknownLength(long timeCreated) {
+return
INF_END_REPLICATION_STATUS_BUILDER.setCreatedTime(timeCreated).build();
+ }
+
+ /**
+ * @return A {@link Status} for an open file of unspecified length, all of
which needs replicating.
+ */
public static Status openWithUnknownLength() {
return INF_END_REPLICATION_STATUS;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c8f3b7d3/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
index d6d4afd..02bd4e1 100644
---
a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
+++
b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
@@ -110,6 +110,8 @@ import org.slf4j.LoggerFactory;
public ListActiveCompaction
getActiveCompactions(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.security.thrift.TCredentials credentials) throws
org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException,
org.apache.thrift.TException;
+public void removeLogs(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.security.thrift.TCredentials credentials, ListString
filenames) throws org.apache.thrift.TException;
+
public ListString
getActiveLogs(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.security.thrift.TCredentials credentials) throws
org.apache.thrift.TException;
}
@@ -174,6 +176,8 @@ import org.slf4j.LoggerFactory;
public void
getActiveCompactions(org.apache.accumulo.core.trace.thrift.TInfo tinfo,
org.apache.accumulo.core.security.thrift.TCredentials credentials,
[32/34] accumulo git commit: ACCUMULO-3488 ACCUMULO-3423 sync these two changes
ACCUMULO-3488 ACCUMULO-3423 sync these two changes
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/55981ad8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/55981ad8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/55981ad8
Branch: refs/heads/master
Commit: 55981ad881e7b79423799a3473856b4b9d768689
Parents: 3fdd29f
Author: Eric C. Newton eric.new...@gmail.com
Authored: Fri Apr 24 19:05:50 2015 -0400
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Fri Apr 24 19:05:50 2015 -0400
--
.../base/src/main/java/org/apache/accumulo/server/TabletLevel.java | 2 +-
.../accumulo/gc/replication/CloseWriteAheadLogReferencesTest.java | 1 -
.../test/java/org/apache/accumulo/tserver/log/LogEntryTest.java| 2 +-
.../apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/WALSunnyDayIT.java| 2 +-
.../java/org/apache/accumulo/test/replication/ReplicationIT.java | 2 +-
6 files changed, 5 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/55981ad8/server/base/src/main/java/org/apache/accumulo/server/TabletLevel.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/TabletLevel.java
b/server/base/src/main/java/org/apache/accumulo/server/TabletLevel.java
index 91e5ee9..c87c7e1 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/TabletLevel.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/TabletLevel.java
@@ -16,7 +16,7 @@
*/
package org.apache.accumulo.server;
-import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.impl.KeyExtent;
public enum TabletLevel {
ROOT,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/55981ad8/server/gc/src/test/java/org/apache/accumulo/gc/replication/CloseWriteAheadLogReferencesTest.java
--
diff --git
a/server/gc/src/test/java/org/apache/accumulo/gc/replication/CloseWriteAheadLogReferencesTest.java
b/server/gc/src/test/java/org/apache/accumulo/gc/replication/CloseWriteAheadLogReferencesTest.java
index 78a5bd5..9fcfec9 100644
---
a/server/gc/src/test/java/org/apache/accumulo/gc/replication/CloseWriteAheadLogReferencesTest.java
+++
b/server/gc/src/test/java/org/apache/accumulo/gc/replication/CloseWriteAheadLogReferencesTest.java
@@ -48,7 +48,6 @@ import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.metadata.MetadataTable;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.CurrentLogsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/55981ad8/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LogEntryTest.java
--
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LogEntryTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LogEntryTest.java
index 44058d3..ee3c621 100644
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LogEntryTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/LogEntryTest.java
@@ -19,8 +19,8 @@ package org.apache.accumulo.tserver.log;
import static org.junit.Assert.assertEquals;
import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.hadoop.io.Text;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/55981ad8/test/src/test/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java
--
diff --git
a/test/src/test/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java
b/test/src/test/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java
index 27f1f69..e315841 100644
---
a/test/src/test/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java
+++
b/test/src/test/java/org/apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java
@@ -26,8 +26,8 @@ import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import
[2/5] accumulo git commit: ACCUMULO-3423 optimize WAL metadata table updates
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3fdd29f5/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
--
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index afd3454..aeb73b4 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -19,7 +19,6 @@ package org.apache.accumulo.tserver;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.server.problems.ProblemType.TABLET_LOAD;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.net.UnknownHostException;
@@ -30,6 +29,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -45,6 +45,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CancellationException;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadPoolExecutor;
@@ -147,8 +148,8 @@ import
org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.GarbageCollectionLogger;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.ServerOpts;
+import org.apache.accumulo.server.TabletLevel;
import org.apache.accumulo.server.client.ClientServiceHandler;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfigurationFactory;
@@ -1440,6 +1441,7 @@ public class TabletServer extends AccumuloServerContext
implements Runnable {
}
}
+
@Override
public void loadTablet(TInfo tinfo, TCredentials credentials, String lock,
final TKeyExtent textent) {
@@ -1500,6 +1502,7 @@ public class TabletServer extends AccumuloServerContext
implements Runnable {
final AssignmentHandler ah = new AssignmentHandler(extent);
// final Runnable ah = new LoggingRunnable(log, );
// Root tablet assignment must take place immediately
+
if (extent.isRootTablet()) {
new Daemon(Root Tablet Assignment) {
@Override
@@ -1692,66 +1695,6 @@ public class TabletServer extends AccumuloServerContext
implements Runnable {
}
@Override
-public void removeLogs(TInfo tinfo, TCredentials credentials, ListString
filenames) throws TException {
- String myname = getClientAddressString();
- myname = myname.replace(':', '+');
- SetString loggers = new HashSetString();
- logger.getLogFiles(loggers);
- SetString loggerUUIDs = new HashSetString();
- for (String logger : loggers)
-loggerUUIDs.add(new Path(logger).getName());
-
- nextFile: for (String filename : filenames) {
-String uuid = new Path(filename).getName();
-// skip any log we're currently using
-if (loggerUUIDs.contains(uuid))
- continue nextFile;
-
-ListTablet onlineTabletsCopy = new ArrayListTablet();
-synchronized (onlineTablets) {
- onlineTabletsCopy.addAll(onlineTablets.values());
-}
-for (Tablet tablet : onlineTabletsCopy) {
- for (String current : tablet.getCurrentLogFiles()) {
-if (current.contains(uuid)) {
- log.info(Attempted to delete + filename + from tablet +
tablet.getExtent());
- continue nextFile;
-}
- }
-}
-
-try {
- Path source = new Path(filename);
- if
(TabletServer.this.getConfiguration().getBoolean(Property.TSERV_ARCHIVE_WALOGS))
{
-Path walogArchive = fs.matchingFileSystem(source,
ServerConstants.getWalogArchives());
-fs.mkdirs(walogArchive);
-Path dest = new Path(walogArchive, source.getName());
-log.info(Archiving walog + source + to + dest);
-if (!fs.rename(source, dest))
- log.error(rename is unsuccessful);
- } else {
-log.info(Deleting walog + filename);
-Path sourcePath = new Path(filename);
-if
(!(!TabletServer.this.getConfiguration().getBoolean(Property.GC_TRASH_IGNORE)
fs.moveToTrash(sourcePath))
- !fs.deleteRecursively(sourcePath))
- log.warn(Failed to delete walog + source);
-for (String recovery : ServerConstants.getRecoveryDirs()) {
-
[1/5] accumulo git commit: ACCUMULO-3423 optimize WAL metadata table updates
Repository: accumulo
Updated Branches:
refs/heads/1.7 ea25e98c9 - 55981ad88
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3fdd29f5/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
--
diff --git
a/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
b/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
new file mode 100644
index 000..490bd7c
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.apache.accumulo.core.conf.Property.GC_CYCLE_DELAY;
+import static org.apache.accumulo.core.conf.Property.GC_CYCLE_START;
+import static org.apache.accumulo.core.conf.Property.INSTANCE_ZK_TIMEOUT;
+import static org.apache.accumulo.core.conf.Property.TSERV_WALOG_MAX_SIZE;
+import static org.apache.accumulo.core.conf.Property.TSERV_WAL_REPLICATION;
+import static org.apache.accumulo.core.security.Authorizations.EMPTY;
+import static org.apache.accumulo.minicluster.ServerType.GARBAGE_COLLECTOR;
+import static org.apache.accumulo.minicluster.ServerType.TABLET_SERVER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.CurrentLogsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.master.state.SetGoalState;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterControl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class WALSunnyDayIT extends ConfigurableMacIT {
+
+ private static final Text CF = new Text(new byte[0]);
+
+ @Override
+ protected void configure(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
+cfg.setProperty(GC_CYCLE_DELAY, 1s);
+cfg.setProperty(GC_CYCLE_START, 0s);
+cfg.setProperty(TSERV_WALOG_MAX_SIZE, 1M);
+cfg.setProperty(TSERV_WAL_REPLICATION, 1);
+cfg.setProperty(INSTANCE_ZK_TIMEOUT, 3s);
+cfg.setNumTservers(1);
+hadoopCoreSite.set(fs.file.impl, RawLocalFileSystem.class.getName());
+ }
+
+ int countTrue(CollectionBoolean bools) {
+int result = 0;
+for (Boolean b : bools) {
+ if (b.booleanValue())
+result ++;
+}
+return result;
+ }
+
+ @Test
+ public void test() throws Exception {
+MiniAccumuloClusterImpl mac = getCluster();
+MiniAccumuloClusterControl control = mac.getClusterControl();
+control.stop(GARBAGE_COLLECTOR);
+Connector c = getConnector();
+ZooKeeper zoo = new ZooKeeper(c.getInstance().getZooKeepers(),
c.getInstance().getZooKeepersSessionTimeOut(), new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+
[3/5] accumulo git commit: ACCUMULO-3423 optimize WAL metadata table updates
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3fdd29f5/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
--
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 1735c0d..9f537af 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -18,7 +18,7 @@ package org.apache.accumulo.gc;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -28,49 +28,56 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
-import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.gc.thrift.GCStatus;
import org.apache.accumulo.core.gc.thrift.GcCycleStats;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.CurrentLogsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
import org.apache.accumulo.core.protobuf.ProtobufUtil;
import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection;
import org.apache.accumulo.core.replication.ReplicationTable;
import org.apache.accumulo.core.replication.ReplicationTableOfflineException;
-import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
-import org.apache.accumulo.core.trace.Span;
import org.apache.accumulo.core.trace.Trace;
-import org.apache.accumulo.core.trace.Tracer;
-import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.master.LiveTServerSet;
+import org.apache.accumulo.server.master.LiveTServerSet.Listener;
+import org.apache.accumulo.server.master.state.MetaDataStateStore;
+import org.apache.accumulo.server.master.state.RootTabletStateStore;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletState;
import org.apache.accumulo.server.replication.StatusUtil;
import org.apache.accumulo.server.replication.proto.Replication.Status;
-import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.thrift.TException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.io.Text;
+import org.apache.htrace.Span;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
import com.google.common.net.HostAndPort;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -79,8 +86,8 @@ public class GarbageCollectWriteAheadLogs {
private final AccumuloServerContext context;
private final VolumeManager fs;
-
- private boolean useTrash;
+ private final boolean useTrash;
+ private final LiveTServerSet liveServers;
/**
* Creates a new GC WAL object.
@@ -96,56 +103,37 @@ public class GarbageCollectWriteAheadLogs {
this.context = context;
this.fs = fs;
this.useTrash = useTrash;
- }
-
- /**
- * Gets the instance used by this object.
- *
- * @return instance
- */
- Instance getInstance() {
-return context.getInstance();
- }
-
- /**
- * Gets the
[23/34] accumulo git commit: ACCUMULO-3423 got most ITs working again
ACCUMULO-3423 got most ITs working again
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/35e3f12c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/35e3f12c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/35e3f12c
Branch: refs/heads/master
Commit: 35e3f12c031b6ddf2c3936c77c1ccec8f981ce72
Parents: 0177e3f
Author: Eric C. Newton eric.new...@gmail.com
Authored: Tue Apr 21 13:26:17 2015 -0400
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Tue Apr 21 13:26:17 2015 -0400
--
.../org/apache/accumulo/tserver/log/TabletServerLogger.java | 9 -
.../org/apache/accumulo/test/functional/ReadWriteIT.java| 8
2 files changed, 8 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/35e3f12c/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
--
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
index 2541e50..cdee51b 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -54,7 +53,6 @@ import org.apache.accumulo.tserver.TabletMutations;
import org.apache.accumulo.tserver.TabletServer;
import org.apache.accumulo.tserver.log.DfsLogger.LoggerOperation;
import org.apache.accumulo.tserver.tablet.CommitSession;
-import org.apache.accumulo.tserver.tablet.Tablet;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -229,13 +227,6 @@ public class TabletServerLogger {
log.debug(Creating next WAL);
DfsLogger alog = new DfsLogger(tserver.getServerConfig(),
syncCounter, flushCounter);
alog.open(tserver.getClientAddressString());
-EnumSetTabletLevel levels = EnumSet.noneOf(TabletLevel.class);
-for (Tablet tablet : tserver.getOnlineTablets()) {
- levels.add(TabletLevel.getLevel(tablet.getExtent()));
-}
-for (TabletLevel level : levels) {
- tserver.addLoggersToMetadata(alog, level);
-}
log.debug(Created next WAL + alog.getFileName());
while (!nextLog.offer(alog, 12, TimeUnit.HOURS)) {
log.info(Our WAL was not used for 12 hours: +
alog.getFileName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/35e3f12c/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
--
diff --git
a/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
b/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
index 099743d..1f3e600 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ReadWriteIT.java
@@ -57,6 +57,7 @@ import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.client.security.tokens.KerberosToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
@@ -72,9 +73,11 @@ import org.apache.accumulo.fate.zookeeper.ZooLock;
import org.apache.accumulo.fate.zookeeper.ZooReader;
import org.apache.accumulo.harness.AccumuloClusterIT;
import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.accumulo.test.TestIngest;
import org.apache.accumulo.test.TestMultiTableIngest;
import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.Test;
@@ -84,6 +87,11 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Charsets;
public class ReadWriteIT extends AccumuloClusterIT {
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
+cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, 5s);
+ }
+
private static final Logger log =
[34/34] accumulo git commit: Merge branch '1.7'
Merge branch '1.7' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/51f39d29 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/51f39d29 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/51f39d29 Branch: refs/heads/master Commit: 51f39d292eb0aec15007fdd49fbf709792a3a2a7 Parents: d0a0ac0 55981ad Author: Eric C. Newton eric.new...@gmail.com Authored: Fri Apr 24 19:06:05 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Fri Apr 24 19:06:05 2015 -0400 -- .../base/src/main/java/org/apache/accumulo/server/TabletLevel.java | 2 +- .../accumulo/gc/replication/CloseWriteAheadLogReferencesTest.java | 1 - .../test/java/org/apache/accumulo/tserver/log/LogEntryTest.java| 2 +- .../apache/accumulo/test/MissingWalHeaderCompletesRecoveryIT.java | 2 +- .../java/org/apache/accumulo/test/functional/WALSunnyDayIT.java| 2 +- .../java/org/apache/accumulo/test/replication/ReplicationIT.java | 2 +- 6 files changed, 5 insertions(+), 6 deletions(-) --
[18/34] accumulo git commit: ACCUMULO-3423 updates from reviews
ACCUMULO-3423 updates from reviews
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9c2ca7a5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9c2ca7a5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9c2ca7a5
Branch: refs/heads/master
Commit: 9c2ca7a5cce8f7d0bacb100fed6405c86bde2a2d
Parents: a42
Author: Eric C. Newton eric.new...@gmail.com
Authored: Tue Apr 14 14:52:07 2015 -0400
Committer: Eric C. Newton eric.new...@gmail.com
Committed: Tue Apr 14 14:52:07 2015 -0400
--
.../core/tabletserver/log/LogEntry.java | 4 +++
.../server/master/state/MetaDataStateStore.java | 3 ++
.../accumulo/server/replication/StatusUtil.java | 12 +---
.../gc/GarbageCollectWriteAheadLogs.java| 30 ++--
.../apache/accumulo/tserver/TabletServer.java | 4 +--
.../apache/accumulo/tserver/log/DfsLogger.java | 1 +
.../tserver/log/TabletServerLogger.java | 10 +--
7 files changed, 46 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9c2ca7a5/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java
b/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java
index 964e3b3..90ce692 100644
--- a/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java
+++ b/core/src/main/java/org/apache/accumulo/core/tabletserver/log/LogEntry.java
@@ -81,6 +81,10 @@ public class LogEntry {
static private final Text EMPTY_TEXT = new Text();
public static LogEntry fromKeyValue(Key key, Value value) {
+String qualifier = key.getColumnQualifier().toString();
+if (qualifier.indexOf('/') 1) {
+ throw new IllegalArgumentException(Bad key for log entry: + key);
+}
KeyExtent extent = new KeyExtent(key.getRow(), EMPTY_TEXT);
String[] parts = key.getColumnQualifier().toString().split(/, 2);
String server = parts[0];
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9c2ca7a5/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
index adcf04d..c154bd0 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataStateStore.java
@@ -168,6 +168,9 @@ public class MetaDataStateStore extends TabletStateStore {
BatchWriter writer = createBatchWriter();
try {
for (EntryTServerInstance,ListPath entry : logs.entrySet()) {
+if (entry.getValue().isEmpty()) {
+ continue;
+}
Mutation m = new
Mutation(MetadataSchema.CurrentLogsSection.getRowPrefix() +
entry.getKey().toString());
for (Path log : entry.getValue()) {
m.put(MetadataSchema.CurrentLogsSection.COLF, new
Text(log.toString()), MetadataSchema.CurrentLogsSection.UNUSED);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9c2ca7a5/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
index e973ebc..d72eea2 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
@@ -32,7 +32,6 @@ public class StatusUtil {
private static final Value INF_END_REPLICATION_STATUS_VALUE,
CLOSED_STATUS_VALUE;
private static final Status.Builder CREATED_STATUS_BUILDER;
- private static final Status.Builder INF_END_REPLICATION_STATUS_BUILDER;
static {
CREATED_STATUS_BUILDER = Status.newBuilder();
@@ -46,7 +45,6 @@ public class StatusUtil {
builder.setEnd(0);
builder.setInfiniteEnd(true);
builder.setClosed(false);
-INF_END_REPLICATION_STATUS_BUILDER = builder;
INF_END_REPLICATION_STATUS = builder.build();
INF_END_REPLICATION_STATUS_VALUE =
ProtobufUtil.toValue(INF_END_REPLICATION_STATUS);
@@ -155,8 +153,14 @@ public class StatusUtil {
/**
* @return A {@link Status} for an open file of unspecified length, all of
which needs replicating.
*/
- public static synchronized Status openWithUnknownLength(long
[02/34] accumulo git commit: ACCUMULO-3625 use log markers against tservers, not tablets
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b2539fb1/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
--
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
index 755e322..edea93f 100644
---
a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
+++
b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
@@ -161,6 +161,7 @@ class TabletGroupWatcher extends Daemon {
ListAssignment assigned = new ArrayListAssignment();
ListTabletLocationState assignedToDeadServers = new
ArrayListTabletLocationState();
MapKeyExtent,TServerInstance unassigned = new
HashMapKeyExtent,TServerInstance();
+MapTServerInstance, ListString logsForDeadServers = new
TreeMap();
MasterState masterState = master.getMasterState();
int[] counts = new int[TabletState.values().length];
@@ -173,6 +174,7 @@ class TabletGroupWatcher extends Daemon {
if (tls == null) {
continue;
}
+ Master.log.debug(store.name() + location State: + tls);
// ignore entries for tables that do not exist in zookeeper
if
(TableManager.getInstance().getTableState(tls.extent.getTableId().toString())
== null)
continue;
@@ -182,7 +184,7 @@ class TabletGroupWatcher extends Daemon {
// Don't overwhelm the tablet servers with work
if (unassigned.size() + unloaded Master.MAX_TSERVER_WORK_CHUNK *
currentTServers.size()) {
-flushChanges(destinations, assignments, assigned,
assignedToDeadServers, unassigned);
+flushChanges(destinations, assignments, assigned,
assignedToDeadServers, logsForDeadServers, unassigned);
assignments.clear();
assigned.clear();
assignedToDeadServers.clear();
@@ -237,7 +239,7 @@ class TabletGroupWatcher extends Daemon {
assignedToDeadServers.add(tls);
if (server.equals(this.master.migrations.get(tls.extent)))
this.master.migrations.remove(tls.extent);
-// log.info(Current servers + currentTServers.keySet());
+MetadataTableUtil.fetchLogsForDeadServer(master,
master.getMasterLock(), tls.extent, tls.futureOrCurrent(), logsForDeadServers);
break;
case UNASSIGNED:
// maybe it's a finishing migration
@@ -266,7 +268,7 @@ class TabletGroupWatcher extends Daemon {
break;
case ASSIGNED_TO_DEAD_SERVER:
assignedToDeadServers.add(tls);
-// log.info(Current servers + currentTServers.keySet());
+MetadataTableUtil.fetchLogsForDeadServer(master,
master.getMasterLock(), tls.extent, tls.futureOrCurrent(), logsForDeadServers);
break;
case HOSTED:
TServerConnection conn =
this.master.tserverSet.getConnection(server);
@@ -285,7 +287,8 @@ class TabletGroupWatcher extends Daemon {
counts[state.ordinal()]++;
}
-flushChanges(destinations, assignments, assigned,
assignedToDeadServers, unassigned);
+flushChanges(destinations, assignments, assigned,
assignedToDeadServers, logsForDeadServers, unassigned);
+store.markLogsAsUnused(master, logsForDeadServers);
// provide stats after flushing changes to avoid race conditions w/
delete table
stats.end(masterState);
@@ -723,12 +726,19 @@ class TabletGroupWatcher extends Daemon {
}
}
- private void flushChanges(SortedMapTServerInstance,TabletServerStatus
currentTServers, ListAssignment assignments, ListAssignment assigned,
- ListTabletLocationState assignedToDeadServers,
MapKeyExtent,TServerInstance unassigned) throws DistributedStoreException,
TException {
+ private void flushChanges(
+ SortedMapTServerInstance,TabletServerStatus currentTServers,
+ ListAssignment assignments,
+ ListAssignment assigned,
+ ListTabletLocationState assignedToDeadServers,
+ MapTServerInstance, ListString logsForDeadServers,
+ MapKeyExtent,TServerInstance unassigned)
+ throws DistributedStoreException, TException {
if (!assignedToDeadServers.isEmpty()) {
int maxServersToShow = min(assignedToDeadServers.size(), 100);
Master.log.debug(assignedToDeadServers.size() + assigned to dead
servers: + assignedToDeadServers.subList(0, maxServersToShow) + ...);
- store.unassign(assignedToDeadServers);
+ Master.log.debug(logs for dead servers: + logsForDeadServers);
+ store.unassign(assignedToDeadServers, logsForDeadServers);
this.master.nextEvent.event(Marked %d tablets as unassigned because
they don't have current servers,
[22/34] accumulo git commit: ACCUMULO-3423 merge apache master, everything is broken
ACCUMULO-3423 merge apache master, everything is broken Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0177e3f6 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0177e3f6 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0177e3f6 Branch: refs/heads/master Commit: 0177e3f644943be1e3d3700e1be3b68c64a1b7f1 Parents: bd6dbba 7480eed Author: Eric C. Newton eric.new...@gmail.com Authored: Fri Apr 17 18:51:26 2015 -0400 Committer: Eric C. Newton eric.new...@gmail.com Committed: Fri Apr 17 18:51:26 2015 -0400 -- core/src/main/findbugs/exclude-filter.xml | 1 + .../accumulo/core/bloomfilter/Filter.java | 48 -- .../core/client/impl/ReplicationClient.java | 41 -- .../core/client/impl/ScannerIterator.java | 5 - .../core/client/impl/ScannerOptions.java| 5 - .../accumulo/core/client/impl/Tables.java | 5 - .../core/client/mapred/AbstractInputFormat.java | 263 +++ .../core/client/mapred/AccumuloInputFormat.java | 8 +- .../client/mapred/AccumuloOutputFormat.java | 1 + .../core/client/mapred/InputFormatBase.java | 60 ++- .../client/mapred/impl/BatchInputSplit.java | 42 ++ .../client/mapreduce/AbstractInputFormat.java | 249 +++ .../client/mapreduce/AccumuloInputFormat.java | 12 +- .../AccumuloMultiTableInputFormat.java | 12 +- .../client/mapreduce/AccumuloOutputFormat.java | 1 + .../core/client/mapreduce/InputFormatBase.java | 65 ++- .../core/client/mapreduce/RangeInputSplit.java | 424 ++ .../mapreduce/impl/AccumuloInputSplit.java | 445 +++ .../client/mapreduce/impl/BatchInputSplit.java | 152 +++ .../mapreduce/lib/impl/InputConfigurator.java | 36 +- .../accumulo/core/client/mock/MockAccumulo.java | 4 - .../accumulo/core/client/mock/MockTable.java| 6 +- .../org/apache/accumulo/core/conf/Property.java | 11 - .../accumulo/core/conf/SiteConfiguration.java | 17 - .../apache/accumulo/core/data/PartialKey.java | 2 + .../core/file/blockfile/ABlockReader.java | 2 - .../core/file/blockfile/ABlockWriter.java | 3 - .../core/file/blockfile/BlockFileWriter.java| 2 - .../core/file/blockfile/cache/BlockCache.java | 5 - .../core/file/blockfile/cache/CachedBlock.java | 4 - .../core/file/blockfile/cache/ClassSize.java| 98 .../file/blockfile/cache/LruBlockCache.java | 1 - .../file/blockfile/impl/CachableBlockFile.java | 28 -- .../file/keyfunctor/ColumnFamilyFunctor.java| 2 +- .../accumulo/core/file/rfile/bcfile/BCFile.java | 19 - .../core/file/rfile/bcfile/CompareUtils.java| 36 -- .../accumulo/core/file/rfile/bcfile/Utils.java | 82 .../core/metadata/schema/MetadataSchema.java| 14 - .../apache/accumulo/core/rpc/ThriftUtil.java| 70 --- .../apache/accumulo/core/util/AddressUtil.java | 4 - .../apache/accumulo/core/util/ByteArraySet.java | 4 - .../org/apache/accumulo/core/util/Daemon.java | 20 - .../apache/accumulo/core/util/MapCounter.java | 4 - .../apache/accumulo/core/util/StopWatch.java| 15 - .../core/volume/VolumeConfiguration.java| 5 - .../mapreduce/AccumuloInputFormatTest.java | 30 +- .../mapreduce/impl/BatchInputSplitTest.java | 122 + .../simple/mapreduce/TeraSortIngest.java| 2 - .../accumulo/fate/zookeeper/ZooCache.java | 9 - .../apache/accumulo/fate/zookeeper/ZooLock.java | 18 - .../accumulo/fate/zookeeper/ZooQueueLock.java | 7 - .../accumulo/fate/zookeeper/ZooSession.java | 4 - .../impl/MiniAccumuloClusterImpl.java | 15 +- .../impl/ZooKeeperBindException.java| 8 - .../apache/accumulo/server/ServerConstants.java | 4 - .../accumulo/server/client/BulkImporter.java| 9 - .../server/conf/NamespaceConfWatcher.java | 4 - .../accumulo/server/conf/TableConfWatcher.java | 4 - .../server/conf/TableParentConfiguration.java | 10 - .../server/conf/ZooConfigurationFactory.java| 11 - .../accumulo/server/log/SortedLogState.java | 8 - .../accumulo/server/master/LiveTServerSet.java | 9 - .../master/state/DistributedStoreException.java | 3 - .../server/master/state/TServerInstance.java| 4 - .../server/master/state/TabletServerState.java | 5 - .../server/metrics/AbstractMetricsImpl.java | 4 - .../server/metrics/MetricsConfiguration.java| 18 - .../server/replication/StatusFormatter.java | 5 - .../server/rpc/TBufferedServerSocket.java | 71 --- .../server/rpc/TNonblockingServerSocket.java| 7 - .../server/security/SecurityOperation.java | 4 - .../security/handler/KerberosAuthorizor.java| 8 - .../handler/KerberosPermissionHandler.java | 8 - .../accumulo/server/tables/TableManager.java| 4 -
[29/34] accumulo git commit: ACCUMULO-3423 optimize WAL metadata table updates
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3fdd29f5/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
--
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 1735c0d..9f537af 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -18,7 +18,7 @@ package org.apache.accumulo.gc;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -28,49 +28,56 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
-import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.gc.thrift.GCStatus;
import org.apache.accumulo.core.gc.thrift.GcCycleStats;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.CurrentLogsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
import org.apache.accumulo.core.protobuf.ProtobufUtil;
import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection;
import org.apache.accumulo.core.replication.ReplicationTable;
import org.apache.accumulo.core.replication.ReplicationTableOfflineException;
-import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
-import org.apache.accumulo.core.trace.Span;
import org.apache.accumulo.core.trace.Trace;
-import org.apache.accumulo.core.trace.Tracer;
-import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.server.master.LiveTServerSet;
+import org.apache.accumulo.server.master.LiveTServerSet.Listener;
+import org.apache.accumulo.server.master.state.MetaDataStateStore;
+import org.apache.accumulo.server.master.state.RootTabletStateStore;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.server.master.state.TabletState;
import org.apache.accumulo.server.replication.StatusUtil;
import org.apache.accumulo.server.replication.proto.Replication.Status;
-import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.thrift.TException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.io.Text;
+import org.apache.htrace.Span;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
import com.google.common.net.HostAndPort;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -79,8 +86,8 @@ public class GarbageCollectWriteAheadLogs {
private final AccumuloServerContext context;
private final VolumeManager fs;
-
- private boolean useTrash;
+ private final boolean useTrash;
+ private final LiveTServerSet liveServers;
/**
* Creates a new GC WAL object.
@@ -96,56 +103,37 @@ public class GarbageCollectWriteAheadLogs {
this.context = context;
this.fs = fs;
this.useTrash = useTrash;
- }
-
- /**
- * Gets the instance used by this object.
- *
- * @return instance
- */
- Instance getInstance() {
-return context.getInstance();
- }
-
- /**
- * Gets the
