[accumulo] branch master updated (0acb7f8 -> 503e030)

2019-08-14 Thread kturner 
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git.


from 0acb7f8  fix #936 store root tablet file list in Zookeeper (#1313)
 add fa1d893  fix #1309 prevent cloning accumulo.metadata (#1311)
 add 54b2c32  Examine WALs used by root tablet during GC (#1310)
 add c573676  Merge branch '1.9' into 2.0
 new 503e030  Merge branch '2.0'

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../accumulo/gc/GarbageCollectWriteAheadLogs.java  |  5 +++--
 .../apache/accumulo/master/FateServiceHandler.java |  3 ++-
 .../accumulo/master/util/TableValidators.java  | 22 ++
 .../accumulo/test/functional/CloneTestIT.java  | 16 
 4 files changed, 43 insertions(+), 3 deletions(-)

[accumulo] 01/01: Merge branch '2.0'

2019-08-14 Thread kturner 
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 503e0301b32b4c390c07dffd6581dfff4bd41eb3
Merge: 0acb7f8 c573676
Author: Keith Turner 
AuthorDate: Wed Aug 14 17:18:22 2019 -0400

Merge branch '2.0'

 .../accumulo/gc/GarbageCollectWriteAheadLogs.java  |  5 +++--
 .../apache/accumulo/master/FateServiceHandler.java |  3 ++-
 .../accumulo/master/util/TableValidators.java  | 22 ++
 .../accumulo/test/functional/CloneTestIT.java  | 16 
 4 files changed, 43 insertions(+), 3 deletions(-)

diff --cc 
server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 5bb8201,289e69d..5637a8d
--- 
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ 
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@@ -94,8 -95,8 +95,8 @@@ public class GarbageCollectWriteAheadLo
  });
  liveServers.startListeningForTabletServerChanges();
  this.walMarker = new WalStateManager(context);
- this.store = () -> Iterators.concat(new 
RootTabletStateStore(context).iterator(),
- new MetaDataStateStore(context).iterator());
 -this.store = () -> Iterators.concat(new 
ZooTabletStateStore(context).iterator(),
++this.store = () -> Iterators.concat(new 
ZooTabletStateStore(context.getAmple()).iterator(),
+ new RootTabletStateStore(context).iterator(), new 
MetaDataStateStore(context).iterator());
}
  
/**

[accumulo] 01/01: Merge branch '1.9' into 2.0

2019-08-14 Thread kturner 
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit c5736768d8ff3cb8799b5e254d7df36f1850a648
Merge: 5c58b08 54b2c32
Author: Keith Turner 
AuthorDate: Wed Aug 14 16:55:06 2019 -0400

Merge branch '1.9' into 2.0

 .../accumulo/gc/GarbageCollectWriteAheadLogs.java  |  5 +++--
 .../apache/accumulo/master/FateServiceHandler.java |  3 ++-
 .../accumulo/master/util/TableValidators.java  | 22 ++
 .../accumulo/test/functional/CloneTestIT.java  | 16 
 4 files changed, 43 insertions(+), 3 deletions(-)

diff --cc 
server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 5bb8201,23c4932..289e69d
--- 
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ 
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@@ -53,6 -60,8 +53,7 @@@ import org.apache.accumulo.server.maste
  import org.apache.accumulo.server.master.state.TServerInstance;
  import org.apache.accumulo.server.master.state.TabletLocationState;
  import org.apache.accumulo.server.master.state.TabletState;
+ import org.apache.accumulo.server.master.state.ZooTabletStateStore;
 -import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
  import org.apache.hadoop.fs.FileStatus;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.io.Text;
@@@ -88,14 -97,28 +89,14 @@@ public class GarbageCollectWriteAheadLo
  this.context = context;
  this.fs = fs;
  this.useTrash = useTrash;
 -this.liveServers = new LiveTServerSet(context, new Listener() {
 -  @Override
 -  public void update(LiveTServerSet current, Set deleted,
 -  Set added) {
 -log.debug("New tablet servers noticed: " + added);
 -log.debug("Tablet servers removed: " + deleted);
 -  }
 +this.liveServers = new LiveTServerSet(context, (current, deleted, added) 
-> {
 +  log.debug("New tablet servers noticed: {}", added);
 +  log.debug("Tablet servers removed: {}", deleted);
  });
  liveServers.startListeningForTabletServerChanges();
 -this.walMarker = new WalStateManager(context.getInstance(), 
ZooReaderWriter.getInstance());
 -this.store = new Iterable() {
 -  @Override
 -  public Iterator iterator() {
 -try {
 -  return Iterators.concat(new ZooTabletStateStore().iterator(),
 -  new RootTabletStateStore(context).iterator(),
 -  new MetaDataStateStore(context).iterator());
 -} catch (DistributedStoreException e) {
 -  throw new RuntimeException(e);
 -}
 -  }
 -};
 +this.walMarker = new WalStateManager(context);
- this.store = () -> Iterators.concat(new 
RootTabletStateStore(context).iterator(),
- new MetaDataStateStore(context).iterator());
++this.store = () -> Iterators.concat(new 
ZooTabletStateStore(context).iterator(),
++new RootTabletStateStore(context).iterator(), new 
MetaDataStateStore(context).iterator());
}
  
/**
diff --cc 
server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
index c19ee7b,7949ac2..0cf7c14
--- 
a/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
+++ 
b/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
@@@ -250,12 -225,12 +251,12 @@@ class FateServiceHandler implements Fat
case TABLE_CLONE: {
  TableOperation tableOp = TableOperation.CLONE;
  validateArgumentCount(arguments, tableOp, 2);
- TableId srcTableId = validateTableIdArgument(arguments.get(0), 
tableOp, NOT_ROOT_ID);
 -String srcTableId = validateTableIdArgument(arguments.get(0), 
tableOp, CAN_CLONE);
++TableId srcTableId = validateTableIdArgument(arguments.get(0), 
tableOp, CAN_CLONE);
  String tableName = validateTableNameArgument(arguments.get(1), 
tableOp, NOT_SYSTEM);
 -String namespaceId;
 +NamespaceId namespaceId;
  try {
namespaceId =
 -  Namespaces.getNamespaceId(master.getInstance(), 
Tables.qualify(tableName).getFirst());
 +  Namespaces.getNamespaceId(master.getContext(), 
Tables.qualify(tableName).getFirst());
  } catch (NamespaceNotFoundException e) {
// shouldn't happen, but possible once cloning between namespaces 
is supported
throw new ThriftTableOperationException(null, tableName, tableOp,
diff --cc 
server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java
index ddbce98,a9d3edb..4d44765
--- 
a/server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java
+++ 
b/server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java
@@@ -80,6 -79,28 +80,28 @@@ public class TableValidators 
  }
};
  
 -  public static final Validator CAN_CLONE = new

[accumulo] branch 2.0 updated (5c58b08 -> c573676)

2019-08-14 Thread kturner 
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a change to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git.


from 5c58b08  Update javadoc:aggregate config to match public API
 add fa1d893  fix #1309 prevent cloning accumulo.metadata (#1311)
 add 54b2c32  Examine WALs used by root tablet during GC (#1310)
 new c573676  Merge branch '1.9' into 2.0

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../accumulo/gc/GarbageCollectWriteAheadLogs.java  |  5 +++--
 .../apache/accumulo/master/FateServiceHandler.java |  3 ++-
 .../accumulo/master/util/TableValidators.java  | 22 ++
 .../accumulo/test/functional/CloneTestIT.java  | 16 
 4 files changed, 43 insertions(+), 3 deletions(-)

[accumulo] branch 1.9 updated: Examine WALs used by root tablet during GC (#1310)

2019-08-14 Thread kturner 
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/1.9 by this push:
 new 54b2c32  Examine WALs used by root tablet during GC (#1310)
54b2c32 is described below

commit 54b2c327d02b9ea30785ba2ae9f5be59264a9713
Author: Keith Turner 
AuthorDate: Wed Aug 14 16:07:07 2019 -0400

Examine WALs used by root tablet during GC (#1310)
---
 .../org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java  | 11 +--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git 
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
 
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 405a7a0..23c4932 100644
--- 
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ 
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -54,11 +54,13 @@ import 
org.apache.accumulo.server.log.WalStateManager.WalMarkerException;
 import org.apache.accumulo.server.log.WalStateManager.WalState;
 import org.apache.accumulo.server.master.LiveTServerSet;
 import org.apache.accumulo.server.master.LiveTServerSet.Listener;
+import org.apache.accumulo.server.master.state.DistributedStoreException;
 import org.apache.accumulo.server.master.state.MetaDataStateStore;
 import org.apache.accumulo.server.master.state.RootTabletStateStore;
 import org.apache.accumulo.server.master.state.TServerInstance;
 import org.apache.accumulo.server.master.state.TabletLocationState;
 import org.apache.accumulo.server.master.state.TabletState;
+import org.apache.accumulo.server.master.state.ZooTabletStateStore;
 import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
@@ -108,8 +110,13 @@ public class GarbageCollectWriteAheadLogs {
 this.store = new Iterable() {
   @Override
   public Iterator iterator() {
-return Iterators.concat(new RootTabletStateStore(context).iterator(),
-new MetaDataStateStore(context).iterator());
+try {
+  return Iterators.concat(new ZooTabletStateStore().iterator(),
+  new RootTabletStateStore(context).iterator(),
+  new MetaDataStateStore(context).iterator());
+} catch (DistributedStoreException e) {
+  throw new RuntimeException(e);
+}
   }
 };
   }

[accumulo] branch 1.9 updated: fix #1309 prevent cloning accumulo.metadata (#1311)

2019-08-14 Thread kturner 
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/1.9 by this push:
 new fa1d893  fix #1309 prevent cloning accumulo.metadata (#1311)
fa1d893 is described below

commit fa1d893f26f1f5e7974e02fc5757c5e2ff0a29df
Author: Keith Turner 
AuthorDate: Wed Aug 14 16:06:26 2019 -0400

fix #1309 prevent cloning accumulo.metadata (#1311)
---
 .../apache/accumulo/master/FateServiceHandler.java |  3 ++-
 .../accumulo/master/util/TableValidators.java  | 22 ++
 .../accumulo/test/functional/CloneTestIT.java  | 13 +
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git 
a/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
 
b/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
index c9b20cf..7949ac2 100644
--- 
a/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
+++ 
b/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java
@@ -16,6 +16,7 @@
  */
 package org.apache.accumulo.master;
 
+import static org.apache.accumulo.master.util.TableValidators.CAN_CLONE;
 import static org.apache.accumulo.master.util.TableValidators.NOT_METADATA;
 import static org.apache.accumulo.master.util.TableValidators.NOT_ROOT_ID;
 import static org.apache.accumulo.master.util.TableValidators.NOT_SYSTEM;
@@ -224,7 +225,7 @@ class FateServiceHandler implements FateService.Iface {
   case TABLE_CLONE: {
 TableOperation tableOp = TableOperation.CLONE;
 validateArgumentCount(arguments, tableOp, 2);
-String srcTableId = validateTableIdArgument(arguments.get(0), tableOp, 
NOT_ROOT_ID);
+String srcTableId = validateTableIdArgument(arguments.get(0), tableOp, 
CAN_CLONE);
 String tableName = validateTableNameArgument(arguments.get(1), 
tableOp, NOT_SYSTEM);
 String namespaceId;
 try {
diff --git 
a/server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java
 
b/server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java
index c6ef8c5..a9d3edb 100644
--- 
a/server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java
+++ 
b/server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java
@@ -79,6 +79,28 @@ public class TableValidators {
 }
   };
 
+  public static final Validator CAN_CLONE = new Validator() {
+
+private List metaIDs = Arrays.asList(RootTable.ID, 
MetadataTable.ID);
+
+@Override
+public boolean apply(String tableId) {
+  return !metaIDs.contains(tableId);
+}
+
+@Override
+public String invalidMessage(String tableId) {
+  String msg;
+  if (tableId.equals(MetadataTable.ID)) {
+msg = " Cloning " + MetadataTable.NAME
++ " is dangerous and no longer supported, see 
https://github.com/apache/accumulo/issues/1309.;;
+  } else {
+msg = "Can not clone " + RootTable.NAME;
+  }
+  return msg;
+}
+  };
+
   public static final Validator NOT_SYSTEM = new Validator() {
 
 @Override
diff --git 
a/test/src/main/java/org/apache/accumulo/test/functional/CloneTestIT.java 
b/test/src/main/java/org/apache/accumulo/test/functional/CloneTestIT.java
index 74c6885..d76a28a 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CloneTestIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CloneTestIT.java
@@ -33,6 +33,7 @@ import java.util.Set;
 import java.util.TreeSet;
 
 import org.apache.accumulo.cluster.AccumuloCluster;
+import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
 import org.apache.accumulo.core.client.Connector;
@@ -46,6 +47,7 @@ import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.harness.AccumuloClusterHarness;
@@ -301,4 +303,15 @@ public class CloneTestIT extends AccumuloClusterHarness {
 assertEquals(rows, actualRows);
   }
 
+  @Test(expected = AccumuloException.class)
+  public void testCloneRootTable() throws Exception {
+Connector conn = getConnector();
+conn.tableOperations().clone(RootTable.NAME, "rc1", true, null, null);
+  }
+
+  @Test(expected = AccumuloException.class)
+  public void testCloneMetadataTable() throws Exception {
+Connector conn = getConnector();
+conn.tableOperations().clone(MetadataTable.NAME, "mc1", true, null, null);
+  }
 }

[accumulo] branch master updated: fix #936 store root tablet file list in Zookeeper (#1313)

2019-08-14 Thread kturner 
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
 new 0acb7f8  fix #936 store root tablet file list in Zookeeper (#1313)
0acb7f8 is described below

commit 0acb7f898460c482612678db9585ba1a4fa8389c
Author: Keith Turner 
AuthorDate: Wed Aug 14 15:18:04 2019 -0400

fix #936 store root tablet file list in Zookeeper (#1313)
---
 .../apache/accumulo/core/metadata/RootTable.java   |   5 +
 .../accumulo/core/metadata/schema/Ample.java   |  34 -
 .../core/metadata/schema/RootTabletMetadata.java   |  13 +-
 .../core/metadata/schema/TabletsMetadata.java  |  39 +++---
 server/base/pom.xml|   4 +
 .../org/apache/accumulo/server/fs/VolumeUtil.java  |  81 ++--
 .../apache/accumulo/server/init/Initialize.java|  20 ++-
 .../accumulo/server/metadata/RootGcCandidates.java | 117 +
 .../accumulo/server/metadata/ServerAmpleImpl.java  |  99 ++-
 .../accumulo/server/util/MasterMetadataUtil.java   |  29 -
 .../accumulo/server/util/MetadataTableUtil.java|  50 ++--
 .../apache/accumulo/gc/SimpleGarbageCollector.java |  95 ++
 .../accumulo/master/upgrade/Upgrader9to10.java | 132 +++
 .../master/state/RootTabletStateStoreTest.java |   3 +-
 .../master/upgrade/RootFilesUpgradeTest.java}  |  48 +--
 .../org/apache/accumulo/tserver/TabletServer.java  |   7 +-
 .../accumulo/tserver/tablet/DatafileManager.java   |  76 +++
 .../apache/accumulo/tserver/tablet/RootFiles.java  | 139 -
 .../org/apache/accumulo/tserver/tablet/Tablet.java |  32 +
 .../apache/accumulo/tserver/tablet/TabletData.java |  53 
 20 files changed, 555 insertions(+), 521 deletions(-)

diff --git 
a/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java 
b/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
index 4e08c46..d943f17 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
@@ -37,6 +37,11 @@ public class RootTable {
*/
   public static final String ZROOT_TABLET = ROOT_TABLET_LOCATION;
 
+  /**
+   * ZK path relative to the zookeeper node where the root tablet gc 
candidates are stored.
+   */
+  public static final String ZROOT_TABLET_GC_CANDIDATES = ZROOT_TABLET + 
"/gc_candidates";
+
   public static final KeyExtent EXTENT = new KeyExtent(ID, null, null);
   public static final KeyExtent OLD_EXTENT =
   new KeyExtent(MetadataTable.ID, TabletsSection.getRow(MetadataTable.ID, 
null), null);
diff --git 
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java 
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 88fefa1..9b8a692 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -18,9 +18,12 @@
 package org.apache.accumulo.core.metadata.schema;
 
 import java.util.Collection;
+import java.util.Iterator;
 
 import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
 import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
 import org.apache.accumulo.core.tabletserver.log.LogEntry;
@@ -56,6 +59,31 @@ import org.apache.hadoop.io.Text;
 public interface Ample {
 
   /**
+   * Accumulo is a distributed tree with three levels. This enum is used to 
communicate to Ample
+   * that code is interested in operating on the metadata of a data level. 
Sometimes tables ids or
+   * key extents are passed to Ample in lieu of a data level, in these cases 
the data level is
+   * derived from the table id.
+   */
+  public enum DataLevel {
+ROOT(null), METADATA(RootTable.NAME), USER(MetadataTable.NAME);
+
+private final String table;
+
+private DataLevel(String table) {
+  this.table = table;
+}
+
+/**
+ * @return The name of the Accumulo table in which this data level stores 
its metadata.
+ */
+public String metaTable() {
+  if (table == null)
+throw new UnsupportedOperationException();
+  return table;
+}
+  }
+
+  /**
* Read a single tablets metadata. No checking is done for prev row, so it 
could differ.
*
* @param extent
@@ -90,7 +118,11 @@ public interface Ample {
 throw new UnsupportedOperationException();
   }
 
-  default void deleteGcCandidates(TableId tableId, Collection paths) {
+  default void deleteGcCandidates(DataLevel level, Collection paths) {
+throw new