[accumulo] 01/01: Merge branch '1.9' into main
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a165264e714e01329af41954b00d0217e982dfce Merge: 5815bf6 e3a14cd Author: Mike Miller AuthorDate: Thu Aug 13 08:12:31 2020 -0400 Merge branch '1.9' into main .../org/apache/accumulo/core/iterators/Combiner.java | 16 1 file changed, 8 insertions(+), 8 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java index 6581195,7e24f88..c943a8e --- a/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java @@@ -277,18 -278,18 +277,18 @@@ public abstract class Combiner extends combineAllColumns = false; if (options.containsKey(ALL_OPTION)) { combineAllColumns = Boolean.parseBoolean(options.get(ALL_OPTION)); - if (combineAllColumns) - return; } - if (!options.containsKey(COLUMNS_OPTION)) - throw new IllegalArgumentException("Must specify " + COLUMNS_OPTION + " option"); + if (!combineAllColumns) { + if (!options.containsKey(COLUMNS_OPTION)) + throw new IllegalArgumentException("Must specify " + COLUMNS_OPTION + " option"); - String encodedColumns = options.get(COLUMNS_OPTION); - if (encodedColumns.isEmpty()) - throw new IllegalArgumentException("The " + COLUMNS_OPTION + " must not be empty"); + String encodedColumns = options.get(COLUMNS_OPTION); - if (encodedColumns.length() == 0) ++ if (encodedColumns.isEmpty()) + throw new IllegalArgumentException("The " + COLUMNS_OPTION + " must not be empty"); - combiners = new ColumnSet(Lists.newArrayList(Splitter.on(",").split(encodedColumns))); + combiners = new ColumnSet(Lists.newArrayList(Splitter.on(",").split(encodedColumns))); + } isMajorCompaction = env.getIteratorScope() == IteratorScope.majc;
[accumulo] 01/01: Merge branch '1.9' into main
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit bd550e432d320540ccaf33509d33ca35759e2ef0 Merge: 9f29f37 18e8239 Author: Mike Miller AuthorDate: Wed Aug 12 09:08:11 2020 -0400 Merge branch '1.9' into main Conflicts: server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java .../accumulo/tserver/tablet/MinorCompactionTask.java | 15 --- 1 file changed, 12 insertions(+), 3 deletions(-) diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java index 5209fe2,8f922d2..83d060f --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java @@@ -57,47 -56,54 +57,56 @@@ class MinorCompactionTask implements Ru @Override public void run() { tablet.minorCompactionStarted(); -ProbabilitySampler sampler = new ProbabilitySampler(tracePercent); -Span minorCompaction = Trace.on("minorCompaction", sampler); +ProbabilitySampler sampler = TraceUtil.probabilitySampler(tracePercent); try { - Span span = Trace.start("waitForCommits"); - synchronized (tablet) { -commitSession.waitForCommitsToFinish(); - } - span.stop(); - span = Trace.start("start"); - FileRef newMapfileLocation = null; - FileRef tmpFileRef = null; - while (true) { -try { - if (newMapfileLocation == null) { -newMapfileLocation = tablet.getNextMapFilename(mergeFile == null ? "F" : "M"); -tmpFileRef = new FileRef(newMapfileLocation.path() + "_tmp"); + try (TraceScope minorCompaction = Trace.startSpan("minorCompaction", sampler)) { - TabletFile newFile = tablet.getNextMapFilename("F"); - TabletFile tmpFile = new TabletFile(new Path(newFile.getPathStr() + "_tmp")); +try (TraceScope span = Trace.startSpan("waitForCommits")) { + synchronized (tablet) { +commitSession.waitForCommitsToFinish(); } - // the purpose of the minor compaction start event is to keep track of the filename... in - // the case - // where the metadata table write for the minor compaction finishes and the process dies - // before - // writing the minor compaction finish event, then the start event+filename in metadata - // table will - // prevent recovery of duplicate data... the minor compaction start event could be written - // at any time - // before the metadata write for the minor compaction - tablet.getTabletServer().minorCompactionStarted(commitSession, - commitSession.getWALogSeq() + 1, newMapfileLocation.path().toString()); - break; -} catch (IOException e) { - // An IOException could have occurred while creating the new file - if (newMapfileLocation == null) -log.warn("Failed to create new file for minor compaction {}", e.getMessage(), e); - else -log.warn("Failed to write to write ahead log {}", e.getMessage(), e); } - } - span.stop(); - span = Trace.start("compact"); - this.stats = tablet.minorCompact(tablet.getTabletServer().getFileSystem(), - tablet.getTabletMemory().getMinCMemTable(), tmpFileRef, newMapfileLocation, mergeFile, - true, queued, commitSession, flushId, mincReason); - span.stop(); ++TabletFile newFile = null; ++TabletFile tmpFile = null; +try (TraceScope span = Trace.startSpan("start")) { + while (true) { +try { ++ if (newFile == null) { ++newFile = tablet.getNextMapFilename("F"); ++tmpFile = new TabletFile(new Path(newFile.getPathStr() + "_tmp")); ++ } + /* + * the purpose of the minor compaction start event is to keep track of the filename... + * in the case where the metadata table write for the minor compaction finishes and + * the process dies before writing the minor compaction finish event, then the start + * event+filename in metadata table will prevent recovery of duplicate data... the + * minor compaction start event could be written at any time before the metadata write + * for the minor compaction + */ + tablet.getTabletServer().minorCompactionStarted(commitSession, + commitSession.getWALogSeq() + 1, newFile.getMetaInsert()); + break; +} catch (IOException e) { - log.warn("Failed to write to write ahead log {}", e.getMessage(), e); ++
[accumulo] 01/01: Merge branch '1.9' into main
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 5dec4a2e5d87bf73ff52999664ee4b1c1c635980 Merge: 3b07c6d c1b3531 Author: Christopher Tubbs AuthorDate: Fri Aug 7 04:41:34 2020 -0400 Merge branch '1.9' into main .github/workflows/maven.yaml | 3 +++ 1 file changed, 3 insertions(+)
[accumulo] 01/01: Merge branch '1.9' into main
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 3b07c6d5540b3c0be9c587c3949e7008bd61248d Merge: 1f5ffd2 ed70fca Author: Christopher Tubbs AuthorDate: Fri Aug 7 04:31:58 2020 -0400 Merge branch '1.9' into main .github/workflows/maven.yaml | 15 +++ 1 file changed, 15 insertions(+)
[accumulo] 01/01: Merge branch '1.9' into main
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 24ea3bb7ffd340fc443639763a0c9b9f0ca43c68 Merge: d28ed21 37f6d78 Author: Christopher Tubbs AuthorDate: Wed Aug 5 19:11:00 2020 -0400 Merge branch '1.9' into main .../apache/accumulo/core/data/doc-files/mutation-serialization.html| 3 +++ 1 file changed, 3 insertions(+)
[accumulo] 01/01: Merge branch '1.9' into main
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d28ed21353c2b10f03fe45ea5e51d8fc8944fa0f Merge: 382054d b3d516d Author: Christopher Tubbs AuthorDate: Wed Aug 5 15:00:48 2020 -0400 Merge branch '1.9' into main .github/workflows/maven.yaml | 19 +++ 1 file changed, 11 insertions(+), 8 deletions(-) diff --cc .github/workflows/maven.yaml index dd78a36,21cf1f8..01b2acb --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@@ -24,19 -24,19 +24,20 @@@ name: Q on: push: - branches: [ 2.1, main ] -branches: [ '1.*' ] ++branches: [ main, '2.*' ] pull_request: - branches: [ 2.1, main ] -branches: [ '1.*' ] ++branches: [ main, '2.*' ] jobs: - build: + mvn: strategy: matrix: profile: - - {name: 'verify', args: 'verify javadoc:jar -DskipITs'} - - {name: 'sec-bugs', args: 'verify -PskipQA,sec-bugs -Dspotbugs.skip=false -Dspotbugs.timeout=360'} - - {name: 'hadoop-3.0.3', args: 'package -DskipTests -Dhadoop.version=3.0.3'} - - {name: 'Hadoop 2 (default)', args: 'verify javadoc:jar -DskipITs -DforkCount=1C'} - - {name: 'Hadoop 3', args: 'verify javadoc:jar -DskipITs -DforkCount=1C -Dhadoop.profile=3'} ++ - {name: 'unit-tests',args: 'verify -PskipQA -DskipTests=false -DforkCount=1C'} ++ - {name: 'qa-checks', args: 'verify javadoc:jar -Psec-bugs -DskipTests=true -Dspotbugs.timeout=360'} ++ - {name: 'hadoop-compat', args: 'package -DskipTests -Dhadoop.version=3.0.3'} fail-fast: false + timeout-minutes: 60 runs-on: ubuntu-latest steps: - uses: actions/checkout@v2
[accumulo] 01/01: Merge branch '1.9' into main
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit dfb4be4dcc0c829afd7ce83fabe5932737da67e7 Merge: 10cbb3d 346400d Author: Christopher Tubbs AuthorDate: Tue Aug 4 16:53:10 2020 -0400 Merge branch '1.9' into main assemble/conf/templates/hadoop-metrics2-accumulo.properties | 5 + test/pom.xml| 9 + 2 files changed, 14 insertions(+)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 317aa43344a2cc98804934c766f96f5eb29c1760 Merge: 287f9b3 7ed5cf2 Author: Christopher Tubbs AuthorDate: Thu Jul 16 14:25:50 2020 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d879f5849001dfcfcbe88fe69da0e77cf5a75470 Merge: 2196bc8 6b5b905 Author: Christopher Tubbs AuthorDate: Wed Jul 15 23:13:08 2020 -0400 Merge branch '1.9' (Merging #1646 into master branch to resolve #1644) .../main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java index 9657e5b,c7d040f..5ee84ab --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java @@@ -126,10 -135,10 +126,10 @@@ public class MinorCompactor extends Com return ret; } catch (IOException | UnsatisfiedLinkError e) { log.warn("MinC failed ({}) to create {} retrying ...", e.getMessage(), outputFileName); - ProblemReports.getInstance(tabletServer).report(new ProblemReport( + ProblemReports.getInstance(tabletServer.getContext()).report(new ProblemReport( getExtent().getTableId(), ProblemType.FILE_WRITE, outputFileName, e)); reportedProblem = true; - } catch (RuntimeException e) { + } catch (RuntimeException | NoClassDefFoundError e) { // if this is coming from a user iterator, it is possible that the user could change the // iterator config and that the // minor compaction would succeed
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 1d91d163b448ba9ce4086ac8eb2d58ccaa71179a Merge: 3f38d49 eaac4b0 Author: Christopher Tubbs AuthorDate: Tue Apr 21 16:51:33 2020 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 84d5b995302c8774c6e9024612aff7495dd73727 Merge: 77b515f 3b9dadc Author: Keith Turner AuthorDate: Thu Mar 12 14:17:59 2020 -0400 Merge branch '1.9' .../tableOps/tableImport/MoveExportedFiles.java| 46 +- 1 file changed, 36 insertions(+), 10 deletions(-) diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MoveExportedFiles.java index b5bb6be,000..b324f6b mode 100644,00..100644 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MoveExportedFiles.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/tableImport/MoveExportedFiles.java @@@ -1,79 -1,0 +1,105 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.master.tableOps.tableImport; + +import java.io.IOException; ++import java.util.Arrays; ++import java.util.HashSet; +import java.util.Map; ++import java.util.Set; ++import java.util.function.Function; ++import java.util.stream.Collectors; + +import org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException; +import org.apache.accumulo.core.clientImpl.thrift.TableOperation; +import org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType; +import org.apache.accumulo.fate.Repo; +import org.apache.accumulo.master.Master; +import org.apache.accumulo.master.tableOps.MasterRepo; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + ++import com.google.common.collect.Sets; ++ +class MoveExportedFiles extends MasterRepo { + private static final Logger log = LoggerFactory.getLogger(MoveExportedFiles.class); + + private static final long serialVersionUID = 1L; + + private ImportedTableInfo tableInfo; + + MoveExportedFiles(ImportedTableInfo ti) { +this.tableInfo = ti; + } + + @Override + public Repo call(long tid, Master master) throws Exception { +try { + VolumeManager fs = master.getVolumeManager(); + + Map fileNameMappings = PopulateMetadataTable.readMappingFile(fs, tableInfo); + - for (String oldFileName : fileNameMappings.keySet()) { - if (!fs.exists(new Path(tableInfo.exportDir, oldFileName))) { - throw new AcceptableThriftTableOperationException(tableInfo.tableId.canonical(), - tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER, - "File referenced by exported table does not exists " + oldFileName); - } ++ FileStatus[] exportedFiles = fs.listStatus(new Path(tableInfo.exportDir)); ++ FileStatus[] importedFiles = fs.listStatus(new Path(tableInfo.importDir)); ++ ++ Function fileStatusName = fstat -> fstat.getPath().getName(); ++ ++ Set importing = Arrays.stream(exportedFiles).map(fileStatusName) ++ .map(fileNameMappings::get).collect(Collectors.toSet()); ++ ++ Set imported = ++ Arrays.stream(importedFiles).map(fileStatusName).collect(Collectors.toSet()); ++ ++ if (log.isDebugEnabled()) { ++log.debug("Files already present in imported (target) directory: {}", ++imported.stream().collect(Collectors.joining(","))); + } + - FileStatus[] files = fs.listStatus(new Path(tableInfo.exportDir)); ++ Set missingFiles = Sets.difference(new HashSet(fileNameMappings.values()), ++ new HashSet(Sets.union(importing, imported))); ++ ++ if (!missingFiles.isEmpty()) { ++throw new AcceptableThriftTableOperationException(tableInfo.tableId.canonical(), ++tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER, ++"Missing source files corresponding to files " +++ missingFiles.stream().collect(Collectors.joining(","))); ++ } + - for (FileStatus fileStatus : files) { ++ for (FileStatus fileStatus : exportedFiles) { +
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 3a61cd23e4ef34cab0bb12474d720a43aee534da Merge: 91ae9de 99166f1 Author: Christopher Tubbs AuthorDate: Sun Mar 8 10:22:33 2020 -0400 Merge branch '1.9' NOTICE | 2 +- assemble/src/main/resources/NOTICE | 2 +- server/native/src/main/resources/NOTICE | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 20853e55fc3d556c444376ad803fe8b0a6ce4e49 Merge: e4b69f9 105c4f2 Author: Mike Miller AuthorDate: Tue Mar 3 12:03:41 2020 -0500 Merge branch '1.9' * git merge 1.9 -sours
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 08882e43ff02dc10516c47b159b0d8d5b825954c Merge: 94844b3 03deb51 Author: Christopher Tubbs AuthorDate: Thu Feb 27 14:47:37 2020 -0500 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 9b633b32e494f7b970ff43145f879550b47f7789 Merge: 9f487a0 1775486 Author: Christopher Tubbs AuthorDate: Mon Feb 24 17:10:04 2020 -0500 Merge branch '1.9' .../accumulo/test/gc/replication/CloseWriteAheadLogReferencesIT.java| 2 ++ .../java/org/apache/accumulo/test/replication/CyclicReplicationIT.java | 2 ++ .../org/apache/accumulo/test/replication/FinishedWorkUpdaterIT.java | 2 ++ .../test/replication/GarbageCollectorCommunicatesWithTServersIT.java| 2 ++ .../org/apache/accumulo/test/replication/KerberosReplicationIT.java | 2 ++ .../apache/accumulo/test/replication/MultiInstanceReplicationIT.java| 2 ++ .../org/apache/accumulo/test/replication/MultiTserverReplicationIT.java | 2 ++ .../accumulo/test/replication/RemoveCompleteReplicationRecordsIT.java | 2 ++ .../main/java/org/apache/accumulo/test/replication/ReplicationIT.java | 2 ++ .../apache/accumulo/test/replication/ReplicationOperationsImplIT.java | 2 ++ .../org/apache/accumulo/test/replication/SequentialWorkAssignerIT.java | 2 ++ .../java/org/apache/accumulo/test/replication/StatusCombinerMacIT.java | 2 ++ .../main/java/org/apache/accumulo/test/replication/StatusMakerIT.java | 2 ++ .../org/apache/accumulo/test/replication/UnorderedWorkAssignerIT.java | 2 ++ .../accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java | 2 ++ .../test/replication/UnusedWalDoesntCloseReplicationStatusIT.java | 2 ++ .../src/main/java/org/apache/accumulo/test/replication/WorkMakerIT.java | 2 ++ 17 files changed, 34 insertions(+) diff --cc test/src/main/java/org/apache/accumulo/test/gc/replication/CloseWriteAheadLogReferencesIT.java index 088e01f,55e9087..c3e8a5c --- a/test/src/main/java/org/apache/accumulo/test/gc/replication/CloseWriteAheadLogReferencesIT.java +++ b/test/src/main/java/org/apache/accumulo/test/gc/replication/CloseWriteAheadLogReferencesIT.java @@@ -58,8 -55,9 +58,9 @@@ import org.apache.accumulo.server.repli import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.hadoop.io.Text; import org.easymock.EasyMock; -import org.easymock.IAnswer; +import org.junit.After; import org.junit.Before; + import org.junit.Ignore; import org.junit.Test; import com.google.common.collect.Iterables; diff --cc test/src/main/java/org/apache/accumulo/test/replication/CyclicReplicationIT.java index 15f2773,300d043..86b437f --- a/test/src/main/java/org/apache/accumulo/test/replication/CyclicReplicationIT.java +++ b/test/src/main/java/org/apache/accumulo/test/replication/CyclicReplicationIT.java @@@ -69,8 -69,7 +70,9 @@@ import org.slf4j.LoggerFactory import com.google.common.collect.Iterables; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + + @Ignore("Replication ITs are not stable and not currently maintained") @Category(MiniClusterOnlyTests.class) public class CyclicReplicationIT { private static final Logger log = LoggerFactory.getLogger(CyclicReplicationIT.class); diff --cc test/src/main/java/org/apache/accumulo/test/replication/FinishedWorkUpdaterIT.java index 53ba61d,d8264f1..725 --- a/test/src/main/java/org/apache/accumulo/test/replication/FinishedWorkUpdaterIT.java +++ b/test/src/main/java/org/apache/accumulo/test/replication/FinishedWorkUpdaterIT.java @@@ -46,9 -43,10 +47,10 @@@ import org.junit.Test import com.google.common.collect.Iterables; + @Ignore("Replication ITs are not stable and not currently maintained") public class FinishedWorkUpdaterIT extends ConfigurableMacBase { - private Connector conn; + private AccumuloClient client; private FinishedWorkUpdater updater; @Before diff --cc test/src/main/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java index 5cef857,5343b32..d5fecb9 --- a/test/src/main/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java +++ b/test/src/main/java/org/apache/accumulo/test/replication/MultiTserverReplicationIT.java @@@ -35,11 -31,12 +35,12 @@@ import org.apache.accumulo.core.conf.Pr import org.apache.accumulo.core.replication.ReplicationConstants; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.HostAndPort; -import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.zookeeper.ZooReader; -import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; +import org.apache.accumulo.fate.zookeeper.ZooUtil; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.hadoop.conf.Configuration; + import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc test/src/main/java/org/apache/accumulo/test/replica
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 83b171d9c4f071a39ef6da1158c34dcf70f83f6a Merge: 23dc432 b67d3f9 Author: Christopher Tubbs AuthorDate: Wed Feb 19 18:26:35 2020 -0500 Merge branch '1.9' .../accumulo/core/util/UnsynchronizedBuffer.java | 7 -- .../accumulo/core/file/rfile/RelativeKeyTest.java | 24 -- .../core/util/UnsynchronizedBufferTest.java| 29 ++ 3 files changed, 34 insertions(+), 26 deletions(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 16608e9b312e969ae05d47bcba8380922c726ae2 Merge: c9c9993 b75f449 Author: Christopher Tubbs AuthorDate: Wed Feb 12 12:44:37 2020 -0500 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit c9c99935416a0ac1c929ddd6dfad34344961a5df Merge: 34c329c 7513626 Author: Mike Miller AuthorDate: Wed Feb 12 08:45:08 2020 -0500 Merge branch '1.9' server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java | 2 ++ 1 file changed, 2 insertions(+) diff --cc server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java index 2e720f9,221bb4c..9457067 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java @@@ -133,11 -183,13 +133,13 @@@ public class VolumeUtil * configured in instance.volumes.replacements. Second, if a tablet dir is no longer configured * for use it chooses a new tablet directory. */ - public static TabletFiles updateTabletVolumes(AccumuloServerContext context, ZooLock zooLock, - VolumeManager vm, KeyExtent extent, TabletFiles tabletFiles, boolean replicate) - throws IOException { -List> replacements = ServerConstants.getVolumeReplacements(); + public static TabletFiles updateTabletVolumes(ServerContext context, ZooLock zooLock, + KeyExtent extent, TabletFiles tabletFiles, boolean replicate) { +List> replacements = +ServerConstants.getVolumeReplacements(context.getConfiguration(), context.getHadoopConf()); + if (replacements.isEmpty()) + return tabletFiles; -log.trace("Using volume replacements: " + replacements); +log.trace("Using volume replacements: {}", replacements); List logsToRemove = new ArrayList<>(); List logsToAdd = new ArrayList<>();
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a6230566ef784b27adbf19b83c6a7099c8f0a77a Merge: 0f78b2d 60e293b Author: Christopher Tubbs AuthorDate: Wed Feb 5 19:02:43 2020 -0500 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit c02f701a819bcfb1481f1610187a3358b61e876b Merge: 374e517 2c1c72f Author: Christopher Tubbs AuthorDate: Fri Jan 31 08:47:49 2020 -0500 Merge branch '1.9' test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java | 2 +- .../org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java| 2 +- test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java| 2 +- .../java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java | 2 +- .../java/org/apache/accumulo/test/master/SuspendedTabletsIT.java | 5 + .../apache/accumulo/test/replication/MultiInstanceReplicationIT.java | 2 +- 6 files changed, 10 insertions(+), 5 deletions(-) diff --cc test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java index 786d9ed,b1009df..72dc26d --- a/test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java @@@ -81,9 -84,13 +81,14 @@@ public class SuspendedTabletsIT extend public static final int TABLETS = 100; @Override + protected int defaultTimeoutSeconds() { + return 5 * 60; + } + + @Override public void configure(MiniAccumuloConfigImpl cfg, Configuration fsConf) { cfg.setProperty(Property.TABLE_SUSPEND_DURATION, SUSPEND_DURATION + "ms"); +cfg.setClientProperty(ClientProperty.INSTANCE_ZOOKEEPERS_TIMEOUT, "5s"); cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s"); cfg.setNumTservers(TSERVERS); }
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 80dde6105da995af5e9b4368d21194ceda9a66b2 Merge: 3bd262f ed1bb67 Author: Mike Miller AuthorDate: Tue Jan 28 10:29:19 2020 -0500 Merge branch '1.9' Conflicts: test/src/main/java/org/apache/accumulo/test/BalanceFasterIT.java test/src/main/java/org/apache/accumulo/test/ManySplitIT.java test/src/main/java/org/apache/accumulo/test/MetaGetsReadersIT.java test/src/main/java/org/apache/accumulo/test/TotalQueuedIT.java test/src/main/java/org/apache/accumulo/test/UnusedWALIT.java test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java test/src/main/java/org/apache/accumulo/test/replication/ReplicationRandomWalkIT.java .../org/apache/accumulo/test/GarbageCollectWALIT.java | 7 ++- .../java/org/apache/accumulo/test/LargeSplitRowIT.java | 7 ++- .../org/apache/accumulo/test/MetaGetsReadersIT.java | 7 +-- .../java/org/apache/accumulo/test/MetaRecoveryIT.java | 7 ++- .../org/apache/accumulo/test/MultiTableRecoveryIT.java | 7 ++- .../org/apache/accumulo/test/ScanFlushWithTimeIT.java | 7 ++- .../java/org/apache/accumulo/test/ShellServerIT.java| 2 +- .../org/apache/accumulo/test/TabletServerGivesUpIT.java | 7 ++- .../apache/accumulo/test/TabletServerHdfsRestartIT.java | 7 ++- .../java/org/apache/accumulo/test/TotalQueuedIT.java| 7 ++- .../main/java/org/apache/accumulo/test/UnusedWALIT.java | 10 +- .../apache/accumulo/test/VerifySerialRecoveryIT.java| 7 ++- .../apache/accumulo/test/functional/DeleteRowsIT.java | 4 ++-- .../apache/accumulo/test/functional/DurabilityIT.java | 17 +++-- .../accumulo/test/functional/RegexGroupBalanceIT.java | 7 ++- .../accumulo/test/functional/SessionDurabilityIT.java | 13 + .../GarbageCollectorCommunicatesWithTServersIT.java | 2 +- .../test/replication/MultiInstanceReplicationIT.java| 2 +- 18 files changed, 95 insertions(+), 32 deletions(-) diff --cc test/src/main/java/org/apache/accumulo/test/MetaGetsReadersIT.java index c902129,43eed8b..0981dd1 --- a/test/src/main/java/org/apache/accumulo/test/MetaGetsReadersIT.java +++ b/test/src/main/java/org/apache/accumulo/test/MetaGetsReadersIT.java @@@ -57,12 -53,19 +56,17 @@@ public class MetaGetsReadersIT extends cfg.setProperty(Property.TABLE_BLOCKCACHE_ENABLED, "false"); } + @Override + protected int defaultTimeoutSeconds() { + return 2 * 60; + } + - private static Thread slowScan(final Connector c, final String tableName, + private static Thread slowScan(final AccumuloClient c, final String tableName, final AtomicBoolean stop) { -Thread thread = new Thread() { - @Override - public void run() { -try { - while (stop.get() == false) { -Scanner s = c.createScanner(tableName, Authorizations.EMPTY); +return new Thread(() -> { + try { +while (!stop.get()) { + try (Scanner s = c.createScanner(tableName, Authorizations.EMPTY)) { IteratorSetting is = new IteratorSetting(50, SlowIterator.class); SlowIterator.setSleepTime(is, 10); s.addScanIterator(is); @@@ -71,52 -74,48 +75,51 @@@ iterator.next(); } } -} catch (Exception ex) { - log.trace("{}", ex.getMessage(), ex); - stop.set(true); } + } catch (Exception ex) { +log.trace("{}", ex.getMessage(), ex); +stop.set(true); } -}; -return thread; +}); } - @Test(timeout = 2 * 60 * 1000) public void test() throws Exception { final String tableName = getUniqueNames(1)[0]; -final Connector c = getConnector(); -c.tableOperations().create(tableName); -Random random = new Random(); -BatchWriter bw = c.createBatchWriter(tableName, null); -for (int i = 0; i < 5; i++) { - byte[] row = new byte[100]; - random.nextBytes(row); - Mutation m = new Mutation(row); - m.put("", "", ""); - bw.addMutation(m); +try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) { + c.tableOperations().create(tableName); + Random random = new SecureRandom(); + try (BatchWriter bw = c.createBatchWriter(tableName)) { +for (int i = 0; i < 5; i++) { + byte[] row = new byte[100]; + random.nextBytes(row); + Mutation m = new Mutation(row); + m.put("", "", ""); + bw.addMutation(m); +} + } + c.tableOperations().flush(tableName, null, null, true); + final AtomicBool
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 3bd262f8eb8cdf81b461055f9cb9b7ce95530b3e Merge: 53db996 ce5085e Author: Mike Miller AuthorDate: Tue Jan 28 10:08:47 2020 -0500 Merge branch '1.9' * git merge 1.9 -sours * of ce5085eb4cc705bf8140b416fe508c2a2bc6ed85
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 18b68ff5a29574e25db05c50a74b695c3557bbc1 Merge: c95cd29 44e0851 Author: Christopher Tubbs AuthorDate: Wed Jan 22 19:12:19 2020 -0500 Merge branch '1.9' .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --cc .travis.yml index b41a9d2,145e531..415aa1d --- a/.travis.yml +++ b/.travis.yml @@@ -27,8 -27,7 +27,8 @@@ before_script - unset _JAVA_OPTIONS env: - BUILD_CMD="mvn clean verify javadoc:jar -DskipITs" # main build of unit tests and javadoc - - BUILD_CMD="mvn clean verify javadoc:jar -DskipITs -Dhadoop.profile=3" + - BUILD_CMD="travis_wait 30 mvn clean verify -PskipQA,sec-bugs -Dspotbugs.skip=false -Dspotbugs.timeout=180" # check for spotbugs+security + - BUILD_CMD="mvn clean package -DskipTests -Dhadoop.version=3.0.3" # quick compile to verify older API # suppress download logs which cause travis to exceed the 4MB build log output limit script: - - $BUILD_CMD -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + - $BUILD_CMD -B -Dstyle.color=always -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 273961cf40da19093728d2d91dbc6607c10cff15 Merge: 66f296e 9c957ab Author: Mike Miller AuthorDate: Thu Jan 16 15:42:18 2020 -0500 Merge branch '1.9' * git merge 1.9 -sours * Merging "Update to Java 1.8 and related fixes"
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 73dc54a44b7e24b9418e974ba9b077be2347f547 Merge: 0dae213 4627660 Author: Mike Miller AuthorDate: Tue Dec 10 15:39:24 2019 -0500 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit ccb357f540145313e2f1df349dfd02c57f359a82 Merge: 3fd5cad 2f4ad2d Author: Christopher Tubbs AuthorDate: Fri Nov 15 22:25:10 2019 -0500 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 8f96071472f7d60a4f59d93b0fc03eaf5f4515b8 Merge: c7bc980 2b8243b Author: Christopher Tubbs AuthorDate: Thu Nov 7 18:15:50 2019 -0500 Merge branch '1.9' .../clientImpl/AccumuloBulkMergeException.java | 1 + .../test/functional/FateConcurrencyIT.java | 354 ++--- .../org/apache/accumulo/test/util/SlowOps.java | 347 3 files changed, 438 insertions(+), 264 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/AccumuloBulkMergeException.java index 2a7527e,000..98d7015 mode 100644,00..100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/AccumuloBulkMergeException.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/AccumuloBulkMergeException.java @@@ -1,32 -1,0 +1,33 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.clientImpl; + +import org.apache.accumulo.core.client.AccumuloException; + +/** + * Internal class indicating a concurrent merge occurred during the new bulk import. + */ +public class AccumuloBulkMergeException extends AccumuloException { + ++ private static final long serialVersionUID = 1L; + private static final String MSG = "Concurrent merge happened"; + + public AccumuloBulkMergeException(final Throwable cause) { +super(MSG, cause); + } + +} diff --cc test/src/main/java/org/apache/accumulo/test/functional/FateConcurrencyIT.java index 8367891,c3a4d79..47a315a --- a/test/src/main/java/org/apache/accumulo/test/functional/FateConcurrencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/FateConcurrencyIT.java @@@ -33,34 -30,22 +30,25 @@@ import java.util.concurrent.Future import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; - import org.apache.accumulo.core.client.BatchWriter; - import org.apache.accumulo.core.client.IteratorSetting; - import org.apache.accumulo.core.client.Scanner; - import org.apache.accumulo.core.client.TableExistsException; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.client.impl.Tables; +import org.apache.accumulo.core.clientImpl.ClientContext; - import org.apache.accumulo.core.clientImpl.ClientInfo; +import org.apache.accumulo.core.clientImpl.Tables; import org.apache.accumulo.core.conf.Property; - import org.apache.accumulo.core.data.Key; - import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.TableId; - import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.master.state.tables.TableState; - import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.AdminUtil; import org.apache.accumulo.fate.ZooStore; import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; +import org.apache.accumulo.fate.zookeeper.ZooUtil; import org.apache.accumulo.harness.AccumuloClusterHarness; import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory; - import org.apache.hadoop.io.Text; + import org.apache.accumulo.test.util.SlowOps; import org.apache.zookeeper.KeeperException; +import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; @@@ -85,8 -70,7 +73,8 @@@ public class FateConcurrencyIT extends private static final int NUM_ROWS = 1000; private static final long SLOW_SCAN_SLEEP_MS = 250L; - private AccumuloClient accumuloClient; - private Connector connector; ++ private AccumuloClient client; + private ClientContext context; private static final ExecutorService pool = Executors.newCached
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 8faa96200e21ec2274820c6e8c57bc30eef613a8 Merge: 2e8144e f3b3590 Author: Christopher Tubbs AuthorDate: Wed Nov 6 00:35:53 2019 -0500 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 433e71703f660360c5ec17d9af7ccadf270bccd9 Merge: aabe05c 328ffa0 Author: Christopher Tubbs AuthorDate: Thu Oct 10 16:07:09 2019 -0400 Merge branch '1.9' into 2.0
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit db339e0bd2b519d5013eb0f85906e661a2cdafd4 Merge: c03e0fe 375cfbc Author: Christopher Tubbs AuthorDate: Fri Oct 4 20:14:50 2019 -0400 Merge branch '1.9' into 2.0 .../mapreduce/lib/InputConfigurator.java | 20 - .../test/mapreduce/AccumuloInputFormatIT.java | 86 ++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/mapreduce/lib/InputConfigurator.java index 62be56f,000..dd9568a mode 100644,00..100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/mapreduce/lib/InputConfigurator.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/mapreduce/lib/InputConfigurator.java @@@ -1,948 -1,0 +1,964 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.clientImpl.mapreduce.lib; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringTokenizer; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.ClientSideIteratorScanner; +import org.apache.accumulo.core.client.IsolatedScanner; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.RowIterator; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.sample.SamplerConfiguration; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.clientImpl.Tables; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.PartialKey; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.master.state.tables.TableState; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema; +import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl; +import org.apache.accumulo.core.security.Authorizations; ++import org.apache.accumulo.core.security.NamespacePermission; +import org.apache.accumulo.core.security.TablePermission; +import org.apache.accumulo.core.util.Pair; +import org.apache.accumulo.core.util.TextUtil; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.MapWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.collect.Maps; + +@SuppressWarnings("deprecation") +public class InputConfigurator extends ConfiguratorBase { + + /** + * Configuration keys for {@link Scanner}. + * + * @since 1.6.0 + */ + public static enum ScanOpts { +TABLE_NAME, +AUTHORIZATIONS, +RANGES, +COLUMNS, +ITERATORS, +TABLE_CONFIGS, +SAMPLER_CONFIG, +CLASSLOADER_CONTEXT + } + + /** + * Configuration keys for various features. + * + * @since 1.6.0 + */ + public static enum Features { +AUTO_ADJUST_RANGES, +SCAN_ISOLATION, +USE_LOCAL_ITERATORS, +SCAN_OFFLI
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 3b627a979caf612eea45876e275614311265767b Merge: d3ff3ae af05b27 Author: Christopher Tubbs AuthorDate: Thu Sep 19 19:51:51 2019 -0400 Merge branch '1.9' into 2.0 .../test/performance/ContinuousIngest.java | 25 +- 1 file changed, 15 insertions(+), 10 deletions(-) diff --cc test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java index 3e1d497,000..e46d871 mode 100644,00..100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/ContinuousIngest.java @@@ -1,272 -1,0 +1,277 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.performance; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.BufferedReader; ++import java.io.IOException; +import java.io.InputStreamReader; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import java.util.zip.CRC32; +import java.util.zip.Checksum; + +import org.apache.accumulo.core.cli.ClientOpts; +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.ColumnVisibility; +import org.apache.accumulo.core.trace.TraceUtil; +import org.apache.accumulo.core.util.FastFormat; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.htrace.TraceScope; +import org.apache.htrace.wrappers.TraceProxy; + +import com.beust.jcommander.Parameter; + +public class ContinuousIngest { + + private static final byte[] EMPTY_BYTES = new byte[0]; + + private static List visibilities; + + private static void initVisibilities(ContinuousOpts opts) throws Exception { +if (opts.visFile == null) { + visibilities = Collections.singletonList(new ColumnVisibility()); + return; +} + - visibilities = new ArrayList<>(); - - FileSystem fs = FileSystem.get(new Configuration()); - BufferedReader in = - new BufferedReader(new InputStreamReader(fs.open(new Path(opts.visFile)), UTF_8)); ++visibilities = readVisFromFile(opts.visFile); ++ } + - String line; ++ public static List readVisFromFile(String visFile) { ++List vis = new ArrayList<>(); + - while ((line = in.readLine()) != null) { - visibilities.add(new ColumnVisibility(line)); ++try (BufferedReader in = new BufferedReader(new InputStreamReader( ++FileSystem.get(new Configuration()).open(new Path(visFile)), UTF_8))) { ++ String line; ++ while ((line = in.readLine()) != null) { ++vis.add(new ColumnVisibility(line)); ++ } ++} catch (IOException e) { ++ System.out.println("ERROR reading visFile " + visFile + ": "); ++ e.printStackTrace(); +} - - in.close(); ++return vis; + } + + private static ColumnVisibility getVisibility(Random rand) { +return visibilities.get(rand.nextInt(visibilities.size())); + } + + static class TestOpts extends ClientOpts { +@Parameter(names = "--table", description = "table to use") +String tableName = "ci"; + } + + public static void main(String[] args) throws Exception { + +ContinuousOpts opts = new ContinuousOpts(); +TestOpts clientOpts = new TestOpts(); +try (TraceScope clientSpan = +clientOpts.parseArgsAndTrace(ContinuousIngest.class.getName(), args, opts)) { + + initVisibilities(opts); + + if (opts.min < 0 || opts.max < 0 || opts.ma
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d3ff3aed25db77d3b6231c1b141f893c8c419f2f Merge: ce86a72 92551e4 Author: Mike Miller AuthorDate: Wed Sep 18 17:11:59 2019 -0400 Merge branch '1.9' into 2.0 * git merge 1.9 -sours
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d04109d06a41a7ae190941380ee7afb36fa2b4ef Merge: 1c19584 d077b46 Author: Mike Miller AuthorDate: Tue Sep 3 11:52:56 2019 -0400 Merge branch '1.9' into 2.0 Conflicts resolved: server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java .../main/java/org/apache/accumulo/master/TabletGroupWatcher.java | 7 +++ 1 file changed, 3 insertions(+), 4 deletions(-) diff --cc server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java index db00210,6c98f72..b801e3b --- a/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java +++ b/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java @@@ -102,9 -103,9 +102,8 @@@ abstract class TabletGroupWatcher exten private static final int ASSINGMENT_BUFFER_MAX_LENGTH = 4096; private final Master master; - final TabletStateStore store; - final TabletGroupWatcher dependentWatcher; - + private final TabletStateStore store; + private final TabletGroupWatcher dependentWatcher; - final TableStats stats = new TableStats(); private SortedSet lastScanServers = ImmutableSortedSet.of();
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f4931827bcba4189181473681c6379a744256b59 Merge: 7a83445 1c902ec Author: Mike Miller AuthorDate: Thu Aug 29 13:19:00 2019 -0400 Merge branch '1.9' into 2.0 .../apache/accumulo/shell/commands/ImportDirectoryCommand.java | 9 + 1 file changed, 9 insertions(+)
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 6ae7ddb99d10635e713742f35fc1db0ce9332ba2 Merge: 95c92ed ab90032 Author: Christopher Tubbs AuthorDate: Thu Aug 15 15:45:02 2019 -0400 Merge branch '1.9' into 2.0 .../accumulo/gc/GarbageCollectWriteAheadLogs.java | 10 -- .../org/apache/accumulo/gc/SimpleGarbageCollector.java | 15 ++- .../accumulo/gc/GarbageCollectWriteAheadLogsTest.java | 18 ++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --cc server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java index 289e69d,7afcdc8..e671219 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java @@@ -85,18 -91,26 +85,15 @@@ public class GarbageCollectWriteAheadLo * @param useTrash * true to move files to trash rather than delete them */ - GarbageCollectWriteAheadLogs(final ServerContext context, VolumeManager fs, boolean useTrash) { - GarbageCollectWriteAheadLogs(final AccumuloServerContext context, final VolumeManager fs, - final LiveTServerSet liveServers, boolean useTrash) throws IOException { ++ GarbageCollectWriteAheadLogs(final ServerContext context, final VolumeManager fs, ++ final LiveTServerSet liveServers, boolean useTrash) { this.context = context; this.fs = fs; this.useTrash = useTrash; - this.liveServers = new LiveTServerSet(context, (current, deleted, added) -> { - log.debug("New tablet servers noticed: {}", added); - log.debug("Tablet servers removed: {}", deleted); - }); - liveServers.startListeningForTabletServerChanges(); + this.liveServers = liveServers; - -this.walMarker = new WalStateManager(context.getInstance(), ZooReaderWriter.getInstance()); -this.store = new Iterable() { - @Override - public Iterator iterator() { -try { - return Iterators.concat(new ZooTabletStateStore().iterator(), - new RootTabletStateStore(context).iterator(), - new MetaDataStateStore(context).iterator()); -} catch (DistributedStoreException e) { - throw new RuntimeException(e); -} - } -}; +this.walMarker = new WalStateManager(context); +this.store = () -> Iterators.concat(new ZooTabletStateStore(context).iterator(), +new RootTabletStateStore(context).iterator(), new MetaDataStateStore(context).iterator()); } /** @@@ -124,34 -138,31 +121,35 @@@ } public void collect(GCStatus status) { - -Span span = Trace.start("getCandidates"); try { - status.currentLog.started = System.currentTimeMillis(); - - Map recoveryLogs = getSortedWALogs(); - - Map> logsByServer = new HashMap<>(); - Map> logsState = new HashMap<>(); - // Scan for log file info first: the order is important - // Consider: - // * get live servers - // * new server gets a lock, creates a log - // * get logs - // * the log appears to belong to a dead server - long count = getCurrent(logsByServer, logsState); - long fileScanStop = System.currentTimeMillis(); - - log.info(String.format("Fetched %d files for %d servers in %.2f seconds", count, - logsByServer.size(), (fileScanStop - status.currentLog.started) / 1000.)); - status.currentLog.candidates = count; - span.stop(); + long count; + long fileScanStop; + Map> logsByServer; + Map> logsState; + Map recoveryLogs; + try (TraceScope span = Trace.startSpan("getCandidates")) { +status.currentLog.started = System.currentTimeMillis(); + +recoveryLogs = getSortedWALogs(); + +logsByServer = new HashMap<>(); +logsState = new HashMap<>(); +// Scan for log file info first: the order is important +// Consider: +// * get live servers +// * new server gets a lock, creates a log +// * get logs +// * the log appears to belong to a dead server +count = getCurrent(logsByServer, logsState); +fileScanStop = System.currentTimeMillis(); + +log.info(String.format("Fetched %d files for %d servers in %.2f seconds", count, +logsByServer.size(), (fileScanStop - status.currentLog.started) / 1000.)); +status.currentLog.candidates = count; + } // now it's safe to get the liveServers + liveServers.scanServers(); Set currentServers = liveServers.getCurrentServers(); Map uuidToTServer; diff --cc server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 147239a,24ce61c..2a0a19f --- a/server/gc/src/main/java/org/apache/accumulo/g
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 95c92ed95404ec1663ea97d8b4b00d58c26ba762 Merge: f63da3d 556c29e Author: Keith Turner AuthorDate: Thu Aug 15 14:01:51 2019 -0400 Merge branch '1.9' into 2.0 .../functional/TabletStateChangeIteratorIT.java| 73 -- 1 file changed, 53 insertions(+), 20 deletions(-) diff --cc test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java index b83e735,2fd8589..35c53cc --- a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java @@@ -35,7 -34,10 +36,9 @@@ import org.apache.accumulo.core.client. import org.apache.accumulo.core.client.BatchDeleter; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Connector; + import org.apache.accumulo.core.client.IsolatedScanner; import org.apache.accumulo.core.client.MutationsRejectedException; + import org.apache.accumulo.core.client.RowIterator; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; @@@ -81,70 -81,70 +84,75 @@@ public class TabletStateChangeIteratorI @Test public void test() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException { -String[] tables = getUniqueNames(6); -final String t1 = tables[0]; -final String t2 = tables[1]; -final String t3 = tables[2]; -final String metaCopy1 = tables[3]; -final String metaCopy2 = tables[4]; -final String metaCopy3 = tables[5]; - -// create some metadata -createTable(t1, true); -createTable(t2, false); -createTable(t3, true); - -// examine a clone of the metadata table, so we can manipulate it -copyTable(MetadataTable.NAME, metaCopy1); - -State state = new State(); -while (findTabletsNeedingAttention(metaCopy1, state) > 0) { - UtilWaitThread.sleep(500); - copyTable(MetadataTable.NAME, metaCopy1); -} -assertEquals("No tables should need attention", 0, -findTabletsNeedingAttention(metaCopy1, state)); - -// The metadata table stabilized and metaCopy1 contains a copy suitable for testing. Before -// metaCopy1 is modified, copy it for subsequent test. -copyTable(metaCopy1, metaCopy2); -copyTable(metaCopy1, metaCopy3); - -// test the assigned case (no location) -removeLocation(metaCopy1, t3); -assertEquals("Should have two tablets without a loc", 2, -findTabletsNeedingAttention(metaCopy1, state)); - -// test the cases where the assignment is to a dead tserver -reassignLocation(metaCopy2, t3); -assertEquals("Should have one tablet that needs to be unassigned", 1, -findTabletsNeedingAttention(metaCopy2, state)); - -// test the cases where there is ongoing merges -state = new State() { - @Override - public Collection merges() { -String tableIdToModify = getConnector().tableOperations().tableIdMap().get(t3); -return Collections.singletonList( -new MergeInfo(new KeyExtent(tableIdToModify, null, null), MergeInfo.Operation.MERGE)); + +try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { + - String[] tables = getUniqueNames(4); ++ String[] tables = getUniqueNames(6); + final String t1 = tables[0]; + final String t2 = tables[1]; + final String t3 = tables[2]; - final String cloned = tables[3]; ++ final String metaCopy1 = tables[3]; ++ final String metaCopy2 = tables[4]; ++ final String metaCopy3 = tables[5]; + + // create some metadata + createTable(client, t1, true); + createTable(client, t2, false); + createTable(client, t3, true); + + // examine a clone of the metadata table, so we can manipulate it - cloneMetadataTable(client, cloned); ++ copyTable(client, MetadataTable.NAME, metaCopy1); + + State state = new State(client); - while (findTabletsNeedingAttention(client, cloned, state) > 0) { ++ while (findTabletsNeedingAttention(client, metaCopy1, state) > 0) { +UtilWaitThread.sleep(500); ++copyTable(client, MetadataTable.NAME, metaCopy1); } -}; -assertEquals("Should have 2 tablets that need to be chopped or unassigned", 1, -findTabletsNeedingAttention(metaCopy2, state)); - -// test the bad tablet location state case (inconsistent metadata) -state = new State(); -addDuplicateLocation(metaCopy3, t3); -assertEquals("Should have 1 tablet that needs
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f63da3d4ce709402feb2169eeda4c9c290ad5b7b Merge: c573676 e770862 Author: Mike Miller AuthorDate: Thu Aug 15 10:08:31 2019 -0400 Merge branch '1.9' into 2.0
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit c5736768d8ff3cb8799b5e254d7df36f1850a648 Merge: 5c58b08 54b2c32 Author: Keith Turner AuthorDate: Wed Aug 14 16:55:06 2019 -0400 Merge branch '1.9' into 2.0 .../accumulo/gc/GarbageCollectWriteAheadLogs.java | 5 +++-- .../apache/accumulo/master/FateServiceHandler.java | 3 ++- .../accumulo/master/util/TableValidators.java | 22 ++ .../accumulo/test/functional/CloneTestIT.java | 16 4 files changed, 43 insertions(+), 3 deletions(-) diff --cc server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java index 5bb8201,23c4932..289e69d --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java @@@ -53,6 -60,8 +53,7 @@@ import org.apache.accumulo.server.maste import org.apache.accumulo.server.master.state.TServerInstance; import org.apache.accumulo.server.master.state.TabletLocationState; import org.apache.accumulo.server.master.state.TabletState; + import org.apache.accumulo.server.master.state.ZooTabletStateStore; -import org.apache.accumulo.server.zookeeper.ZooReaderWriter; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@@ -88,14 -97,28 +89,14 @@@ public class GarbageCollectWriteAheadLo this.context = context; this.fs = fs; this.useTrash = useTrash; -this.liveServers = new LiveTServerSet(context, new Listener() { - @Override - public void update(LiveTServerSet current, Set deleted, - Set added) { -log.debug("New tablet servers noticed: " + added); -log.debug("Tablet servers removed: " + deleted); - } +this.liveServers = new LiveTServerSet(context, (current, deleted, added) -> { + log.debug("New tablet servers noticed: {}", added); + log.debug("Tablet servers removed: {}", deleted); }); liveServers.startListeningForTabletServerChanges(); -this.walMarker = new WalStateManager(context.getInstance(), ZooReaderWriter.getInstance()); -this.store = new Iterable() { - @Override - public Iterator iterator() { -try { - return Iterators.concat(new ZooTabletStateStore().iterator(), - new RootTabletStateStore(context).iterator(), - new MetaDataStateStore(context).iterator()); -} catch (DistributedStoreException e) { - throw new RuntimeException(e); -} - } -}; +this.walMarker = new WalStateManager(context); - this.store = () -> Iterators.concat(new RootTabletStateStore(context).iterator(), - new MetaDataStateStore(context).iterator()); ++this.store = () -> Iterators.concat(new ZooTabletStateStore(context).iterator(), ++new RootTabletStateStore(context).iterator(), new MetaDataStateStore(context).iterator()); } /** diff --cc server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java index c19ee7b,7949ac2..0cf7c14 --- a/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java +++ b/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java @@@ -250,12 -225,12 +251,12 @@@ class FateServiceHandler implements Fat case TABLE_CLONE: { TableOperation tableOp = TableOperation.CLONE; validateArgumentCount(arguments, tableOp, 2); - TableId srcTableId = validateTableIdArgument(arguments.get(0), tableOp, NOT_ROOT_ID); -String srcTableId = validateTableIdArgument(arguments.get(0), tableOp, CAN_CLONE); ++TableId srcTableId = validateTableIdArgument(arguments.get(0), tableOp, CAN_CLONE); String tableName = validateTableNameArgument(arguments.get(1), tableOp, NOT_SYSTEM); -String namespaceId; +NamespaceId namespaceId; try { namespaceId = - Namespaces.getNamespaceId(master.getInstance(), Tables.qualify(tableName).getFirst()); + Namespaces.getNamespaceId(master.getContext(), Tables.qualify(tableName).getFirst()); } catch (NamespaceNotFoundException e) { // shouldn't happen, but possible once cloning between namespaces is supported throw new ThriftTableOperationException(null, tableName, tableOp, diff --cc server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java index ddbce98,a9d3edb..4d44765 --- a/server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java +++ b/server/master/src/main/java/org/apache/accumulo/master/util/TableValidators.java @@@ -80,6 -79,28 +80,28 @@@ public class TableValidators } }; - public static final Validator CAN_CLONE = new Validator(
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f49f6270e9c514c58bc03fe595dd283167f2ce55 Merge: aa1b886 d6c97fd Author: Mike Miller AuthorDate: Fri Aug 2 15:35:02 2019 -0400 Merge branch '1.9' into 2.0 core/src/main/java/org/apache/accumulo/core/conf/Experimental.java | 1 + 1 file changed, 1 insertion(+)
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit fc0f4723fe9502ff7b49b7d27c96abe398e66d71 Merge: 5a6481f 1704d79 Author: Christopher Tubbs AuthorDate: Mon Jul 29 20:05:34 2019 -0400 Merge branch '1.9' into 2.0 pom.xml | 51 +-- test/pom.xml | 5 - 2 files changed, 5 insertions(+), 51 deletions(-) diff --cc pom.xml index bc9d69b,bc548a8..0988bba --- a/pom.xml +++ b/pom.xml @@@ -122,18 -127,16 +122,19 @@@ 1 - 3.1.1 -2.6.5 ++3.1.2 +2.5.0 4.1.0-incubating -3.1.0-incubating +3.2.0-incubating false - -9.2.26.v20180806 -7 -1.7 -1.7 -3.0.5 +2.9.9 +3.0.1-b06 +2.3.0.1 +2.28 +9.4.19.v20190610 ++8 +1.8 +1.8 true 2.0.2 @@@ -1121,24 -841,22 +,6 @@@ -- - enforce-java-signatures -enforce-hadoop-profile -- -- enforce -- - process-test-classes -- -- - - - org.codehaus.mojo.signature - java18 - 1.0 - - - - hadoop.profile - (2|3) - You should specify the Hadoop profile by major Hadoop generation, i.e. 2 or 3, not by a version number. -Use hadoop.version to use a particular Hadoop version within that generation. - -- -- -- @@@ -1579,68 -1361,44 +1547,59 @@@ src/main/spotbugs/exclude-filter.xml - - hadoop-default - - - !hadoop.profile - - - -2 -2.6.4 - - - - - hadoop2 - - - hadoop.profile - 2 - - - -2.6.4 - + + use-errorprone + + + +org.apache.maven.plugins +maven-compiler-plugin + + javac-with-errorprone + true + +-XepDisableAllChecks +-Xep:MissingOverride:ERROR + -XepExcludedPaths:.*/(proto|thrift|generated-sources)/.* + + + + +com.google.errorprone +error_prone_core +2.3.1 + + +org.codehaus.plexus +plexus-compiler-javac-errorprone +2.8 + + + + + - - hadoop3 - - - hadoop.profile - 3 - - - -3.0.0 - + sec-bugs + + + +com.github.spotbugs +spotbugs-maven-plugin + + + + com.h3xstream.findsecbugs + findsecbugs-plugin + 1.8.0 + + + + + + - - jdk-release-flag - - [9,) - - - 8 - -
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 5a6481f7c15253bfc19a0ded5a96cb02ce9edb7e Merge: 0bb3f89 fd4d478 Author: Christopher Tubbs AuthorDate: Mon Jul 29 15:08:16 2019 -0400 Merge branch '1.9' into 2.0 pom.xml| 4 test/src/main/java/org/apache/accumulo/test/ShellServerIT.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --cc pom.xml index c08d56c,ff3f20b..bc9d69b --- a/pom.xml +++ b/pom.xml @@@ -1353,6 -1100,10 +1353,10 @@@ m2e.version + - -7 ++ ++8 +
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0bb3f896a28ec315976fb4570ced8e3913a75587 Merge: 129e8ec 6546fc3 Author: Keith Turner AuthorDate: Fri Jul 26 15:20:31 2019 -0400 Merge branch '1.9' into 2.0 .../apache/accumulo/tserver/BusiestTracker.java| 8 -- .../accumulo/tserver/BusiestTrackerTest.java | 32 -- 2 files changed, 29 insertions(+), 11 deletions(-) diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java index 1eaa1d6,229ed74..cf19b5a --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java @@@ -60,9 -60,16 +60,13 @@@ public abstract class BusiestTracker counts.put(extent, count); - long delta = count - lastCounts.getOrDefault(extent, 0L); - Long lastCount = lastCounts.get(extent); ++ long lastCount = lastCounts.getOrDefault(extent, 0L); - // handle case where tablet leaves tserver and returns OR tablet had no activity - // if a tablet leaves a tserver and come back, then its count will be reset. This could make ++ // if a tablet leaves a tserver and comes back, then its count will be reset. This could make + // lastCount higher than the current count. That is why lastCount > count is checked below. - if (lastCount == null || lastCount > count) -lastCount = 0L; - - long delta = count - lastCount; ++ long delta = (lastCount > count) ? count : count - lastCount; + + // handle case where tablet had no activity if (delta > 0) tabletsWithDelta.add(new ComparablePair<>(delta, extent)); }
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 129e8ec4abf5f94d7163dcf761df3e6687a49f59 Merge: 3a4a4d6 87da062 Author: Christopher Tubbs AuthorDate: Thu Jul 25 17:52:38 2019 -0400 Merge branch '1.9' into 2.0 pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --cc pom.xml index 97cc195,78dbdd3..c08d56c --- a/pom.xml +++ b/pom.xml @@@ -785,7 -567,7 +785,7 @@@ com.github.spotbugs spotbugs-maven-plugin - ${spotbugs.version} - 3.1.12.1 ++ ${spotbugs.version}.1 true Max
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 40d954f22b22ff437b938c6bf3607298b30e1088 Merge: d6dfee4 83429b6 Author: Keith Turner AuthorDate: Thu Jul 25 14:36:59 2019 -0400 Merge branch '1.9' into 2.0 .../org/apache/accumulo/core/conf/Property.java| 5 + .../apache/accumulo/tserver/BusiestTracker.java| 92 +++ .../org/apache/accumulo/tserver/TabletServer.java | 35 + .../org/apache/accumulo/tserver/tablet/Tablet.java | 4 + .../accumulo/tserver/BusiestTrackerTest.java | 168 + 5 files changed, 304 insertions(+) diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java index 7879ae4,494f6b2..302ce06 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@@ -444,8 -483,13 +444,13 @@@ public enum Property "The minimum number of threads to use to handle incoming requests."), TSERV_THREADCHECK("tserver.server.threadcheck.time", "1s", PropertyType.TIMEDURATION, "The time between adjustments of the server thread pool."), - TSERV_MAX_MESSAGE_SIZE("tserver.server.message.size.max", "1G", PropertyType.MEMORY, + TSERV_MAX_MESSAGE_SIZE("tserver.server.message.size.max", "1G", PropertyType.BYTES, "The maximum size of a message that can be sent to a tablet server."), + TSERV_LOG_BUSY_TABLETS_COUNT("tserver.log.busy.tablets.count", "0", PropertyType.COUNT, + "Number of busiest tablets to log. Logged at interval controlled by " + + "tserver.log.busy.tablets.interval. If <= 0, logging of busy tablets is disabled"), + TSERV_LOG_BUSY_TABLETS_INTERVAL("tserver.log.busy.tablets.interval", "1h", + PropertyType.TIMEDURATION, "Time interval between logging out busy tablets information."), TSERV_HOLD_TIME_SUICIDE("tserver.hold.time.max", "5m", PropertyType.TIMEDURATION, "The maximum time for a tablet server to be in the \"memory full\" state." + " If the tablet server cannot write out memory in this much time, it will" diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java index 000,ea60cad..1eaa1d6 mode 00,100644..100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BusiestTracker.java @@@ -1,0 -1,96 +1,92 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.accumulo.tserver; + + import java.util.ArrayList; + import java.util.Collection; + import java.util.Collections; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + -import org.apache.accumulo.core.data.impl.KeyExtent; ++import org.apache.accumulo.core.dataImpl.KeyExtent; + import org.apache.accumulo.core.util.ComparablePair; + import org.apache.accumulo.tserver.tablet.Tablet; + + import com.google.common.collect.Ordering; + + /** + * Computes the N tablets that have the highest deltas for a given monotonically increasing counter. + */ + public abstract class BusiestTracker { + + private Map lastCounts = Collections.emptyMap(); + private final int numBusiestTabletsToLog; + + BusiestTracker(int numBusiestTabletsToLog) { + this.numBusiestTabletsToLog = numBusiestTabletsToLog; + } + + protected abstract long extractCount(Tablet tablet); + + public List> computeBusiest(Collection tablets) { + + HashMap counts = new HashMap<>(); + + ArrayList> tabletsWithDelta = new ArrayList<>(); + + for (Tablet tablet : tablets) { + KeyExtent extent = tablet.getExtent(); + + // only get the count once to ensure consistency in the case of multiple threads + long count = extractCount(tablet); + + if (count == 0) + continue; + + counts.put(extent, count); + - Long lastCount = lastCounts.get(extent); - if (lastCount == null) -lastCount = 0L; - - long delta = count - lastCount; ++ long delta = count - lastCounts.getOrDefault(extent, 0L); + +
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 4724f12c390cb6c0e3a0d44e5f830d3cd029fdc8 Merge: b32d81d 6b1ff04 Author: Christopher Tubbs AuthorDate: Tue Jul 16 17:18:47 2019 -0400 Merge branch '1.9' into 2.0 .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --cc .travis.yml index c883b19,b67e92a..b41a9d2 --- a/.travis.yml +++ b/.travis.yml @@@ -27,7 -27,7 +27,8 @@@ before_script - unset _JAVA_OPTIONS env: - BUILD_CMD="mvn clean verify javadoc:jar -DskipITs" # main build of unit tests and javadoc - - BUILD_CMD="mvn clean verify javadoc:jar -DskipITs -Dhadoop.profile=3" + - BUILD_CMD="travis_wait 30 mvn clean verify -PskipQA,sec-bugs -Dspotbugs.skip=false -Dspotbugs.timeout=180" # check for spotbugs+security + - BUILD_CMD="mvn clean package -DskipTests -Dhadoop.version=3.0.3" # quick compile to verify older API + # suppress download logs which cause travis to exceed the 4MB build log output limit script: - - $BUILD_CMD + - $BUILD_CMD -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit e0e9c471fc5beaba71b1b826675ab867ba47418b Merge: 4160067 d7fccfd Author: Christopher Tubbs AuthorDate: Fri Jun 21 04:20:36 2019 -0400 Merge branch '1.9' into 2.0 .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --cc .travis.yml index c96e059,1b72b5b..c883b19 --- a/.travis.yml +++ b/.travis.yml @@@ -22,12 -22,11 +22,12 @@@ cache - $HOME/.m2 install: echo NOOP Skipping pre-fetch of Maven dependencies jdk: -- - openjdk8 ++ - openjdk11 before_script: - unset _JAVA_OPTIONS env: - BUILD_CMD="mvn clean verify javadoc:jar -DskipITs" # main build of unit tests and javadoc - - BUILD_CMD="mvn clean verify javadoc:jar -DskipITs -Dhadoop.profile=3" + - BUILD_CMD="travis_wait 30 mvn clean verify -PskipQA,sec-bugs -Dspotbugs.skip=false -Dspotbugs.timeout=180" # check for spotbugs+security - - BUILD_CMD="mvn clean compile -Dhadoop.version=3.0.3" # quick compile to verify older API ++ - BUILD_CMD="mvn clean package -DskipTests -Dhadoop.version=3.0.3" # quick compile to verify older API script: - $BUILD_CMD
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit ad64724fd72c26c22dad232eddd7de2f8ca96df8 Merge: 9e32263 89829d7 Author: Christopher Tubbs AuthorDate: Thu Jun 20 16:18:55 2019 -0400 Merge branch '1.9' into 2.0 pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit c95609c513a41f54a145e9b2cc06ce4435dd7dbd Merge: 68f8f5e 0884bfd Author: Christopher Tubbs AuthorDate: Sun Jun 9 22:08:19 2019 -0400 Merge branch '1.9' into 2.0 assemble/pom.xml | 2 +- core/pom.xml | 2 +- hadoop-mapreduce/pom.xml | 2 +- iterator-test-harness/pom.xml | 2 +- minicluster/pom.xml | 2 +- pom.xml | 2 +- server/base/pom.xml | 2 +- server/gc/pom.xml | 2 +- server/master/pom.xml | 2 +- server/monitor/pom.xml| 2 +- server/native/pom.xml | 2 +- server/tracer/pom.xml | 2 +- server/tserver/pom.xml| 2 +- start/pom.xml | 2 +- test/pom.xml | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --cc hadoop-mapreduce/pom.xml index 36c8bd3,000..f940761 mode 100644,00..100644 --- a/hadoop-mapreduce/pom.xml +++ b/hadoop-mapreduce/pom.xml @@@ -1,120 -1,0 +1,120 @@@ + + - http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/maven-v4_0_0.xsd";> ++http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + 4.0.0 + +org.apache.accumulo +accumulo-project +2.0.0-SNAPSHOT + + accumulo-hadoop-mapreduce + Apache Accumulo Hadoop MapReduce + Apache Accumulo MapReduce bindings. + + + com.beust + jcommander + + + com.google.guava + guava + + + org.apache.accumulo + accumulo-core + + + org.apache.hadoop + hadoop-client-api + + + org.slf4j + slf4j-api + + + org.apache.hadoop + hadoop-client-runtime + runtime + + + junit + junit + test + + + org.apache.accumulo + accumulo-minicluster + test + + + org.apache.accumulo + accumulo-server-base + test + + + org.apache.accumulo + accumulo-test + test + + + org.slf4j + slf4j-log4j12 + test + + + + + +net.revelc.code +apilyzer-maven-plugin + + +apilyzer + + analyze + + + + org[.]apache[.]accumulo[.]core[.]client[.]mapred(?:uce)?[.].* + + +.*[.]impl[.].* + + + org[.]apache[.]accumulo[.]core[.](?:client|data|security)[.](?!.*(impl|thrift|crypto).*).* + +org[.]apache[.]accumulo[.]core[.]util[.]Pair + +org[.]apache[.]hadoop[.]conf[.]Configuration +org[.]apache[.]hadoop[.]fs[.](FileSystem|Path) + org[.]apache[.]hadoop[.]io[.](Text|Writable|WritableComparable|WritableComparator) + org[.]apache[.]hadoop[.]mapred[.](JobConf|RecordReader|InputSplit|RecordWriter|Reporter) + org[.]apache[.]hadoop[.]mapred[.]FileOutputFormat[$]Counter + org[.]apache[.]hadoop[.]mapreduce[.](Job|JobContext|RecordReader|InputSplit|TaskAttemptContext|RecordWriter|OutputCommitter|TaskInputOutputContext) + org[.]apache[.]hadoop[.]mapreduce[.]lib[.]output[.]FileOutputFormat[$]Counter +org[.]apache[.]hadoop[.]util[.]Progressable + org[.]apache[.]hadoop[.]mapred[.](FileAlreadyExistsException|InvalidJobConfException) + + + + + + + +
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 3bdb6cd94d0fbdd88bff6da4aeb682bd4391c847 Merge: c8776a9 3311218 Author: Christopher Tubbs AuthorDate: Fri Jun 7 12:41:25 2019 -0400 Merge branch '1.9' into 2.0 core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReader.java | 4 +--- core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooUtil.java | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReader.java index 98e49d3,000..0bddb07 mode 100644,00..100644 --- a/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReader.java +++ b/core/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReader.java @@@ -1,263 -1,0 +1,261 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.fate.zookeeper; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.accumulo.fate.util.Retry; +import org.apache.accumulo.fate.util.Retry.RetryFactory; +import org.apache.accumulo.fate.zookeeper.ZooUtil.ZooKeeperConnectionInfo; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.Code; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZooReader implements IZooReader { + private static final Logger log = LoggerFactory.getLogger(ZooReader.class); + + protected String keepers; + protected int timeout; + private final RetryFactory retryFactory; - private final ZooKeeperConnectionInfo info; + + protected ZooKeeper getSession(String keepers, int timeout, String scheme, byte[] auth) { +return ZooSession.getSession(keepers, timeout, scheme, auth); + } + + protected ZooKeeper getZooKeeper() { +return getSession(keepers, timeout, null, null); + } + + protected RetryFactory getRetryFactory() { +return retryFactory; + } + + protected void retryOrThrow(Retry retry, KeeperException e) throws KeeperException { +log.warn("Saw (possibly) transient exception communicating with ZooKeeper", e); +if (retry.canRetry()) { + retry.useRetry(); + return; +} + +log.error("Retry attempts ({}) exceeded trying to communicate with ZooKeeper", +retry.retriesCompleted()); +throw e; + } + + @Override + public byte[] getData(String zPath, Stat stat) throws KeeperException, InterruptedException { +return getData(zPath, false, stat); + } + + @Override + public byte[] getData(String zPath, boolean watch, Stat stat) + throws KeeperException, InterruptedException { +final Retry retry = getRetryFactory().createRetry(); +while (true) { + try { +return getZooKeeper().getData(zPath, watch, stat); + } catch (KeeperException e) { +final Code code = e.code(); +if (code == Code.CONNECTIONLOSS || code == Code.OPERATIONTIMEOUT +|| code == Code.SESSIONEXPIRED) { + retryOrThrow(retry, e); +} else { + throw e; +} + } + + retry.waitForNextAttempt(); +} + } + + @Override + public byte[] getData(String zPath, Watcher watcher, Stat stat) + throws KeeperException, InterruptedException { +final Retry retry = getRetryFactory().createRetry(); +while (true) { + try { +return getZooKeeper().getData(zPath, watcher, stat); + } catch (KeeperException e) { +final Code code = e.code(); +if (code == Code.CONNECTIONLOSS || code == Code.OPERATIONTIMEOUT +|| code == Code.SESSIONEXPIRED) { + retryOrThrow(retry, e); +} else { + throw e; +} + } + + retry.waitForNextAttempt(); +} + } + + @Override + public Stat getStatus(String zPath) throws KeeperException, InterruptedException { +f
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 21c6a0e489225f3206be19c92fe0c1904cb7825c Merge: 3b010ca 057f14e Author: Keith Turner AuthorDate: Tue May 28 15:17:51 2019 -0400 Merge branch '1.9' into 2.0
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. elserj pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 03986a86a22bc1ff9754768e6fdb456f868901fa Merge: 0ac42c8 1d5ce3d Author: Josh Elser AuthorDate: Fri May 24 13:14:34 2019 -0400 Merge branch '1.9' assemble/pom.xml | 2 +- core/pom.xml | 2 +- hadoop-mapreduce/pom.xml | 2 +- iterator-test-harness/pom.xml | 2 +- minicluster/pom.xml | 2 +- pom.xml | 2 +- proxy/pom.xml | 2 +- server/base/pom.xml | 2 +- server/gc/pom.xml | 2 +- server/master/pom.xml | 2 +- server/monitor/pom.xml| 2 +- server/native/pom.xml | 2 +- server/tracer/pom.xml | 2 +- server/tserver/pom.xml| 2 +- shell/pom.xml | 2 +- start/pom.xml | 2 +- test/pom.xml | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --cc hadoop-mapreduce/pom.xml index f848058,000..36c8bd3 mode 100644,00..100644 --- a/hadoop-mapreduce/pom.xml +++ b/hadoop-mapreduce/pom.xml @@@ -1,120 -1,0 +1,120 @@@ + + - http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> ++http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/maven-v4_0_0.xsd";> + 4.0.0 + +org.apache.accumulo +accumulo-project +2.0.0-SNAPSHOT + + accumulo-hadoop-mapreduce + Apache Accumulo Hadoop MapReduce + Apache Accumulo MapReduce bindings. + + + com.beust + jcommander + + + com.google.guava + guava + + + org.apache.accumulo + accumulo-core + + + org.apache.hadoop + hadoop-client-api + + + org.slf4j + slf4j-api + + + org.apache.hadoop + hadoop-client-runtime + runtime + + + junit + junit + test + + + org.apache.accumulo + accumulo-minicluster + test + + + org.apache.accumulo + accumulo-server-base + test + + + org.apache.accumulo + accumulo-test + test + + + org.slf4j + slf4j-log4j12 + test + + + + + +net.revelc.code +apilyzer-maven-plugin + + +apilyzer + + analyze + + + + org[.]apache[.]accumulo[.]core[.]client[.]mapred(?:uce)?[.].* + + +.*[.]impl[.].* + + + org[.]apache[.]accumulo[.]core[.](?:client|data|security)[.](?!.*(impl|thrift|crypto).*).* + +org[.]apache[.]accumulo[.]core[.]util[.]Pair + +org[.]apache[.]hadoop[.]conf[.]Configuration +org[.]apache[.]hadoop[.]fs[.](FileSystem|Path) + org[.]apache[.]hadoop[.]io[.](Text|Writable|WritableComparable|WritableComparator) + org[.]apache[.]hadoop[.]mapred[.](JobConf|RecordReader|InputSplit|RecordWriter|Reporter) + org[.]apache[.]hadoop[.]mapred[.]FileOutputFormat[$]Counter + org[.]apache[.]hadoop[.]mapreduce[.](Job|JobContext|RecordReader|InputSplit|TaskAttemptContext|RecordWriter|OutputCommitter|TaskInputOutputContext) + org[.]apache[.]hadoop[.]mapreduce[.]lib[.]output[.]FileOutputFormat[$]Counter +org[.]apache[.]hadoop[.]util[.]Progressable + org[.]apache[.]hadoop[.]mapred[.](FileAlreadyExistsException|InvalidJobConfException) + + + + + + + +
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 1a65e6dddc3a560a6e3362b3cd639875ca63b5c4 Merge: 8511c43 c56eada Author: Keith Turner AuthorDate: Mon May 6 18:39:38 2019 -0400 Merge branch '1.9' .../apache/accumulo/master/FateServiceHandler.java | 32 +- 1 file changed, 31 insertions(+), 1 deletion(-) diff --cc server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java index 3ec1320,c9b20cf..c19ee7b --- a/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java +++ b/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java @@@ -109,9 -103,10 +109,10 @@@ class FateServiceHandler implements Fat switch (op) { case NAMESPACE_CREATE: { TableOperation tableOp = TableOperation.CREATE; + validateArgumentCount(arguments, tableOp, 1); String namespace = validateNamespaceArgument(arguments.get(0), tableOp, null); -if (!master.security.canCreateNamespace(c, namespace)) +if (!master.security.canCreateNamespace(c)) throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED); master.fate.seedTransaction(opid, @@@ -150,25 -147,11 +153,31 @@@ } case TABLE_CREATE: { TableOperation tableOp = TableOperation.CREATE; -validateArgumentCount(arguments, tableOp, 2); ++int SPLIT_OFFSET = 4; // offset where split data begins in arguments list ++if (arguments.size() < SPLIT_OFFSET) { ++ throw new ThriftTableOperationException(null, null, tableOp, ++ TableOperationExceptionType.OTHER, ++ "Expected at least " + SPLIT_OFFSET + " arguments, saw :" + arguments.size()); ++} String tableName = validateTableNameArgument(arguments.get(0), tableOp, NOT_SYSTEM); TimeType timeType = TimeType.valueOf(ByteBufferUtil.toString(arguments.get(1))); - -String namespaceId; +InitialTableState initialTableState = + InitialTableState.valueOf(ByteBufferUtil.toString(arguments.get(2))); +int splitCount = Integer.parseInt(ByteBufferUtil.toString(arguments.get(3))); ++validateArgumentCount(arguments, tableOp, SPLIT_OFFSET + splitCount); +String splitFile = null; +String splitDirsFile = null; +if (splitCount > 0) { - int SPLIT_OFFSET = 4; // offset where split data begins in arguments list + try { +splitFile = writeSplitsToFile(opid, arguments, splitCount, SPLIT_OFFSET); +splitDirsFile = createSplitDirsFile(opid); + } catch (IOException e) { +throw new ThriftTableOperationException(null, tableName, tableOp, +TableOperationExceptionType.OTHER, +"Exception thrown while writing splits to file system"); + } +} +NamespaceId namespaceId; try { namespaceId = @@@ -239,12 -223,13 +249,13 @@@ } case TABLE_CLONE: { TableOperation tableOp = TableOperation.CLONE; + validateArgumentCount(arguments, tableOp, 2); -String srcTableId = validateTableIdArgument(arguments.get(0), tableOp, NOT_ROOT_ID); +TableId srcTableId = validateTableIdArgument(arguments.get(0), tableOp, NOT_ROOT_ID); String tableName = validateTableNameArgument(arguments.get(1), tableOp, NOT_SYSTEM); -String namespaceId; +NamespaceId namespaceId; try { namespaceId = - Namespaces.getNamespaceId(master.getInstance(), Tables.qualify(tableName).getFirst()); + Namespaces.getNamespaceId(master.getContext(), Tables.qualify(tableName).getFirst()); } catch (NamespaceNotFoundException e) { // shouldn't happen, but possible once cloning between namespaces is supported throw new ThriftTableOperationException(null, tableName, tableOp, @@@ -290,11 -275,12 +301,12 @@@ } case TABLE_DELETE: { TableOperation tableOp = TableOperation.DELETE; + validateArgumentCount(arguments, tableOp, 1); String tableName = validateTableNameArgument(arguments.get(0), tableOp, NOT_SYSTEM); -final String tableId = -ClientServiceHandler.checkTableId(master.getInstance(), tableName, tableOp); -String namespaceId = getNamespaceIdFromTableId(tableOp, tableId); +final TableId tableId = +ClientServiceHandler.checkTableId(master.getContext(), tableName, tableOp); +NamespaceId namespaceId = getNamespaceIdFromTableId(tableOp, tableId); final boolean canDeleteTable; try { @@@ -312,8 -298,9 +324,9 @@@ } case TABLE_ONLINE: { TableOperation tableOp =
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d4eb55daf5407679f485b9abef207fd2c76e7d73 Merge: c6e6c7a a40b9dc Author: Christopher Tubbs AuthorDate: Wed Apr 17 08:17:09 2019 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 162c7589ce2a5a4539def3e39e3af0014ea6fe4a Merge: 7b69ce6 22b00db Author: Christopher Tubbs AuthorDate: Thu Apr 11 21:42:38 2019 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 7b69ce6bc99ceef1ec2fae973229237f57876baf Merge: f1463bf be847d3 Author: Christopher Tubbs AuthorDate: Wed Apr 10 19:16:56 2019 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 548f3965134588ee6d53c5e5b18deee548329977 Merge: 9487e20 bded593 Author: Christopher Tubbs AuthorDate: Fri Apr 5 21:39:56 2019 -0400 Merge branch '1.9' assemble/build.sh | 13 - 1 file changed, 8 insertions(+), 5 deletions(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 9487e20381be54ccbe09632f0c9208070b3161f4 Merge: 503f351 b52c165 Author: Christopher Tubbs AuthorDate: Fri Apr 5 21:07:46 2019 -0400 Merge branch '1.9' .../test/functional/FateConcurrencyIT.java | 198 ++--- 1 file changed, 130 insertions(+), 68 deletions(-) diff --cc test/src/main/java/org/apache/accumulo/test/functional/FateConcurrencyIT.java index df8399f,dc1c225..3bcaa4e --- a/test/src/main/java/org/apache/accumulo/test/functional/FateConcurrencyIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/FateConcurrencyIT.java @@@ -203,70 -197,109 +206,109 @@@ public class FateConcurrencyIT extends * Validate the the AdminUtil.getStatus works correctly after refactor and validate that * getTransactionStatus can be called without lock map(s). The test starts a long running fate * transaction (slow compaction) and the calls AdminUtil functions to get the FATE. -* -* @throws Exception -* any exception is a test failure */ @Test - public void getFateStatus() throws Exception { + public void getFateStatus() { - assertEquals("verify table online after created", TableState.ONLINE, getTableState(tableName)); -Instance instance = connector.getInstance(); -String tableId; ++TableId tableId; + + try { + + assertEquals("verify table online after created", TableState.ONLINE, + getTableState(tableName)); + - tableId = Tables.getTableId(instance, tableName); ++ tableId = Tables.getTableId(context, tableName); + + log.trace("tid: {}", tableId); + + } catch (TableNotFoundException ex) { + throw new IllegalStateException( + String.format("Table %s does not exist, failing test", tableName)); + } Future compactTask = startCompactTask(); assertTrue("compaction fate transaction exits", findFate(tableName)); + AdminUtil.FateStatus withLocks = null; + List noLocks = null; + + int maxRetries = 3; + AdminUtil admin = new AdminUtil<>(false); - try { + while (maxRetries > 0) { - TableId tableId = Tables.getTableId(context, tableName); + try { - log.trace("tid: {}", tableId); -IZooReaderWriter zk = new ZooReaderWriterFactory().getZooReaderWriter( -instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut(), secret); - -ZooStore zs = new ZooStore<>(ZooUtil.getRoot(instance) + Constants.ZFATE, zk); ++String instanceId = accumuloClient.instanceOperations().getInstanceID(); ++ClientInfo info = ClientInfo.from(accumuloClient.properties()); ++IZooReaderWriter zk = new ZooReaderWriterFactory().getZooReaderWriter(info.getZooKeepers(), ++info.getZooKeepersSessionTimeOut(), secret); ++ZooStore zs = new ZooStore<>(ZooUtil.getRoot(instanceId) + Constants.ZFATE, zk); - String instanceId = accumuloClient.instanceOperations().getInstanceID(); - ClientInfo info = ClientInfo.from(accumuloClient.properties()); - IZooReaderWriter zk = new ZooReaderWriterFactory().getZooReaderWriter(info.getZooKeepers(), - info.getZooKeepersSessionTimeOut(), secret); - ZooStore zs = new ZooStore<>(ZooUtil.getRoot(instanceId) + Constants.ZFATE, zk); + withLocks = admin.getStatus(zs, zk, -ZooUtil.getRoot(instance) + Constants.ZTABLE_LOCKS + "/" + tableId, null, null); ++ZooUtil.getRoot(instanceId) + Constants.ZTABLE_LOCKS + "/" + tableId, null, null); - AdminUtil.FateStatus withLocks = admin.getStatus(zs, zk, - ZooUtil.getRoot(instanceId) + Constants.ZTABLE_LOCKS + "/" + tableId, null, null); + // call method that does not use locks. + noLocks = admin.getTransactionStatus(zs, null, null); - // call method that does not use locks. - List noLocks = admin.getTransactionStatus(zs, null, null); + // no zk exception, no need to retry + break; - // fast check - count number of transactions - assertEquals(withLocks.getTransactions().size(), noLocks.size()); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + fail("Interrupt received - test failed"); + return; + } catch (KeeperException ex) { + maxRetries--; + try { + Thread.sleep(1000); + } catch (InterruptedException intr_ex) { + Thread.currentThread().interrupt(); + return; + } + } + } - int matchCount = 0; + assertNotNull(withLocks); + assertNotNull(noLocks); - for (AdminUtil.TransactionStatus tx : withLocks.getTransactions()) { + // fast check - count number of transactions + assertEq
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit e6b1b11fd88d0be07c86483c3d3148c3605e52ed Merge: 1bc7833 382c0a5 Author: Christopher Tubbs AuthorDate: Thu Apr 4 15:52:20 2019 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0a0f0b40e9c0cfc5766101cb047540c36bb0e6df Merge: fc2b8f0 d6d0b38 Author: Mike Miller AuthorDate: Tue Apr 2 13:03:39 2019 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 393ab12c2d6c1dc2834f0e565cc2fabb6e82f439 Merge: 9b5bbca 07d6346 Author: Christopher Tubbs AuthorDate: Mon Mar 25 18:39:17 2019 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 89f81ba91fecfccb0a1449530910c856e2dbeeb7 Merge: f896c98 995a952 Author: Mike Miller AuthorDate: Mon Mar 25 15:57:12 2019 -0400 Merge branch '1.9' .../accumulo/tserver/log/RecoveryLogReader.java| 3 +++ .../tserver/log/RecoveryLogsReaderTest.java| 25 ++ 2 files changed, 28 insertions(+)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f896c98c2356a52dfa2235d2cc02ae556ab17909 Merge: 6aff988 0a2457b Author: Christopher Tubbs AuthorDate: Fri Mar 22 18:45:50 2019 -0400 Merge branch '1.9' .../org/apache/accumulo/core/conf/Property.java| 4 + .../java/org/apache/accumulo/master/Master.java| 178 +++-- .../master/metrics/MasterMetricsFactory.java | 69 +++- .../master/metrics/fate/FateMetricValues.java | 176 .../accumulo/master/metrics/fate/FateMetrics.java | 146 + .../master/metrics/fate/FateMetricsMBean.java | 27 .../master/metrics/fate/Metrics2FateMetrics.java | 128 +++ .../master/metrics/fate/FateMetricValuesTest.java | 73 + .../resources/hadoop-metrics2-accumulo.properties | 59 +++ 9 files changed, 774 insertions(+), 86 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java index 9529b1a,ede2b8a..2d358e2 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@@ -258,13 -322,17 +258,17 @@@ public enum Property + "completely quit. This delay gives it time before log recoveries begin."), MASTER_LEASE_RECOVERY_WAITING_PERIOD("master.lease.recovery.interval", "5s", PropertyType.TIMEDURATION, - "The amount of time to wait after requesting a WAL file to be recovered"), + "The amount of time to wait after requesting a write-ahead log to be recovered"), MASTER_WALOG_CLOSER_IMPLEMETATION("master.walog.closer.implementation", "org.apache.accumulo.server.master.recovery.HadoopLogCloser", PropertyType.CLASSNAME, - "A class that implements a mechansim to steal write access to a file"), + "A class that implements a mechanism to steal write access to a write-ahead log"), + MASTER_FATE_METRICS_ENABLED("master.fate.metrics.enabled", "false", PropertyType.BOOLEAN, + "Enable reporting of FATE metrics in JMX (and logging with Hadoop Metrics2"), + MASTER_FATE_METRICS_MIN_UPDATE_INTERVAL("master.fate.metrics.min.update.interval", "60s", + PropertyType.TIMEDURATION, "Limit calls from metric sinks to zookeeper to update interval"), MASTER_FATE_THREADPOOL_SIZE("master.fate.threadpool.size", "4", PropertyType.COUNT, - "The number of threads used to run FAult-Tolerant Executions. These are " - + "primarily table operations like merge."), + "The number of threads used to run fault-tolerant executions (FATE)." + + " These are primarily table operations like merge."), MASTER_REPLICATION_SCAN_INTERVAL("master.replication.status.scan.interval", "30s", PropertyType.TIMEDURATION, "Amount of time to sleep before scanning the status section of the " diff --cc server/master/src/main/java/org/apache/accumulo/master/Master.java index a22471e,4b3d691..f8622de --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@@ -126,9 -122,9 +126,8 @@@ import org.apache.accumulo.server.maste import org.apache.accumulo.server.master.state.TabletState; import org.apache.accumulo.server.master.state.ZooStore; import org.apache.accumulo.server.master.state.ZooTabletStateStore; - import org.apache.accumulo.server.metrics.Metrics; -import org.apache.accumulo.server.metrics.MetricsSystemHelper; import org.apache.accumulo.server.replication.ZooKeeperInitialization; -import org.apache.accumulo.server.rpc.RpcWrapper; +import org.apache.accumulo.server.rpc.HighlyAvailableServiceWrapper; import org.apache.accumulo.server.rpc.ServerAddress; import org.apache.accumulo.server.rpc.TCredentialsUpdatingWrapper; import org.apache.accumulo.server.rpc.TServerUtils; @@@ -240,21 -234,21 +239,22 @@@ public class Maste static final boolean X = true; static final boolean O = false; // @formatter:off - static final boolean transitionOK[][] = { + static final boolean[][] transitionOK = { //INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP - /* INITIAL */ {X, X,O,O, O, O, X}, - /* HAVE_LOCK */ {O, X,X,X, O, O, X}, - /* SAFE_MODE */ {O, O,X,X, X, O, X}, - /* NORMAL */ {O, O,X,X, X, O, X}, - /* UNLOAD_METADATA_TABLETS */ {O, O,X,X, X, X, X}, - /* UNLOAD_ROOT_TABLET */ {O, O,O,X, X, X, X}, - /* STOP */{
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 78f8b3f5f203a4845674e563930fea4101813d80 Merge: a86369a e0841b7 Author: Christopher Tubbs AuthorDate: Wed Mar 13 17:47:25 2019 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit ae01a3e2e9230ce434f6017fe78cc9769c377c58 Merge: 394b34d b3ff32a Author: Christopher Tubbs AuthorDate: Wed Feb 27 19:20:50 2019 -0500 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d70a29fd5161c21bb268264070533fd05a607fb6 Merge: ba6d5e1 a3793e6 Author: Christopher Tubbs AuthorDate: Tue Feb 26 19:47:19 2019 -0500 Merge branch '1.9' assemble/bin/accumulo | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) diff --cc assemble/bin/accumulo index 895adac,cad9c9e..4f23770 --- a/assemble/bin/accumulo +++ b/assemble/bin/accumulo @@@ -15,61 -15,174 +15,61 @@@ # See the License for the specific language governing permissions and # limitations under the License. -# Start: Resolve Script Directory -SOURCE="${BASH_SOURCE[0]}" -while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink - bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - SOURCE="$(readlink "${SOURCE}")" - [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" -# Stop: Resolve Script Directory - -. "${bin}"/config.sh - -START_JAR="${ACCUMULO_HOME}/lib/accumulo-start.jar" - -# -# Resolve a program to its installation directory -# -locationByProgram() -{ - RESULT=$(type -P "$1") - if [[ -z "${RESULT}" ]]; then - echo "Cannot find '$1' and '$2' is not set in $ACCUMULO_CONF_DIR/accumulo-env.sh" - exit 1 - fi - while [[ -h "${RESULT}" ]]; do # resolve $RESULT until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$RESULT" )" && pwd )" - RESULT="$(readlink "${RESULT}")" - [[ "${RESULT}" != /* ]] && RESULT="${DIR}/${RESULT}" # if $RESULT was a relative symlink, we need to resolve it relative to the path where the symlink file was located - done - # find the relative home directory, accounting for an extra bin directory - RESULT=$(dirname "$(dirname "${RESULT}")") - echo "Auto-set ${2} to '${RESULT}'. To suppress this message, set ${2} in conf/accumulo-env.sh" - eval "${2}=${RESULT}" +function main() { + + SOURCE="${BASH_SOURCE[0]}" - while [ -h "${SOURCE}" ]; do ++ while [[ -h "${SOURCE}" ]]; do + bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" + SOURCE="$(readlink "${SOURCE}")" + [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" + done + # Set up variables needed by accumulo-env.sh - export bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" - export basedir=$( cd -P "${bin}"/.. && pwd ) ++ bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )"; export bin ++ basedir=$( cd -P "${bin}"/.. && pwd ); export basedir + export conf="${ACCUMULO_CONF_DIR:-${basedir}/conf}" + export lib="${basedir}/lib" + export cmd="$1" + + if [[ -z $conf || ! -d $conf ]]; then +echo "$conf is not a valid directory. Please make sure it exists" +exit 1 + fi + if [[ ! -f "$conf/accumulo-env.sh" ]]; then +echo "accumulo-env.sh must exist in $conf" +exit 1 + fi + source "$conf/accumulo-env.sh" + + # Accumulo is moving away from these variables but they still might be needed + export ACCUMULO_HOME="$basedir" + export ACCUMULO_CONF_DIR="$conf" + + # Verify setting in accumulo-env.sh + : "${JAVA_OPTS:?"variable is not set in accumulo-env.sh"}" + : "${CLASSPATH:?"variable is not set in accumulo-env.sh"}" + : "${ACCUMULO_LOG_DIR:?"variable is not set in accumulo-env.sh"}" + mkdir -p "${ACCUMULO_LOG_DIR}" 2>/dev/null + : "${MALLOC_ARENA_MAX:?"variable is not set in accumulo-env.sh"}" + + if [[ $cmd == "classpath" ]]; then +echo "$CLASSPATH" +exit 0 + fi + + if [[ -x "$JAVA_HOME/bin/java" ]]; then +JAVA="$JAVA_HOME/bin/java" + else - JAVA=$(which java) ++JAVA=$(type -P java) + fi + if [[ ! -x "$JAVA" ]]; then +echo "Could not find any executable java binary. Please set java on your PATH or set JAVA_HOME" +exit 1 + fi + if [[ -n "$ACCUMULO_JAVA_PREFIX" ]]; then - JAVA=($ACCUMULO_JAVA_PREFIX $JAVA) ++JAVA=("${ACCUMULO_JAVA_PREFIX[*]}" "$JAVA") + fi + + exec "${JAVA[@]}" "${JAVA_OPTS[@]}" org.apache.accumulo.start.Main "$@" } -test -z "${JAVA_HOME}" && locationByProgram java JAVA_HOME -test -z "${HADOOP_PREFIX}" && locationByProgram hadoop HADOOP_PREFIX -test -z "${ZOOKEEPER_HOME}" && locationByProgram zkCli.sh ZOOKEEPER_HOME - -# -# ACCUMULO_XTRAJARS is where all of the commandline -add items go into for reading by accumulo. -# It also holds the JAR run with the jar command and, if possible, any items in the JAR manifest's Class-Path. -# -if [[ "$1" = "-add" ]]; then -export ACCUMULO_XTRAJARS="$2" -shift 2 -else -export ACCUMULO_XTRAJARS="" -fi -if [[ "$1" = "jar" && -f "$2" ]]; then -if [[ $2 =~ ^/ ]]; then - jardir="$(dirname "$2")" - jarfile="$2" -else
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 30324d7433bb9edb935e81734594296e98b816b4 Merge: 7c88356 d7fa411 Author: Christopher Tubbs AuthorDate: Wed Feb 20 20:16:21 2019 -0500 Merge branch '1.9' .../core/master/thrift/TabletServerStatus.java | 22 +- core/src/main/thrift/client.thrift | 276 - core/src/main/thrift/data.thrift | 160 +-- core/src/main/thrift/gc.thrift | 30 +- core/src/main/thrift/master.thrift | 269 +++- core/src/main/thrift/replication.thrift| 43 +- core/src/main/thrift/security.thrift | 34 +- core/src/main/thrift/tabletserver.thrift | 535 +--- core/src/main/thrift/trace.thrift |4 +- proxy/src/main/thrift/proxy.thrift | 1284 server/tracer/src/main/thrift/tracer.thrift| 37 +- test/src/main/thrift/test.thrift |6 +- 12 files changed, 2062 insertions(+), 638 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java index f5a36e4,c00558c..e55b59a --- a/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java +++ b/core/src/main/java/org/apache/accumulo/core/master/thrift/TabletServerStatus.java @@@ -41,15 -68,17 +41,15 @@@ public class TabletServerStatus impleme private static final org.apache.thrift.protocol.TField FLUSHS_FIELD_DESC = new org.apache.thrift.protocol.TField("flushs", org.apache.thrift.protocol.TType.I64, (short)15); private static final org.apache.thrift.protocol.TField SYNCS_FIELD_DESC = new org.apache.thrift.protocol.TField("syncs", org.apache.thrift.protocol.TType.I64, (short)16); private static final org.apache.thrift.protocol.TField BULK_IMPORTS_FIELD_DESC = new org.apache.thrift.protocol.TField("bulkImports", org.apache.thrift.protocol.TType.LIST, (short)17); - private static final org.apache.thrift.protocol.TField VERSION_FIELD_DESC = new org.apache.thrift.protocol.TField("version", org.apache.thrift.protocol.TType.STRING, (short)18); - private static final org.apache.thrift.protocol.TField RESPONSE_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("responseTime", org.apache.thrift.protocol.TType.I64, (short)19); ++ private static final org.apache.thrift.protocol.TField VERSION_FIELD_DESC = new org.apache.thrift.protocol.TField("version", org.apache.thrift.protocol.TType.STRING, (short)19); + private static final org.apache.thrift.protocol.TField RESPONSE_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("responseTime", org.apache.thrift.protocol.TType.I64, (short)18); - private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); - static { -schemes.put(StandardScheme.class, new TabletServerStatusStandardSchemeFactory()); -schemes.put(TupleScheme.class, new TabletServerStatusTupleSchemeFactory()); - } + private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new TabletServerStatusStandardSchemeFactory(); + private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new TabletServerStatusTupleSchemeFactory(); - public Map tableMap; // required + public java.util.Map tableMap; // required public long lastContact; // required - public String name; // required + public java.lang.String name; // required public double osLoad; // required public long holdTime; // required public long lookups; // required @@@ -80,13 -108,12 +80,13 @@@ FLUSHS((short)15, "flushs"), SYNCS((short)16, "syncs"), BULK_IMPORTS((short)17, "bulkImports"), - VERSION((short)18, "version"), - RESPONSE_TIME((short)19, "responseTime"); ++VERSION((short)19, "version"), + RESPONSE_TIME((short)18, "responseTime"); -private static final Map byName = new HashMap(); +private static final java.util.Map byName = new java.util.HashMap(); static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { + for (_Fields field : java.util.EnumSet.allOf(_Fields.class)) { byName.put(field.getFieldName(), field); } } @@@ -124,9 -151,7 +124,9 @@@ return SYNCS; case 17: // BULK_IMPORTS return BULK_IMPORTS; - case 18: // VERSION ++case 19: // VERSION + return VERSION; - case 19: // RESPONSE_TIME + case 18: // RESPONSE_TIME return RESPONSE_TIME; default: return null; @@@ -1680,15 -1664,7 +1680,15 @@@ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 18: // VERSION ++ case 19: // VERSION +if (schemeField
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 6d60e3bc16d0b455cb8d035580212af7ffa040d4 Merge: d379b5f 577dd9a Author: Keith Turner AuthorDate: Thu Jan 10 10:19:21 2019 -0500 Merge branch '1.9' .../org/apache/accumulo/tserver/TabletServer.java | 2 +- .../accumulo/tserver/log/RecoveryLogsIterator.java | 10 +++- .../accumulo/tserver/log/SortedLogRecovery.java| 62 -- 3 files changed, 68 insertions(+), 6 deletions(-) diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index a7ddcac,c30b443..a06284d --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@@ -3362,16 -3263,23 +3362,16 @@@ public class TabletServer implements Ru logger.minorCompactionStarted(tablet, lastUpdateSequence, newMapfileLocation, durability); } - public void recover(VolumeManager fs, KeyExtent extent, TableConfiguration tconf, - List logEntries, Set tabletFiles, MutationReceiver mutationReceiver) - throws IOException { + public void recover(VolumeManager fs, KeyExtent extent, List logEntries, + Set tabletFiles, MutationReceiver mutationReceiver) throws IOException { List recoveryLogs = new ArrayList<>(); List sorted = new ArrayList<>(logEntries); -Collections.sort(sorted, new Comparator() { - @Override - public int compare(LogEntry e1, LogEntry e2) { -return (int) (e1.timestamp - e2.timestamp); - } -}); +Collections.sort(sorted, (e1, e2) -> (int) (e1.timestamp - e2.timestamp)); for (LogEntry entry : sorted) { Path recovery = null; - Path finished = RecoveryPath.getRecoveryPath(fs, - fs.getFullPath(FileType.WAL, entry.filename)); + Path finished = RecoveryPath.getRecoveryPath(fs.getFullPath(FileType.WAL, entry.filename)); finished = SortedLogState.getFinishedMarkerPath(finished); - TabletServer.log.info("Looking for " + finished); + TabletServer.log.debug("Looking for " + finished); if (fs.exists(finished)) { recovery = finished.getParent(); } diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogsIterator.java index 64c7af2,8912aeb..31ef699 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogsIterator.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogsIterator.java @@@ -51,10 -52,23 +51,18 @@@ public class RecoveryLogsIterator imple try { for (Path log : recoveryLogPaths) { - iterators.add(new RecoveryLogReader(fs, log, start, end)); + LOG.debug("Opening recovery log {}", log.getName()); + RecoveryLogReader rlr = new RecoveryLogReader(fs, log, start, end); + if (rlr.hasNext()) { + LOG.debug("Write ahead log {} has data in range {} {}", log.getName(), start, end); + iterators.add(rlr); + } else { + LOG.debug("Write ahead log {} has no data in range {} {}", log.getName(), start, end); + rlr.close(); + } } - iter = Iterators.mergeSorted(iterators, new Comparator>() { -@Override -public int compare(Entry o1, Entry o2) { - return o1.getKey().compareTo(o2.getKey()); -} - }); + iter = Iterators.mergeSorted(iterators, (o1, o2) -> o1.getKey().compareTo(o2.getKey())); } catch (RuntimeException | IOException e) { try {
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 2c06451b858a29fb312b6d4717a1b92747560522 Merge: 9198818 c8b4132 Author: Christopher Tubbs AuthorDate: Mon Jan 7 18:10:44 2019 -0500 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a21ea07beb78bbba81bf20418efd6ee82d6185e5 Merge: 017d31a ba572b4 Author: Christopher Tubbs AuthorDate: Thu Nov 15 18:43:39 2018 -0500 Merge branch '1.9' .../native/src/main/c++/nativeMap/BlockAllocator.h | 402 ++--- server/native/src/main/c++/nativeMap/Field.h | 215 ++- server/native/src/main/c++/nativeMap/Key.h | 208 +-- server/native/src/main/c++/nativeMap/NativeMap.h | 329 - server/native/src/main/c++/nativeMap/SubKey.h | 229 ++-- .../org_apache_accumulo_tserver_NativeMap.cc | 102 +++--- 6 files changed, 741 insertions(+), 744 deletions(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. busbey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d0162e53952ba2320cd0cea9875349a3fde7dcc1 Merge: 2e892b3 be6154b Author: Sean Busbey AuthorDate: Thu Nov 15 09:16:49 2018 -0600 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 5cdf125296005728ed767a61681170ec7fa2a536 Merge: 95214d4 b6409ef Author: Keith Turner AuthorDate: Fri Nov 9 10:37:30 2018 -0500 Merge branch '1.9' .../apache/accumulo/tserver/tablet/CompactionRunner.java | 16 +++- 1 file changed, 11 insertions(+), 5 deletions(-) diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionRunner.java index d51c4eb,8de7a9e..5b127d5 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionRunner.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionRunner.java @@@ -34,13 -34,25 +34,19 @@@ final class CompactionRunner implement @Override public void run() { -if (tablet.getTabletServer().isMajorCompactionDisabled()) { - // this will make compaction tasks that were queued when shutdown was - // initiated exit - tablet.removeMajorCompactionQueuedReason(reason); - return; -} - tablet.majorCompact(reason, queued); - - // if there is more work to be done, queue another major compaction - synchronized (tablet) { - if (reason == MajorCompactionReason.NORMAL && tablet.needsMajorCompaction(reason)) - tablet.initiateMajorCompaction(reason); + CompactionStats stats = tablet.majorCompact(reason, queued); + + // Some compaction strategies may always return true for shouldCompact() because they need to + // make blocking calls to gather information. Without the following check these strategies would + // endlessly requeue. So only check if a subsequent compaction is needed if the previous + // compaction actually did something. + if (stats != null && stats.getEntriesRead() > 0) { + // if there is more work to be done, queue another major compaction + synchronized (tablet) { + if (reason == MajorCompactionReason.NORMAL && tablet.needsMajorCompaction(reason)) + tablet.initiateMajorCompaction(reason); + } } }
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0f79ee6a4165f1358114ba7aa92626329c7474f1 Merge: d83837c 1d35788 Author: Keith Turner AuthorDate: Tue Oct 30 16:54:41 2018 -0400 Merge branch '1.9' pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. elserj pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 398c59604484f54a916e56edb2df0fbb561e77e0 Merge: feeb1e1 725a450 Author: Josh Elser AuthorDate: Mon Oct 29 18:00:20 2018 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit feeb1e1c518a9cc578c92051f5835fb3774932a6 Merge: 628336f ff452a4 Author: Keith Turner AuthorDate: Mon Oct 29 13:15:45 2018 -0400 Merge branch '1.9' .../main/java/org/apache/accumulo/server/tables/TableManager.java | 8 .../java/org/apache/accumulo/master/tableOps/delete/CleanUp.java | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --cc server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java index dcbe8b6,95f791c..f62584c --- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java @@@ -49,26 -49,32 +49,28 @@@ import org.apache.zookeeper.Watcher.Eve import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import com.google.common.base.Preconditions; + public class TableManager { - private static SecurityPermission TABLE_MANAGER_PERMISSION = new SecurityPermission( - "tableManagerPermission"); private static final Logger log = LoggerFactory.getLogger(TableManager.class); - private static final Set observers = Collections - .synchronizedSet(new HashSet()); - private static final Map tableStateCache = Collections - .synchronizedMap(new HashMap()); - private static final byte[] ZERO_BYTE = new byte[] {'0'}; - - private static TableManager tableManager = null; - - private final Instance instance; + private static final Set observers = Collections.synchronizedSet(new HashSet<>()); + private static final Map tableStateCache = Collections + .synchronizedMap(new HashMap<>()); + private static final byte[] ZERO_BYTE = {'0'}; + + private final ServerContext context; + private final String zkRoot; + private final String instanceID; + private final ZooReaderWriter zoo; private ZooCache zooStateCache; - public static void prepareNewNamespaceState(String instanceId, String namespaceId, - String namespace, NodeExistsPolicy existsPolicy) + public static void prepareNewNamespaceState(IZooReaderWriter zoo, String instanceId, + Namespace.ID namespaceId, String namespace, NodeExistsPolicy existsPolicy) throws KeeperException, InterruptedException { -log.debug( -"Creating ZooKeeper entries for new namespace " + namespace + " (ID: " + namespaceId + ")"); +log.debug("Creating ZooKeeper entries for new namespace {} (ID: {})", namespace, namespaceId); String zPath = Constants.ZROOT + "/" + instanceId + Constants.ZNAMESPACES + "/" + namespaceId; -IZooReaderWriter zoo = ZooReaderWriter.getInstance(); zoo.putPersistentData(zPath, new byte[0], existsPolicy); zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME, namespace.getBytes(UTF_8), existsPolicy); @@@ -148,43 -164,52 +150,49 @@@ } - public synchronized void transitionTableState(final String tableId, final TableState newState) { + public synchronized void transitionTableState(final Table.ID tableId, final TableState newState) { + Preconditions.checkArgument(newState != TableState.UNKNOWN); -String statePath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZTABLES + "/" -+ tableId + Constants.ZTABLE_STATE; +String statePath = zkRoot + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_STATE; try { - ZooReaderWriter.getInstance().mutate(statePath, newState.name().getBytes(UTF_8), - ZooUtil.PUBLIC, new Mutator() { -@Override -public byte[] mutate(byte[] oldData) throws Exception { - TableState oldState = TableState.UNKNOWN; - if (oldData != null) -oldState = TableState.valueOf(new String(oldData, UTF_8)); - - // this check makes the transition operation idempotent - if (oldState == newState) -return null; // already at desired state, so nothing to do - - boolean transition = true; - // ++ - // v | - // NEW -> (ONLINE|OFFLINE)+--- DELETING - switch (oldState) { -case NEW: - transition = (newState == TableState.OFFLINE || newState == TableState.ONLINE); - break; -case ONLINE: // fall-through intended -case UNKNOWN:// fall through intended -case OFFLINE: - transition = (newState != TableState.NEW); - break; -case DELETING: - // Can't transition to any state from DELETING - transition = false; - break; - } - if (!transition) -throw new IllegalTableTransitionE
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d8e7bd40abb1beaef3698606d2d9cea6d879815f Merge: 2e48434 5de8d0f Author: Mike Miller AuthorDate: Wed Oct 24 18:08:50 2018 -0400 Merge branch '1.9' * Closes #705 * Closes #706 .../mapreduce/lib/impl/InputConfigurator.java | 26 ++ 1 file changed, 22 insertions(+), 4 deletions(-) diff --cc client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java index d381b6d,000..86c9a17 mode 100644,00..100644 --- a/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java +++ b/client/mapreduce/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/InputConfigurator.java @@@ -1,990 -1,0 +1,1008 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.client.mapreduce.lib.impl; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringTokenizer; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.ClientInfo; +import org.apache.accumulo.core.client.ClientSideIteratorScanner; +import org.apache.accumulo.core.client.IsolatedScanner; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.RowIterator; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.impl.ClientContext; +import org.apache.accumulo.core.client.impl.Table; +import org.apache.accumulo.core.client.impl.Tables; +import org.apache.accumulo.core.client.impl.TabletLocator; +import org.apache.accumulo.core.client.mapreduce.InputTableConfig; +import org.apache.accumulo.core.client.sample.SamplerConfiguration; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.PartialKey; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.data.impl.KeyExtent; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.master.state.tables.TableState; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.schema.MetadataSchema; +import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.TablePermission; +import org.apache.accumulo.core.util.Pair; +import org.apache.accumulo.core.util.TextUtil; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.MapWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.collect.Maps; + +/** + * @since 1.6.0 + */ +public class InputConfigurator extends ConfiguratorBase { + + /** + * Configuration keys for {@link Scanner}. + * + * @since 1.6.0 + */ + public static enum ScanOpts { +TABLE_NAME, +AUTHORIZATIONS, +RANGES, +COLUMNS, +ITERATORS, +TABLE_CONFIGS, +SAMPLER_CONFIG, +CLASSLOADER_CONTEXT, +EXECUTION_HINTS + } + + /** + * Configuration keys
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 2aeca7ae1edf2c02d0998c51476bcb174861c77e Merge: caf75c1 987ae65 Author: Mike Miller AuthorDate: Wed Oct 3 12:26:55 2018 -0400 Merge branch '1.9' Conflicts: core/src/main/java/org/apache/accumulo/core/data/Mutation.java .../org/apache/accumulo/core/data/Mutation.java| 13 +-- .../accumulo/core/util/UnsynchronizedBuffer.java | 101 +++-- .../core/util/UnsynchronizedBufferTest.java| 72 +++ .../accumulo/server/data/ServerMutation.java | 4 +- 4 files changed, 157 insertions(+), 33 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/data/Mutation.java index 42094a2,338444d..97dc926 --- a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java +++ b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java @@@ -1613,17 -1125,18 +1614,17 @@@ public class Mutation implements Writab } out.write((byte) (0x80 | hasValues)); - WritableUtils.writeVInt(out, row.length); + UnsynchronizedBuffer.writeVInt(out, integerBuffer, row.length); out.write(row); - WritableUtils.writeVInt(out, data.length); + UnsynchronizedBuffer.writeVInt(out, integerBuffer, data.length); out.write(data); - WritableUtils.writeVInt(out, entries); + UnsynchronizedBuffer.writeVInt(out, integerBuffer, entries); if (0x01 == (0x01 & hasValues)) { - WritableUtils.writeVInt(out, values.size()); + UnsynchronizedBuffer.writeVInt(out, integerBuffer, values.size()); - for (int i = 0; i < values.size(); i++) { -byte val[] = values.get(i); + for (byte[] val : values) { - WritableUtils.writeVInt(out, val.length); + UnsynchronizedBuffer.writeVInt(out, integerBuffer, val.length); out.write(val); } } diff --cc core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java index 59f45fa,2993f8b..f477231 --- a/core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java +++ b/core/src/main/java/org/apache/accumulo/core/util/UnsynchronizedBuffer.java @@@ -136,37 -138,8 +138,12 @@@ public class UnsynchronizedBuffer */ public void writeVLong(long i) { reserve(9); - if (i >= -112 && i <= 127) { - data[offset++] = (byte) i; - return; - } - - int len = -112; - if (i < 0) { - i ^= -1L; // take one's complement' - len = -120; - } - - long tmp = i; - while (tmp != 0) { - tmp = tmp >> 8; - len--; - } - - data[offset++] = (byte) len; - - len = (len < -120) ? -(len + 120) : -(len + 112); - - for (int idx = len; idx != 0; idx--) { - int shiftbits = (idx - 1) * 8; - long mask = 0xFFL << shiftbits; - data[offset++] = (byte) ((i & mask) >> shiftbits); - } + offset = UnsynchronizedBuffer.writeVLong(data, offset, i); } + +public int size() { + return offset; +} } /**
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. elserj pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit c35f17f5354340ee6da773d84c42fee017f8bb81 Merge: 5af231f 159c97a Author: Josh Elser AuthorDate: Mon Sep 24 17:54:34 2018 -0400 Merge branch '1.9' .../java/org/apache/accumulo/core/conf/Property.java | 4 .../org/apache/accumulo/monitor/EmbeddedWebServer.java | 18 -- 2 files changed, 16 insertions(+), 6 deletions(-) diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java index ee2ebaa,e8309be..69222ab --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/EmbeddedWebServer.java @@@ -42,29 -49,23 +42,35 @@@ public class EmbeddedWebServer public EmbeddedWebServer(String host, int port) { server = new Server(); final AccumuloConfiguration conf = Monitor.getContext().getConfiguration(); -if (EMPTY.equals(conf.get(Property.MONITOR_SSL_KEYSTORE)) -|| EMPTY.equals(conf.get(Property.MONITOR_SSL_KEYSTOREPASS)) -|| EMPTY.equals(conf.get(Property.MONITOR_SSL_TRUSTSTORE)) -|| EMPTY.equals(conf.get(Property.MONITOR_SSL_TRUSTSTOREPASS))) { +connector = new ServerConnector(server, getConnectionFactories(conf)); +connector.setHost(host); +connector.setPort(port); + +handler = new ServletContextHandler( +ServletContextHandler.SESSIONS | ServletContextHandler.SECURITY); +handler.getSessionHandler().getSessionCookieConfig().setHttpOnly(true); +handler.setContextPath("/"); + } + + private static AbstractConnectionFactory[] getConnectionFactories(AccumuloConfiguration conf) { +HttpConnectionFactory httpFactory = new HttpConnectionFactory(); +EnumSet requireForSecure = EnumSet.of(Property.MONITOR_SSL_KEYSTORE, - Property.MONITOR_SSL_KEYSTOREPASS, Property.MONITOR_SSL_TRUSTSTORE); ++Property.MONITOR_SSL_KEYSTOREPASS, Property.MONITOR_SSL_TRUSTSTORE, ++Property.MONITOR_SSL_TRUSTSTOREPASS); + +if (requireForSecure.stream().map(p -> conf.get(p)).anyMatch(s -> s == null || s.isEmpty())) { + LOG.debug("Not configuring Jetty to use TLS"); - connector = new ServerConnector(server, new HttpConnectionFactory()); - usingSsl = false; + return new AbstractConnectionFactory[] {httpFactory}; } else { - final String trustStorePass = conf.get(Property.MONITOR_SSL_TRUSTSTOREPASS); - if (trustStorePass.isEmpty()) { - LOG.warn("Truststore JKS file has an empty password which prevents any integrity checks."); + LOG.debug("Configuring Jetty to use TLS"); + final SslContextFactory sslContextFactory = new SslContextFactory(); + // If the key password is the same as the keystore password, we don't + // have to explicitly set it. Thus, if the user doesn't provide a key + // password, don't set anything. + final String keyPass = conf.get(Property.MONITOR_SSL_KEYPASS); + if (!Property.MONITOR_SSL_KEYPASS.getDefaultValue().equals(keyPass)) { + sslContextFactory.setKeyManagerPassword(keyPass); } - SslContextFactory sslContextFactory = new SslContextFactory(); sslContextFactory.setKeyStorePath(conf.get(Property.MONITOR_SSL_KEYSTORE)); sslContextFactory.setKeyStorePassword(conf.get(Property.MONITOR_SSL_KEYSTOREPASS)); sslContextFactory.setKeyStoreType(conf.get(Property.MONITOR_SSL_KEYSTORETYPE));
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 16795c526f6d39d1cffd5be39168850bd1d6fbbd Merge: d55ce35 a590a0f Author: Christopher Tubbs AuthorDate: Fri Sep 21 23:04:52 2018 -0400 Merge branch '1.9' .../org/apache/accumulo/plugin/it/PluginIT.java| 26 +++--- maven-plugin/src/it/settings.xml | 8 --- pom.xml| 5 + 3 files changed, 23 insertions(+), 16 deletions(-) diff --cc maven-plugin/src/it/plugin-test/src/test/java/org/apache/accumulo/plugin/it/PluginIT.java index a00c241,e9518bf..0d58a05 --- a/maven-plugin/src/it/plugin-test/src/test/java/org/apache/accumulo/plugin/it/PluginIT.java +++ b/maven-plugin/src/it/plugin-test/src/test/java/org/apache/accumulo/plugin/it/PluginIT.java @@@ -23,8 -23,6 +23,8 @@@ import java.io.File import java.io.IOException; import java.util.Map.Entry; - import org.apache.accumulo.core.Accumulo; ++import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; @@@ -46,41 -45,42 +46,41 @@@ import org.junit.Test public class PluginIT { - private static Instance instance; - private static Connector connector; + private static ClientInfo info; - private static AccumuloClient connector; ++ private static AccumuloClient client; @BeforeClass public static void setUp() throws Exception { String instanceName = "plugin-it-instance"; -instance = new MiniAccumuloInstance(instanceName, new File("target/accumulo-maven-plugin/" + instanceName)); -connector = instance.getConnector("root", new PasswordToken("ITSecret")); +info = MiniAccumuloCluster.getClientInfo(new File("target/accumulo-maven-plugin/" + instanceName)); - connector = Accumulo.newClient().usingClientInfo(info).build(); ++client = Accumulo.newClient().usingClientInfo(info).build(); } @Test - public void testInstanceConnection() { -assertTrue(instance != null); -assertTrue(instance instanceof MiniAccumuloInstance); -assertTrue(connector != null); -assertTrue(connector instanceof Connector); + public void testConnection() { +assertTrue(info != null); - assertTrue(connector != null); - assertTrue(connector instanceof AccumuloClient); ++assertTrue(client != null); ++assertTrue(client instanceof AccumuloClient); } @Test public void testCreateTable() throws AccumuloException, AccumuloSecurityException, TableExistsException, IOException { String tableName = "testCreateTable"; --connector.tableOperations().create(tableName); --assertTrue(connector.tableOperations().exists(tableName)); -assertTrue(new File("target/accumulo-maven-plugin/" + instance.getInstanceName() + "/testCreateTablePassed").createNewFile()); ++client.tableOperations().create(tableName); ++assertTrue(client.tableOperations().exists(tableName)); +assertTrue(new File("target/accumulo-maven-plugin/" + info.getInstanceName() + "/testCreateTablePassed").createNewFile()); } @Test public void writeToTable() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, IOException { String tableName = "writeToTable"; --connector.tableOperations().create(tableName); --BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig()); ++client.tableOperations().create(tableName); ++BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig()); Mutation m = new Mutation("ROW"); m.put("CF", "CQ", "V"); bw.addMutation(m); bw.close(); --Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY); ++Scanner scanner = client.createScanner(tableName, Authorizations.EMPTY); int count = 0; for (Entry entry : scanner) { count++; @@@ -96,8 -96,8 +96,8 @@@ @Test public void checkIterator() throws IOException, AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException { String tableName = "checkIterator"; --connector.tableOperations().create(tableName); --BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig()); ++client.tableOperations().create(tableName); ++BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig()); Mutation m = new Mutation("ROW1"); m.put("allowed", "CQ1", "V1"); m.put("denied", "CQ2", "V2"); @@@ -111,7 -111,7 +111,7 @@@ bw.close(); // check filter --Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY); ++Sca
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. busbey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 02f58119a283e2552f0577c732ef8aa0aa5226a1 Merge: 65639f3 d832765 Author: Sean Busbey AuthorDate: Thu Sep 20 13:58:44 2018 -0500 Merge branch '1.9' .../org/apache/accumulo/test/mapred/AccumuloFileOutputFormatIT.java | 5 +++-- .../apache/accumulo/test/mapreduce/AccumuloFileOutputFormatIT.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. busbey pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 4b7428fae1e1372e1558b6bb4a68e4bffd973027 Merge: 5e01176 bc1b55b Author: Sean Busbey AuthorDate: Tue Sep 18 13:19:04 2018 -0500 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0db0a3f41383d6136a6eafbb0be2b57adc11a89a Merge: c033667 1213c9a Author: Christopher Tubbs AuthorDate: Fri Sep 14 16:47:22 2018 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 351193ca869d2863c739504a451b21e03f97a797 Merge: deab42b 25472ae Author: Christopher Tubbs AuthorDate: Sat Sep 8 22:14:48 2018 -0400 Merge branch '1.9' .../org/apache/accumulo/core/iterators/system/VisibilityFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 9fb9a1630fee4c6f29db2da8f26d0f942bfb243f Merge: ac4bbe6 eee579d Author: Christopher Tubbs AuthorDate: Fri Aug 10 17:25:24 2018 -0400 Merge branch '1.9' (No change to master)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d627908151eaf1c04fb6ac68d1d9490e1c7046e1 Merge: 02eaab9 7fd1f78 Author: Mike Miller AuthorDate: Wed Aug 1 15:36:03 2018 -0400 Merge branch '1.9' Conflicts: core/src/main/java/org/apache/accumulo/core/iterators/system/VisibilityFilter.java .../apache/accumulo/core/iterators/system/VisibilityFilter.java| 7 +-- 1 file changed, 5 insertions(+), 2 deletions(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 16d71a407beb3357133f983376cf0fe690a7e20b Merge: 223e239 bffa3c4 Author: Keith Turner AuthorDate: Thu Jul 19 14:41:04 2018 -0400 Merge branch '1.9' .../org/apache/accumulo/tserver/tablet/Tablet.java | 23 +++--- 1 file changed, 20 insertions(+), 3 deletions(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0199675374124c3eea3ab0d1dd15934217dd33e5 Merge: bf942d7 69e52a7 Author: Christopher Tubbs AuthorDate: Wed Jul 18 23:44:17 2018 -0400 Merge branch '1.9'
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit bf942d7329b83d5bd55646b04cc24d613b8182a5 Merge: 8589173 a7aee79 Author: Keith Turner AuthorDate: Tue Jul 17 09:43:20 2018 -0400 Merge branch '1.9' .../org/apache/accumulo/core/util/FastFormat.java | 26 +++-- .../apache/accumulo/core/util/FastFormatTest.java | 114 + 2 files changed, 132 insertions(+), 8 deletions(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 147e6ce0790f400572059eb1a22e5a3c60b429b1 Merge: 5e5c74f 42a3534 Author: Keith Turner AuthorDate: Tue Jul 10 18:11:45 2018 -0400 Merge branch '1.9' .../org/apache/accumulo/core/conf/Property.java| 5 +-- .../java/org/apache/accumulo/master/Master.java| 36 -- 2 files changed, 29 insertions(+), 12 deletions(-) diff --cc server/master/src/main/java/org/apache/accumulo/master/Master.java index bb188ce,2f124e3..22e1b82 --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@@ -168,6 -163,8 +169,7 @@@ import org.apache.zookeeper.data.Stat import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Optional; + import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Iterables; /**
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f1f0784cef72d7e338d7666a48a48be0485072eb Merge: 86e66e7 2ee5f49 Author: Christopher Tubbs AuthorDate: Fri Jul 6 11:27:12 2018 -0400 Merge branch '1.9' core/src/main/java/org/apache/accumulo/core/util/Duration.java | 6 +- 1 file changed, 1 insertion(+), 5 deletions(-)
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 86e66e7fdd5794b27c743b8550e5e9fa0d202119 Merge: 13a025d ee88055 Author: Christopher Tubbs AuthorDate: Thu Jul 5 17:18:21 2018 -0400 Merge branch '1.9' .../org/apache/accumulo/core/util/AddressUtil.java | 2 -- .../org/apache/accumulo/core/util/ServerServices.java | 2 -- .../core/client/summary/CountingSummarizerTest.java| 4 .../summarizers/EntryLengthSummarizersTest.java| 1 - pom.xml| 18 +++--- .../accumulo/server/rpc/CustomNonBlockingServer.java | 1 - .../accumulo/server/client/BulkImporterTest.java | 1 - 7 files changed, 3 insertions(+), 26 deletions(-) diff --cc core/src/test/java/org/apache/accumulo/core/client/summary/CountingSummarizerTest.java index d7aff9c,000..c9cb457 mode 100644,00..100644 --- a/core/src/test/java/org/apache/accumulo/core/client/summary/CountingSummarizerTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/summary/CountingSummarizerTest.java @@@ -1,262 -1,0 +1,258 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.core.client.summary; + +import static org.apache.accumulo.core.client.summary.CountingSummarizer.COUNTER_STAT_PREFIX; +import static org.apache.accumulo.core.client.summary.CountingSummarizer.DELETES_IGNORED_STAT; +import static org.apache.accumulo.core.client.summary.CountingSummarizer.EMITTED_STAT; +import static org.apache.accumulo.core.client.summary.CountingSummarizer.INGNORE_DELETES_OPT; +import static org.apache.accumulo.core.client.summary.CountingSummarizer.MAX_COUNTERS_OPT; +import static org.apache.accumulo.core.client.summary.CountingSummarizer.MAX_COUNTER_LEN_OPT; +import static org.apache.accumulo.core.client.summary.CountingSummarizer.SEEN_STAT; +import static org.apache.accumulo.core.client.summary.CountingSummarizer.TOO_LONG_STAT; +import static org.apache.accumulo.core.client.summary.CountingSummarizer.TOO_MANY_STAT; + +import java.util.Arrays; +import java.util.HashMap; + - import org.apache.accumulo.core.client.summary.CounterSummary; - import org.apache.accumulo.core.client.summary.CountingSummarizer; - import org.apache.accumulo.core.client.summary.Summarizer; +import org.apache.accumulo.core.client.summary.Summarizer.Collector; - import org.apache.accumulo.core.client.summary.SummarizerConfiguration; +import org.apache.accumulo.core.client.summary.summarizers.FamilySummarizer; +import org.apache.accumulo.core.client.summary.summarizers.VisibilitySummarizer; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.junit.Assert; +import org.junit.Test; + +public class CountingSummarizerTest { + + public static class MultiSummarizer extends CountingSummarizer { +@Override +protected Converter converter() { + return (k, v, c) -> { +c.accept("rp:" + k.getRowData().subSequence(0, 2)); +c.accept("fp:" + k.getColumnFamilyData().subSequence(0, 2)); +c.accept("qp:" + k.getColumnQualifierData().subSequence(0, 2)); + }; +} + } + + @Test + public void testMultipleEmit() { +SummarizerConfiguration sc = SummarizerConfiguration.builder(MultiSummarizer.class).build(); +MultiSummarizer countSum = new MultiSummarizer(); + +Summarizer.Collector collector = countSum.collector(sc); + +Value val = new Value("abc"); + +HashMap expected = new HashMap<>(); + +for (String row : new String[] {"ask", "asleep", "some", "soul"}) { + for (String fam : new String[] {"hop", "hope", "nope", "noop"}) { +for (String qual : new String[] {"mad", "lad", "lab", "map"}) { + collector.accept(new Key(row, fam, qual), val); + + expected.merge("rp:" + row.substring(0, 2), 1L, Long::sum); + expected.merge("fp:" + fam.substring(0, 2), 1L, Long::sum); + expected.merge("qp:" + qual.substring(0, 2), 1L, Long::sum); +} + } +} + +H
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 13a025d740a3f7bc3ca078f3557bff45f7751c60 Merge: 1311520 9a5b7bd Author: Keith Turner AuthorDate: Mon Jul 2 10:20:11 2018 -0400 Merge branch '1.9' .../org/apache/accumulo/tserver/tablet/Tablet.java | 42 +- 1 file changed, 17 insertions(+), 25 deletions(-) diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index 28c32a5,4590872..2ef4087 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@@ -2484,11 -2500,13 +2473,13 @@@ public class Tablet implements TabletCo } for (String logger : currentLogsCopy) { - log.debug("Logs for current memory: " + getExtent() + " " + logger); + log.debug("Logs for current memory: {} {}", getExtent(), logger); } - for (String logger : doomed) { - log.debug("Logs to be destroyed: {} {}", getExtent(), logger); + Set doomed = new HashSet<>(); + for (DfsLogger logger : doomedLogs) { - log.debug("Logs to be destroyed: " + getExtent() + " " + logger.getMeta()); ++ log.debug("Logs to be destroyed: {} {}", getExtent(), logger.getMeta()); + doomed.add(logger.getMeta()); } return doomed;
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 13115207ff721f881cf1f67fca3f2ef6e79002f5 Merge: 99851cd 05e49e1 Author: Keith Turner AuthorDate: Fri Jun 29 13:17:04 2018 -0400 Merge branch '1.9' .../org/apache/accumulo/tserver/TabletServer.java | 85 +-- .../accumulo/tserver/log/SortedLogRecovery.java| 60 +-- .../org/apache/accumulo/tserver/tablet/Tablet.java | 21 +++- .../accumulo/tserver/WalRemovalOrderTest.java | 115 + .../tserver/log/SortedLogRecoveryTest.java | 48 - 5 files changed, 253 insertions(+), 76 deletions(-) diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/log/SortedLogRecovery.java index 6442991,642064f..f9da274 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/SortedLogRecovery.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/SortedLogRecovery.java @@@ -152,9 -153,14 +152,9 @@@ public class SortedLogRecovery return next; } -@Override -public void remove() { - throw new UnsupportedOperationException("remove"); -} - } - private long findLastStartToFinish(List recoveryLogs, Set tabletFiles, int tabletId) + private long findRecoverySeq(List recoveryLogs, Set tabletFiles, int tabletId) throws IOException { HashSet suffixes = new HashSet<>(); for (String path : tabletFiles) diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java index fa178e3,53d6851..2bb0e47 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java @@@ -397,8 -394,8 +397,8 @@@ public class SortedLogRecoveryTest createKeyValue(DEFINE_TABLET, 1, 1, extent), createKeyValue(COMPACTION_START, 3, 1, "/t1/f1"), createKeyValue(MUTATION, 2, 1, ignored), createKeyValue(MUTATION, 4, 1, m),}; -KeyValue entries2[] = new KeyValue[] {createKeyValue(OPEN, 5, -1, "1"), +KeyValue entries2[] = {createKeyValue(OPEN, 5, -1, "1"), - createKeyValue(DEFINE_TABLET, 6, 1, extent), createKeyValue(COMPACTION_FINISH, 7, 1, null), + createKeyValue(DEFINE_TABLET, 4, 1, extent), createKeyValue(COMPACTION_FINISH, 4, 1, null), createKeyValue(MUTATION, 8, 1, m2),}; Map logs = new TreeMap<>(); logs.put("entries", entries); @@@ -420,11 -417,11 +420,11 @@@ m.put(cf, cq, value); Mutation m2 = new ServerMutation(new Text("row2")); m2.put(cf, cq, new Value("123".getBytes())); -KeyValue entries[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"), +KeyValue entries[] = {createKeyValue(OPEN, 0, -1, "1"), createKeyValue(DEFINE_TABLET, 1, 1, extent), createKeyValue(COMPACTION_FINISH, 2, 1, null), createKeyValue(COMPACTION_START, 4, 1, "/t1/f1"), - createKeyValue(COMPACTION_FINISH, 6, 1, null), createKeyValue(MUTATION, 3, 1, ignored), - createKeyValue(MUTATION, 5, 1, m), createKeyValue(MUTATION, 7, 1, m2),}; + createKeyValue(COMPACTION_FINISH, 5, 1, null), createKeyValue(MUTATION, 3, 1, ignored), + createKeyValue(MUTATION, 5, 1, m), createKeyValue(MUTATION, 5, 1, m2),}; Map logs = new TreeMap<>(); logs.put("entries", entries); // Recover @@@ -450,11 -447,11 +450,11 @@@ createKeyValue(DEFINE_TABLET, 1, 1, extent), createKeyValue(COMPACTION_START, 3, 1, "/t1/f1"), createKeyValue(MUTATION, 2, 1, ignored), createKeyValue(MUTATION, 4, 1, m),}; -KeyValue entries2[] = new KeyValue[] {createKeyValue(OPEN, 5, -1, "1"), +KeyValue entries2[] = {createKeyValue(OPEN, 5, -1, "1"), - createKeyValue(DEFINE_TABLET, 6, 1, extent), createKeyValue(MUTATION, 7, 1, m2),}; + createKeyValue(DEFINE_TABLET, 4, 1, extent), createKeyValue(MUTATION, 4, 1, m2),}; -KeyValue entries3[] = new KeyValue[] {createKeyValue(OPEN, 8, -1, "1"), +KeyValue entries3[] = {createKeyValue(OPEN, 8, -1, "1"), - createKeyValue(DEFINE_TABLET, 9, 1, extent), createKeyValue(COMPACTION_FINISH, 10, 1, null), - createKeyValue(MUTATION, 11, 1, m3),}; + createKeyValue(DEFINE_TABLET, 4, 1, extent), createKeyValue(COMPACTION_FINISH, 4, 1, null), + createKeyValue(MUTATION, 4, 1, m3),}; Map logs = new TreeMap<>(); logs.put("entries", entries); logs.put("entries2", entries2); @@@ -499,11 -496,11 +499,11 @@@ m2.put(cf, cq, value); Mutation m3 = new ServerMutation(new Text("row3")); m3.put(cf, cq, value); -KeyValue entries[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"), +KeyValue entries[] = {createKeyValue(OPEN, 0, -1, "1"), createKeyValue(DEFINE_TABLET, 1, 1, extent), -
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 5a0a63151db7fc3c54f8731257499cb08a72af24 Merge: b484841 1180d76 Author: Keith Turner AuthorDate: Tue May 22 18:02:29 2018 -0400 Merge branch '1.9' .../apache/accumulo/core/file/FileOperations.java | 18 .../file/blockfile/impl/CachableBlockFile.java | 50 +- .../accumulo/core/file/rfile/RFileOperations.java | 5 ++- .../org/apache/accumulo/tserver/FileManager.java | 8 +++- .../tserver/TabletServerResourceManager.java | 7 ++- 5 files changed, 72 insertions(+), 16 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/file/FileOperations.java index a6ee3bc,5a26ad2..9c69377 --- a/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java +++ b/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java @@@ -444,7 -438,12 +457,12 @@@ public abstract class FileOperations /** * (Optional) set the index cache to be used to optimize reads within the constructed reader. */ -public SubbuilderType withIndexCache(BlockCache indexCache); +SubbuilderType withIndexCache(BlockCache indexCache); + + /** + * (Optional) set the file len cache to be used to optimize reads within the constructed reader. + */ -public SubbuilderType withFileLenCache(Cache fileLenCache); ++SubbuilderType withFileLenCache(Cache fileLenCache); } /** diff --cc core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java index de3d88d,336ec4d..7c4f97c --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java @@@ -16,25 -16,29 +16,26 @@@ */ package org.apache.accumulo.core.file.blockfile.impl; -import java.io.ByteArrayInputStream; +import java.io.Closeable; import java.io.DataInputStream; -import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; -import java.lang.ref.SoftReference; -import java.util.concurrent.Callable; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.Map; + import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.file.blockfile.ABlockReader; -import org.apache.accumulo.core.file.blockfile.ABlockWriter; -import org.apache.accumulo.core.file.blockfile.BlockFileReader; -import org.apache.accumulo.core.file.blockfile.BlockFileWriter; import org.apache.accumulo.core.file.blockfile.cache.BlockCache; +import org.apache.accumulo.core.file.blockfile.cache.BlockCache.Loader; import org.apache.accumulo.core.file.blockfile.cache.CacheEntry; +import org.apache.accumulo.core.file.blockfile.cache.CacheEntry.Weighbable; import org.apache.accumulo.core.file.rfile.bcfile.BCFile; import org.apache.accumulo.core.file.rfile.bcfile.BCFile.Reader.BlockReader; -import org.apache.accumulo.core.file.rfile.bcfile.BCFile.Writer.BlockAppender; -import org.apache.accumulo.core.file.streams.PositionedOutput; +import org.apache.accumulo.core.file.rfile.bcfile.MetaBlockDoesNotExist; import org.apache.accumulo.core.file.streams.RateLimitedInputStream; -import org.apache.accumulo.core.file.streams.RateLimitedOutputStream; import org.apache.accumulo.core.util.ratelimit.RateLimiter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@@ -43,9 -47,10 +44,10 @@@ import org.apache.hadoop.fs.Seekable import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Preconditions; + import com.google.common.cache.Cache; /** - * * This is a wrapper class for BCFile that includes a cache for independent caches for datablocks * and metadatablocks */ @@@ -60,24 -147,24 +62,25 @@@ public class CachableBlockFile } /** - * - * * Class wraps the BCFile reader. - * */ - public static class Reader implements BlockFileReader { - + public static class Reader implements Closeable { private final RateLimiter readLimiter; -private BCFile.Reader _bc; -private final String fileName; -private BlockCache _dCache = null; -private BlockCache _iCache = null; +// private BCFile.Reader _bc; +private final String cacheId; +private final BlockCache _dCache; +private final BlockCache _iCache; + private Cache fileLenCache = null; -private InputStream fin = null; -private FileSystem fs; -private Configuration conf; +private volatile InputStream fin = null; private boolean closed = false; -private Accum
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 486c20b623e9c88ac7f84bb92a8d9a00428f778c Merge: 96fc4c4 a398f8b Author: Christopher Tubbs AuthorDate: Mon May 14 01:15:28 2018 -0400 Merge branch '1.9' -- To stop receiving notification emails like this one, please contact ctubb...@apache.org.
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 75be8e10368bddb25d0afac1829313f985e00594 Merge: 636229c 737d703 Author: Christopher Tubbs AuthorDate: Wed May 9 19:02:43 2018 -0400 Merge branch '1.9' pom.xml | 15 +-- 1 file changed, 1 insertion(+), 14 deletions(-) diff --cc pom.xml index f7b13f1,8f4df29..f17ab5f --- a/pom.xml +++ b/pom.xml @@@ -1017,9 -811,22 +1017,9 @@@ 1.2.0 - org.apache.rat - apache-rat-plugin - - - src/main/resources/META-INF/services/* - nbproject/** - nb-configuration.xml - nbactions.xml - - - - - net.revelc.code.formatter formatter-maven-plugin - 2.7.2 + 2.7.3 ${eclipseFormatterStyle} ${maven.compiler.source} -- To stop receiving notification emails like this one, please contact ctubb...@apache.org.
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0899fbea3132ac9539855c5a6beb5692781c801d Merge: 64e6da0 dcc23b2 Author: Christopher Tubbs AuthorDate: Wed May 9 14:31:19 2018 -0400 Merge branch '1.9' -- To stop receiving notification emails like this one, please contact ctubb...@apache.org.
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 49192c9fe45de9efdac88afd0c3c96eb82822721 Merge: 2d869d3 b8f574f Author: Keith Turner AuthorDate: Tue May 8 12:13:44 2018 -0400 Merge branch '1.9' .../org/apache/accumulo/tserver/TabletServer.java | 10 +- .../accumulo/tserver/log/CloseableIterator.java| 25 ++ .../org/apache/accumulo/tserver/log/DfsLogger.java | 8 +- .../{MultiReader.java => RecoveryLogReader.java} | 162 - .../accumulo/tserver/log/RecoveryLogsIterator.java | 100 ++ .../accumulo/tserver/log/SortedLogRecovery.java| 378 +++-- .../apache/accumulo/tserver/logger/LogFileKey.java | 46 +-- .../apache/accumulo/tserver/logger/LogReader.java | 16 +- .../tserver/replication/AccumuloReplicaSystem.java | 6 +- .../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +- .../apache/accumulo/tserver/log/LogEventsTest.java | 36 ++ .../accumulo/tserver/log/LogFileKeyTest.java | 98 ++ ...ReaderTest.java => RecoveryLogsReaderTest.java} | 43 ++- .../tserver/log/SortedLogRecoveryTest.java | 296 +++- .../accumulo/tserver/logger/LogFileTest.java | 14 +- .../replication/AccumuloReplicaSystemTest.java | 33 +- .../BatchWriterReplicationReplayerTest.java| 4 +- .../UnusedWalDoesntCloseReplicationStatusIT.java | 2 +- 18 files changed, 991 insertions(+), 288 deletions(-) diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java index 914544a,bdf5210..708f843 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java @@@ -35,7 -35,6 +35,8 @@@ import java.util.Objects import java.util.Set; import java.util.TreeMap; +import org.apache.accumulo.core.client.impl.Table; ++import org.apache.accumulo.core.client.impl.Table.ID; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.data.impl.KeyExtent; @@@ -675,6 -679,85 +681,85 @@@ public class SortedLogRecoveryTest Assert.assertEquals(m, mutations.get(0)); } + @Test + public void testLeaveAndComeBack() throws IOException { + /** + * This test recreates the situation in bug #449 (Github issues). + */ + Mutation m1 = new ServerMutation(new Text("r1")); + m1.put("f1", "q1", "v1"); + + Mutation m2 = new ServerMutation(new Text("r2")); + m2.put("f1", "q1", "v2"); + + KeyValue entries1[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"), + createKeyValue(DEFINE_TABLET, 100, 10, extent), createKeyValue(MUTATION, 100, 10, m1), + createKeyValue(COMPACTION_START, 101, 10, "/t/f1"), + createKeyValue(COMPACTION_FINISH, 102, 10, null)}; + + KeyValue entries2[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"), + createKeyValue(DEFINE_TABLET, 1, 20, extent), createKeyValue(MUTATION, 1, 20, m2)}; + + Arrays.sort(entries1); + Arrays.sort(entries2); + Map logs = new TreeMap<>(); + logs.put("entries1", entries1); + logs.put("entries2", entries2); + + List mutations = recover(logs, extent); + + Assert.assertEquals(1, mutations.size()); + Assert.assertEquals(m2, mutations.get(0)); + } + + @Test + public void testMultipleTablets() throws IOException { -KeyExtent e1 = new KeyExtent("1", new Text("m"), null); -KeyExtent e2 = new KeyExtent("1", null, new Text("m")); ++KeyExtent e1 = new KeyExtent(ID.of("1"), new Text("m"), null); ++KeyExtent e2 = new KeyExtent(ID.of("1"), null, new Text("m")); + + Mutation m1 = new ServerMutation(new Text("b")); + m1.put("f1", "q1", "v1"); + + Mutation m2 = new ServerMutation(new Text("b")); + m2.put("f1", "q2", "v2"); + + Mutation m3 = new ServerMutation(new Text("s")); + m3.put("f1", "q1", "v3"); + + Mutation m4 = new ServerMutation(new Text("s")); + m4.put("f1", "q2", "v4"); + + KeyValue entries1[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"), + createKeyValue(DEFINE_TABLET, 7, 10, e1), createKeyValue(DEFINE_TABLET, 5, 11, e2), + createKeyValue(MUTATION, 8, 10, m1), createKeyValue(COMPACTION_START, 9, 10, "/t/f1"), + createKeyValue(MUTATION, 10, 10, m2), createKeyValue(COMPACTION_FINISH, 11, 10, null), + createKeyValue(MUTATION, 6, 11, m3), createKeyValue(COMPACTION_START, 7, 11, "/t/f2"), + createKeyValue(MUTATION, 8, 11, m4)}; + + Arrays.sort(entries1); + + Map logs = new TreeMap<>(); + logs.put("entries1", entries1); + + List mutations1 = recover(logs, e1); + Assert.assertEquals(1, mutations1.size()); + Assert.assertEquals(m2, mutations1.get(0)); + + List mutations2 = recover(logs, e2
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 38648a499110cfc630c0d4a6d4ce1ae2bb8f88d9 Merge: b056117 5bf5180 Author: Keith Turner AuthorDate: Fri May 4 14:42:27 2018 -0400 Merge branch '1.9' .../accumulo/gc/GarbageCollectWriteAheadLogs.java | 89 +- .../gc/GarbageCollectWriteAheadLogsTest.java | 31 +++- 2 files changed, 97 insertions(+), 23 deletions(-) diff --cc server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java index 02da2c4,405a7a0..c8b2e4e --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java @@@ -244,6 -251,21 +251,21 @@@ public class GarbageCollectWriteAheadLo return result; } + private long removeFile(Path path) { + try { + if (!useTrash || !fs.moveToTrash(path)) { + fs.deleteRecursively(path); + } + return 1; + } catch (FileNotFoundException ex) { - log.debug("Attempted to delete WAL {} that did not exists : {}", path, ex.getMessage()); ++ // ignored + } catch (IOException ex) { + log.error("Unable to delete wal {}", path, ex); + } + + return 0; + } + private long removeFiles(Collection> collection, final GCStatus status) { for (Pair stateFile : collection) { Path path = stateFile.getSecond(); -- To stop receiving notification emails like this one, please contact ktur...@apache.org.
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 89ef9ddbd00f6161aa8d31bca6ff9f65e014e9b8 Merge: 563519a 7e4915a Author: Keith Turner AuthorDate: Thu Apr 26 17:41:29 2018 -0400 Merge branch '1.9' .../src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index 3e30743,18d8daf..5c3507a --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@@ -38,7 -39,7 +38,6 @@@ import java.util.PriorityQueue import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; - import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.Callable; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -- To stop receiving notification emails like this one, please contact ktur...@apache.org.
[accumulo] 01/01: Merge branch '1.9'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 563519a0d2d3f044a9762158f7e10c2f39189fe7 Merge: 88d987f 84791ec Author: Keith Turner AuthorDate: Thu Apr 26 17:18:36 2018 -0400 Merge branch '1.9' .../main/java/org/apache/accumulo/tserver/TabletServer.java | 5 - .../main/java/org/apache/accumulo/tserver/tablet/Tablet.java | 12 +++- 2 files changed, 11 insertions(+), 6 deletions(-) -- To stop receiving notification emails like this one, please contact ktur...@apache.org.