Merge branch '1.5' into 1.6
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9b20a9d4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9b20a9d4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9b20a9d4 Branch: refs/heads/1.6 Commit: 9b20a9d4062f9ac0d34ce4b2acd7b9b72524a84f Parents: 5e0f6a4 2deabd3 Author: Christopher Tubbs <ctubb...@apache.org> Authored: Fri Oct 31 23:37:27 2014 -0400 Committer: Christopher Tubbs <ctubb...@apache.org> Committed: Fri Oct 31 23:37:27 2014 -0400 ---------------------------------------------------------------------- assemble/bin/bootstrap_config.sh | 2 +- .../org/apache/accumulo/core/Constants.java | 13 +++++- .../apache/accumulo/core/cli/ClientOpts.java | 6 ++- .../accumulo/core/client/BatchWriterConfig.java | 11 ++--- .../accumulo/core/client/ZooKeeperInstance.java | 14 ++++--- .../client/impl/InstanceOperationsImpl.java | 4 +- .../accumulo/core/client/impl/Namespaces.java | 6 ++- .../accumulo/core/client/impl/ServerClient.java | 4 +- .../core/client/impl/TableOperationsImpl.java | 32 ++++++++------- .../accumulo/core/client/impl/Tables.java | 10 +++-- .../accumulo/core/client/impl/Writer.java | 5 ++- .../core/client/lexicoder/StringLexicoder.java | 6 +-- .../core/client/mapreduce/RangeInputSplit.java | 7 ++-- .../mapreduce/lib/impl/ConfiguratorBase.java | 4 +- .../mapreduce/lib/impl/InputConfigurator.java | 8 ++-- .../mapreduce/lib/impl/OutputConfigurator.java | 7 ++-- .../lib/partition/RangePartitioner.java | 7 ++-- .../accumulo/core/client/mock/MockShell.java | 11 ++--- .../client/security/tokens/PasswordToken.java | 5 ++- .../accumulo/core/conf/ConfigurationDocGen.java | 9 ++-- .../accumulo/core/data/ArrayByteSequence.java | 8 ++-- .../org/apache/accumulo/core/data/Column.java | 6 +-- .../apache/accumulo/core/data/Condition.java | 9 ++-- .../org/apache/accumulo/core/data/Value.java | 4 +- .../accumulo/core/file/BloomFilterLayer.java | 7 ++-- .../core/iterators/ColumnFamilyCounter.java | 5 ++- .../accumulo/core/iterators/LongCombiner.java | 7 ++-- .../accumulo/core/iterators/conf/ColumnSet.java | 7 ++-- .../core/iterators/user/BigDecimalCombiner.java | 7 ++-- .../core/iterators/user/GrepIterator.java | 5 ++- .../iterators/user/IntersectingIterator.java | 7 ++-- .../core/iterators/user/LargeRowFilter.java | 5 ++- .../core/iterators/user/RegExFilter.java | 7 ++-- .../iterators/user/RowDeletingIterator.java | 5 ++- .../iterators/user/SummingArrayCombiner.java | 7 ++-- .../iterators/user/TransformingIterator.java | 7 ++-- .../core/iterators/user/VisibilityFilter.java | 5 ++- .../accumulo/core/security/Authorizations.java | 13 +++--- .../core/security/ColumnVisibility.java | 43 ++++++++++---------- .../accumulo/core/security/Credentials.java | 11 ++--- .../core/security/VisibilityConstraint.java | 5 ++- .../core/security/VisibilityParseException.java | 6 +-- .../core/trace/InstanceUserPassword.java | 5 ++- .../accumulo/core/trace/ZooTraceClient.java | 5 ++- .../core/util/ByteArrayBackedCharSequence.java | 5 ++- .../apache/accumulo/core/util/ByteArraySet.java | 6 +-- .../accumulo/core/util/ByteBufferUtil.java | 5 ++- .../apache/accumulo/core/util/CreateToken.java | 5 ++- .../org/apache/accumulo/core/util/Encoding.java | 5 ++- .../apache/accumulo/core/util/FastFormat.java | 4 +- .../org/apache/accumulo/core/util/Merge.java | 5 ++- .../apache/accumulo/core/util/MonitorUtil.java | 4 +- .../org/apache/accumulo/core/util/TextUtil.java | 4 +- .../apache/accumulo/core/util/shell/Shell.java | 6 ++- .../util/shell/commands/AddSplitsCommand.java | 7 ++-- .../shell/commands/AuthenticateCommand.java | 5 ++- .../util/shell/commands/CreateTableCommand.java | 7 ++-- .../util/shell/commands/ExecfileCommand.java | 5 ++- .../core/util/shell/commands/HiddenCommand.java | 5 ++- .../core/util/shell/commands/PasswdCommand.java | 5 ++- .../shell/commands/QuotedStringTokenizer.java | 5 ++- .../core/util/shell/commands/UserCommand.java | 5 ++- .../core/client/BatchWriterConfigTest.java | 6 +-- .../core/client/ZooKeeperInstanceTest.java | 9 ++-- .../core/client/mock/MockNamespacesTest.java | 4 +- .../tokens/CredentialProviderTokenTest.java | 7 ++-- .../security/tokens/PasswordTokenTest.java | 7 ++-- .../accumulo/core/data/ConditionTest.java | 12 +++--- .../core/data/ConditionalMutationTest.java | 10 ++--- .../apache/accumulo/core/data/ValueTest.java | 5 ++- .../core/iterators/user/RegExFilterTest.java | 7 ++-- .../core/security/ColumnVisibilityTest.java | 6 +-- .../accumulo/core/security/CredentialsTest.java | 6 +-- .../core/security/VisibilityConstraintTest.java | 4 +- .../security/crypto/BlockedIOStreamTest.java | 10 ++--- .../core/util/format/DeleterFormatterTest.java | 12 +++--- .../ShardedTableDistributionFormatterTest.java | 8 ++-- .../org/apache/accumulo/fate/AdminUtil.java | 8 ++-- .../java/org/apache/accumulo/fate/ZooStore.java | 16 ++++---- .../zookeeper/DistributedReadWriteLock.java | 18 ++++---- .../accumulo/fate/zookeeper/ZooCache.java | 8 ++-- .../apache/accumulo/fate/zookeeper/ZooLock.java | 6 +-- .../accumulo/fate/zookeeper/ZooReservation.java | 9 ++-- .../accumulo/fate/zookeeper/ZooSession.java | 6 +-- .../impl/MiniAccumuloClusterImpl.java | 4 +- .../org/apache/accumulo/proxy/ProxyServer.java | 7 ++-- .../apache/accumulo/proxy/TestProxyClient.java | 15 +++---- .../java/org/apache/accumulo/proxy/Util.java | 5 ++- .../org/apache/accumulo/server/Accumulo.java | 4 +- .../accumulo/server/client/HdfsZooInstance.java | 10 +++-- .../server/conf/ZooCachePropertyAccessor.java | 5 ++- .../accumulo/server/conf/ZooConfiguration.java | 4 +- .../server/constraints/MetadataConstraints.java | 14 ++++--- .../apache/accumulo/server/init/Initialize.java | 15 +++---- .../accumulo/server/master/LiveTServerSet.java | 3 +- .../server/master/state/DeadServerList.java | 7 ++-- .../server/master/state/TServerInstance.java | 7 ++-- .../master/state/TabletStateChangeIterator.java | 5 ++- .../accumulo/server/master/state/ZooStore.java | 5 ++- .../master/state/ZooTabletStateStore.java | 11 ++--- .../server/metrics/AbstractMetricsImpl.java | 5 ++- .../accumulo/server/monitor/LogService.java | 4 +- .../server/security/SecurityOperation.java | 4 +- .../server/security/SystemCredentials.java | 10 +++-- .../security/handler/ZKAuthenticator.java | 4 +- .../server/security/handler/ZKAuthorizor.java | 5 ++- .../server/security/handler/ZKPermHandler.java | 5 ++- .../accumulo/server/tables/TableManager.java | 18 ++++---- .../server/tablets/UniqueNameAllocator.java | 10 +++-- .../server/util/AddFilesWithMissingEntries.java | 5 ++- .../accumulo/server/util/ChangeSecret.java | 5 ++- .../accumulo/server/util/CleanZookeeper.java | 6 ++- .../accumulo/server/util/DeleteZooInstance.java | 6 ++- .../accumulo/server/util/DumpZookeeper.java | 5 ++- .../accumulo/server/util/ListInstances.java | 6 ++- .../accumulo/server/util/MetadataTableUtil.java | 32 ++++++++------- .../accumulo/server/util/NamespacePropUtil.java | 4 +- .../accumulo/server/util/RandomizeVolumes.java | 6 +-- .../accumulo/server/util/RestoreZookeeper.java | 7 ++-- .../accumulo/server/util/SendLogToChainsaw.java | 9 ++-- .../accumulo/server/util/SystemPropUtil.java | 4 +- .../accumulo/server/util/TablePropUtil.java | 4 +- .../accumulo/server/util/TabletServerLocks.java | 4 +- .../server/watcher/MonitorLog4jWatcher.java | 4 +- .../accumulo/server/zookeeper/ZooQueueLock.java | 9 ++-- .../server/zookeeper/ZooReaderWriter.java | 5 ++- .../zookeeper/ZooReaderWriterFactory.java | 5 ++- .../server/conf/NamespaceConfigurationTest.java | 10 +++-- .../conf/ServerConfigurationFactoryTest.java | 4 +- .../server/conf/TableConfigurationTest.java | 9 ++-- .../conf/ZooCachePropertyAccessorTest.java | 4 +- .../java/org/apache/accumulo/master/Master.java | 8 ++-- .../master/recovery/RecoveryManager.java | 4 +- .../accumulo/master/state/SetGoalState.java | 4 +- .../accumulo/master/tableOps/BulkImport.java | 8 ++-- .../master/tableOps/CancelCompactions.java | 10 +++-- .../accumulo/master/tableOps/CompactRange.java | 14 ++++--- .../accumulo/master/tableOps/ExportTable.java | 8 ++-- .../accumulo/master/tableOps/ImportTable.java | 16 ++++---- .../accumulo/master/tableOps/RenameTable.java | 6 ++- .../apache/accumulo/master/tableOps/Utils.java | 14 ++++--- .../master/tserverOps/ShutdownTServer.java | 4 +- .../org/apache/accumulo/monitor/Monitor.java | 6 ++- .../accumulo/monitor/ZooKeeperStatus.java | 7 ++-- .../accumulo/monitor/servlets/BasicServlet.java | 10 +++-- .../monitor/servlets/DefaultServlet.java | 5 ++- .../accumulo/monitor/servlets/ShellServlet.java | 5 ++- .../accumulo/monitor/servlets/XMLServlet.java | 7 ++-- .../accumulo/monitor/servlets/trace/Basic.java | 5 ++- .../org/apache/accumulo/tracer/TraceServer.java | 8 ++-- .../tserver/BulkFailedCopyProcessor.java | 5 ++- .../org/apache/accumulo/tserver/Tablet.java | 10 +++-- .../apache/accumulo/tserver/TabletServer.java | 9 ++-- .../apache/accumulo/tserver/log/DfsLogger.java | 4 +- .../accumulo/tserver/logger/LogFileValue.java | 11 ++--- .../accumulo/tserver/logger/LogReader.java | 5 ++- .../accumulo/test/BulkImportDirectory.java | 5 ++- .../accumulo/test/NativeMapConcurrencyTest.java | 5 ++- .../accumulo/test/NativeMapPerformanceTest.java | 7 ++-- .../accumulo/test/NativeMapStressTest.java | 5 ++- .../apache/accumulo/test/TestBinaryRows.java | 7 ++-- .../org/apache/accumulo/test/TestIngest.java | 7 ++-- .../accumulo/test/TestMultiTableIngest.java | 5 ++- .../test/continuous/ContinuousIngest.java | 8 ++-- .../test/continuous/ContinuousMoru.java | 7 ++-- .../test/continuous/ContinuousQuery.java | 5 ++- .../test/continuous/ContinuousScanner.java | 5 ++- .../test/continuous/ContinuousVerify.java | 7 ++-- .../test/continuous/ContinuousWalk.java | 17 ++++---- .../accumulo/test/continuous/Histogram.java | 6 +-- .../test/continuous/PrintScanTimeHistogram.java | 5 ++- .../accumulo/test/continuous/TimeBinner.java | 5 ++- .../test/continuous/UndefinedAnalyzer.java | 9 ++-- .../test/functional/CacheTestReader.java | 7 ++-- .../test/functional/CacheTestWriter.java | 11 ++--- .../accumulo/test/functional/ZombieTServer.java | 4 +- .../metadata/MetadataBatchScanTest.java | 7 ++-- .../apache/accumulo/test/randomwalk/Module.java | 7 ++-- .../test/randomwalk/bulk/BulkMinusOne.java | 5 ++- .../accumulo/test/randomwalk/bulk/Verify.java | 5 ++- .../test/randomwalk/concurrent/BatchWrite.java | 5 ++- .../test/randomwalk/concurrent/BulkImport.java | 5 ++- .../concurrent/ChangeAuthorizations.java | 7 ++-- .../randomwalk/concurrent/StopTabletServer.java | 4 +- .../accumulo/test/randomwalk/image/Verify.java | 5 ++- .../accumulo/test/randomwalk/image/Write.java | 9 ++-- .../test/randomwalk/multitable/Write.java | 5 ++- .../randomwalk/security/SecurityHelper.java | 7 ++-- .../test/randomwalk/security/TableOp.java | 9 ++-- .../test/randomwalk/sequential/Write.java | 5 ++- .../test/randomwalk/shard/BulkInsert.java | 5 ++- .../test/randomwalk/shard/ExportIndex.java | 5 ++- .../accumulo/test/scalability/Ingest.java | 5 ++- .../apache/accumulo/test/AuditMessageIT.java | 6 +-- .../org/apache/accumulo/test/NamespacesIT.java | 8 ++-- .../java/org/apache/accumulo/test/VolumeIT.java | 6 +-- .../accumulo/test/functional/AddSplitIT.java | 5 ++- .../test/functional/BadIteratorMincIT.java | 6 +-- .../test/functional/BatchScanSplitIT.java | 7 ++-- .../test/functional/BatchWriterFlushIT.java | 5 ++- .../accumulo/test/functional/BulkFileIT.java | 5 ++- .../accumulo/test/functional/ConcurrencyIT.java | 7 ++-- .../accumulo/test/functional/ConstraintIT.java | 23 ++++++----- .../test/functional/CreateAndUseIT.java | 5 ++- .../test/functional/DeleteEverythingIT.java | 4 +- .../test/functional/DeleteRowsSplitIT.java | 6 +-- .../accumulo/test/functional/LargeRowIT.java | 7 ++-- .../accumulo/test/functional/NativeMapIT.java | 15 +++---- .../test/functional/ScanIteratorIT.java | 7 ++-- .../accumulo/test/functional/ScanRangeIT.java | 5 ++- .../test/functional/ScanSessionTimeOutIT.java | 5 ++- .../test/functional/SplitRecoveryIT.java | 3 +- .../accumulo/test/functional/TabletIT.java | 4 +- .../accumulo/test/functional/VisibilityIT.java | 11 ++--- .../instrument/receivers/ZooSpanClient.java | 4 +- 215 files changed, 900 insertions(+), 667 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/assemble/bin/bootstrap_config.sh ---------------------------------------------------------------------- diff --cc assemble/bin/bootstrap_config.sh index fb643da,0000000..fd262f0 mode 100755,000000..100755 --- a/assemble/bin/bootstrap_config.sh +++ b/assemble/bin/bootstrap_config.sh @@@ -1,323 -1,0 +1,323 @@@ +#! /usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function usage { + cat <<EOF +Usage: bootstrap_config.sh [-options] +where options include (long options not available on all platforms): + -d, --dir Alternate directory to setup config files + -s, --size Supported sizes: '1GB' '2GB' '3GB' '512MB' + -n, --native Configure to use native libraries + -j, --jvm Configure to use the jvm + -o, --overwrite Overwrite the default config directory + -v, --version Specify the Apache Hadoop version supported versions: '1' '2' + -h, --help Print this help message +EOF +} + +# Start: Resolve Script Directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +# Stop: Resolve Script Directory + +# +# Resolve accumulo home for bootstrapping +# +ACCUMULO_HOME=$( cd -P ${bin}/.. && pwd ) +TEMPLATE_CONF_DIR="${ACCUMULO_HOME}/conf/templates" +CONF_DIR="${ACCUMULO_HOME}/conf" +ACCUMULO_SITE=accumulo-site.xml +ACCUMULO_ENV=accumulo-env.sh + +SIZE= +TYPE= +HADOOP_VERSION= +OVERWRITE="0" +BASE_DIR= + +#Execute getopt +if [[ $(uname -s) == "Linux" ]]; then + args=$(getopt -o "b:d:s:njov:h" -l "basedir:,dir:,size:,native,jvm,overwrite,version:,help" -q -- "$@") +else # Darwin, BSD + args=$(getopt b:d:s:njov:h $*) +fi + +#Bad arguments +if [[ $? != 0 ]]; then + usage 1>&2 + exit 1 +fi +eval set -- $args + +for i +do + case "$i" in + -b|--basedir) #Hidden option used to set general.maven.project.basedir for developers + BASE_DIR=$2; shift + shift;; + -d|--dir) + CONF_DIR=$2; shift + shift;; + -s|--size) + SIZE=$2; shift + shift;; + -n|--native) + TYPE=native + shift;; + -j|--jvm) + TYPE=jvm + shift;; + -o|--overwrite) + OVERWRITE=1 + shift;; + -v|--version) + HADOOP_VERSION=$2; shift + shift;; + -h|--help) + usage + exit 0 + shift;; + --) + shift + break;; + esac +done + +while [[ "${OVERWRITE}" = "0" ]]; do + if [[ -e "${CONF_DIR}/${ACCUMULO_ENV}" || -e "${CONF_DIR}/${ACCUMULO_SITE}" ]]; then + echo "Warning your current config files in ${CONF_DIR} will be overwritten!" + echo + echo "How would you like to proceed?:" + select CHOICE in 'Continue with overwrite' 'Specify new conf dir'; do + if [[ "${CHOICE}" = 'Specify new conf dir' ]]; then + echo -n "Please specifiy new conf directory: " + read CONF_DIR + elif [[ "${CHOICE}" = 'Continue with overwrite' ]]; then + OVERWRITE=1 + fi + break + done + else + OVERWRITE=1 + fi +done - echo "Coppying configuration files to: ${CONF_DIR}" ++echo "Copying configuration files to: ${CONF_DIR}" + +#Native 1GB +native_1GB_tServer="-Xmx128m -Xms128m" +_1GB_master="-Xmx128m -Xms128m" +_1GB_monitor="-Xmx64m -Xms64m" +_1GB_gc="-Xmx64m -Xms64m" +_1GB_other="-Xmx128m -Xms64m" + +_1GB_memoryMapMax="256M" +native_1GB_nativeEnabled="true" +_1GB_cacheDataSize="15M" +_1GB_cacheIndexSize="40M" +_1GB_sortBufferSize="50M" +_1GB_waLogMaxSize="256M" + +#Native 2GB +native_2GB_tServer="-Xmx256m -Xms256m" +_2GB_master="-Xmx256m -Xms256m" +_2GB_monitor="-Xmx128m -Xms64m" +_2GB_gc="-Xmx128m -Xms128m" +_2GB_other="-Xmx256m -Xms64m" + +_2GB_memoryMapMax="512M" +native_2GB_nativeEnabled="true" +_2GB_cacheDataSize="30M" +_2GB_cacheIndexSize="80M" +_2GB_sortBufferSize="50M" +_2GB_waLogMaxSize="512M" + +#Native 3GB +native_3GB_tServer="-Xmx1g -Xms1g -XX:NewSize=500m -XX:MaxNewSize=500m" +_3GB_master="-Xmx1g -Xms1g" +_3GB_monitor="-Xmx1g -Xms256m" +_3GB_gc="-Xmx256m -Xms256m" +_3GB_other="-Xmx1g -Xms256m" + +_3GB_memoryMapMax="1G" +native_3GB_nativeEnabled="true" +_3GB_cacheDataSize="128M" +_3GB_cacheIndexSize="128M" +_3GB_sortBufferSize="200M" +_3GB_waLogMaxSize="1G" + +#Native 512MB +native_512MB_tServer="-Xmx48m -Xms48m" +_512MB_master="-Xmx128m -Xms128m" +_512MB_monitor="-Xmx64m -Xms64m" +_512MB_gc="-Xmx64m -Xms64m" +_512MB_other="-Xmx128m -Xms64m" + +_512MB_memoryMapMax="80M" +native_512MB_nativeEnabled="true" +_512MB_cacheDataSize="7M" +_512MB_cacheIndexSize="20M" +_512MB_sortBufferSize="50M" +_512MB_waLogMaxSize="100M" + +#JVM 1GB +jvm_1GB_tServer="-Xmx384m -Xms384m" + +jvm_1GB_nativeEnabled="false" + +#JVM 2GB +jvm_2GB_tServer="-Xmx768m -Xms768m" + +jvm_2GB_nativeEnabled="false" + +#JVM 3GB +jvm_3GB_tServer="-Xmx2g -Xms2g -XX:NewSize=1G -XX:MaxNewSize=1G" + +jvm_3GB_nativeEnabled="false" + +#JVM 512MB +jvm_512MB_tServer="-Xmx128m -Xms128m" + +jvm_512MB_nativeEnabled="false" + + +if [[ -z "${SIZE}" ]]; then + echo "Choose the heap configuration:" + select DIRNAME in 1GB 2GB 3GB 512MB; do + echo "Using '${DIRNAME}' configuration" + SIZE=${DIRNAME} + break + done +elif [[ "${SIZE}" != "1GB" && "${SIZE}" != "2GB" && "${SIZE}" != "3GB" && "${SIZE}" != "512MB" ]]; then + echo "Invalid memory size" + echo "Supported sizes: '1GB' '2GB' '3GB' '512MB'" + exit 1 +fi + +if [[ -z "${TYPE}" ]]; then + echo + echo "Choose the Accumulo memory-map type:" + select TYPENAME in Java Native; do + if [[ "${TYPENAME}" == "Native" ]]; then + TYPE="native" + echo "Don't forget to build the native libraries using the bin/build_native_library.sh script" + elif [[ "${TYPENAME}" == "Java" ]]; then + TYPE="jvm" + fi + echo "Using '${TYPE}' configuration" + echo + break + done +fi + +if [[ -z "${HADOOP_VERSION}" ]]; then + echo + echo "Choose the Apache Hadoop version:" + select HADOOP in 'HADOOP 1' 'HADOOP 2' ; do + if [ "${HADOOP}" == "HADOOP 2" ]; then + HADOOP_VERSION="2" + elif [ "${HADOOP}" == "HADOOP 1" ]; then + HADOOP_VERSION="1" + fi + echo "Using Apache Hadoop version '${HADOOP_VERSION}' configuration" + echo + break + done +elif [[ "${HADOOP_VERSION}" != "1" && "${HADOOP_VERSION}" != "2" ]]; then + echo "Invalid Apache Hadoop version" + echo "Supported Apache Hadoop versions: '1' '2'" + exit 1 +fi + +for var in SIZE TYPE HADOOP_VERSION; do + if [[ -z ${!var} ]]; then + echo "Invalid $var configuration" + exit 1 + fi +done + +TSERVER="${TYPE}_${SIZE}_tServer" +MASTER="_${SIZE}_master" +MONITOR="_${SIZE}_monitor" +GC="_${SIZE}_gc" +OTHER="_${SIZE}_other" + +MEMORY_MAP_MAX="_${SIZE}_memoryMapMax" +NATIVE="${TYPE}_${SIZE}_nativeEnabled" +CACHE_DATA_SIZE="_${SIZE}_cacheDataSize" +CACHE_INDEX_SIZE="_${SIZE}_cacheIndexSize" +SORT_BUFFER_SIZE="_${SIZE}_sortBufferSize" +WAL_MAX_SIZE="_${SIZE}_waLogMaxSize" + +MAVEN_PROJ_BASEDIR="" + +if [[ ! -z "${BASE_DIR}" ]]; then + MAVEN_PROJ_BASEDIR="\n <property>\n <name>general.maven.project.basedir</name>\n <value>${BASE_DIR}</value>\n </property>\n" +fi + +#Configure accumulo-env.sh +mkdir -p "${CONF_DIR}" && cp ${TEMPLATE_CONF_DIR}/* ${CONF_DIR}/ +sed -e "s/\${tServerHigh_tServerLow}/${!TSERVER}/" \ + -e "s/\${masterHigh_masterLow}/${!MASTER}/" \ + -e "s/\${monitorHigh_monitorLow}/${!MONITOR}/" \ + -e "s/\${gcHigh_gcLow}/${!GC}/" \ + -e "s/\${otherHigh_otherLow}/${!OTHER}/" \ + ${TEMPLATE_CONF_DIR}/$ACCUMULO_ENV > ${CONF_DIR}/$ACCUMULO_ENV + +#Configure accumulo-site.xml +sed -e "s/\${memMapMax}/${!MEMORY_MAP_MAX}/" \ + -e "s/\${nativeEnabled}/${!NATIVE}/" \ + -e "s/\${cacheDataSize}/${!CACHE_DATA_SIZE}/" \ + -e "s/\${cacheIndexSize}/${!CACHE_INDEX_SIZE}/" \ + -e "s/\${sortBufferSize}/${!SORT_BUFFER_SIZE}/" \ + -e "s/\${waLogMaxSize}/${!WAL_MAX_SIZE}/" \ + -e "s=\${mvnProjBaseDir}=${MAVEN_PROJ_BASEDIR}=" ${TEMPLATE_CONF_DIR}/$ACCUMULO_SITE > ${CONF_DIR}/$ACCUMULO_SITE + +#Configure for hadoop 1 +if [[ "$HADOOP_VERSION" = "1" ]]; then + sed -e 's/^test -z \"$HADOOP_CONF_DIR\"/#test -z \"$HADOOP_CONF_DIR\"/' -e 's/^# test -z "$HADOOP_CONF_DIR"/test -z \"$HADOOP_CONF_DIR\"/' ${CONF_DIR}/$ACCUMULO_ENV > temp + mv temp ${CONF_DIR}/$ACCUMULO_ENV + sed -e 's/<!-- Hadoop 2 requirements -->/<!-- Hadoop 2 requirements -->\n <!--/' \ + -e 's/<!-- End Hadoop 2 requirements -->/-->\n <!-- End Hadoop 2 requirements -->/' \ + ${CONF_DIR}/$ACCUMULO_SITE > temp + mv temp ${CONF_DIR}/$ACCUMULO_SITE +fi + +#Additional setup steps for native configuration. +if [[ "${TYPE}" = "native" ]]; then + if [[ "$(uname)" = 'Linux' ]]; then + if [[ -z $HADOOP_PREFIX ]]; then + echo "HADOOP_PREFIX not set cannot automatically configure LD_LIBRARY_PATH" + else + NATIVE_LIB=$(readlink -ef $(dirname $(for x in $(find $HADOOP_PREFIX -name libhadoop.so); do ld $x 2>/dev/null && echo $x && break; done) 2>>/dev/null) 2>>/dev/null) + if [[ -z $NATIVE_LIB ]]; then + echo -e "Native libraries could not be found for your sytem in: $HADOOP_PREFIX" + else + sed "/# Should the monitor/ i export LD_LIBRARY_PATH=${NATIVE_LIB}:\${LD_LIBRARY_PATH}" ${CONF_DIR}/$ACCUMULO_ENV > temp + mv temp ${CONF_DIR}/$ACCUMULO_ENV + echo -e "Added ${NATIVE_LIB} to the LD_LIBRARY_PATH" + fi + fi + fi + echo -e "Please remember to compile the native libraries using the bin/build_native_library.sh script and to set the LD_LIBRARY_PATH variable in the ${CONF_DIR}/accumulo-env.sh script if needed." +fi +echo "Setup complete" http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/Constants.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java index 2a570f4,240de73..9897262 --- a/core/src/main/java/org/apache/accumulo/core/Constants.java +++ b/core/src/main/java/org/apache/accumulo/core/Constants.java @@@ -16,17 -16,37 +16,26 @@@ */ package org.apache.accumulo.core; + import static com.google.common.base.Charsets.UTF_8; + import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.KeyExtent; -import org.apache.accumulo.core.data.PartialKey; -import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.util.ColumnFQ; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Text; + import com.google.common.base.Charsets; + public class Constants { - public static final Charset UTF8 = Charset.forName("UTF-8"); + /** - * @deprecated since 1.5.3; statically import Guava's {@link Charsets#UTF_8} or Java 7's built-in StandardCharsets.UTF_8 or use ++ * @deprecated since 1.5.3, 1.6.2; statically import Guava's {@link Charsets#UTF_8} or Java 7's built-in StandardCharsets.UTF_8 or use + * {@link Charset#forName(String)} with "UTF-8" instead + */ + @Deprecated + public static final Charset UTF8 = UTF_8; public static final String VERSION = FilteredConstants.VERSION; - - // versions should never be negative - public static final Integer WIRE_VERSION = 2; - public static final int DATA_VERSION = 5; - public static final int PREV_DATA_VERSION = 4; - + // Zookeeper locations public static final String ZROOT = "/accumulo"; public static final String ZINSTANCES = "/instances"; @@@ -56,62 -76,117 +65,62 @@@ public static final String ZMONITOR_LOCK = ZMONITOR + "/lock"; public static final String ZMONITOR_HTTP_ADDR = ZMONITOR + "/http_addr"; public static final String ZMONITOR_LOG4J_ADDR = ZMONITOR + "/log4j_addr"; - + public static final String ZCONFIG = "/config"; - + public static final String ZTSERVERS = "/tservers"; - + public static final String ZDEAD = "/dead"; - public static final String ZDEADTSERVERS = "/dead/tservers"; - + public static final String ZDEADTSERVERS = ZDEAD + "/tservers"; + public static final String ZTRACERS = "/tracers"; - + public static final String ZPROBLEMS = "/problems"; - public static final String ZUSERS = "/users"; - + public static final String BULK_ARBITRATOR_TYPE = "bulkTx"; - + public static final String ZFATE = "/fate"; - + public static final String ZNEXT_FILE = "/next_file"; - + public static final String ZBULK_FAILED_COPYQ = "/bulk_failed_copyq"; - + public static final String ZHDFS_RESERVATIONS = "/hdfs_reservations"; public static final String ZRECOVERY = "/recovery"; - - public static final String METADATA_TABLE_ID = "!0"; - public static final String METADATA_TABLE_NAME = "!METADATA"; - public static final String DEFAULT_TABLET_LOCATION = "/default_tablet"; - public static final String TABLE_TABLET_LOCATION = "/table_info"; - public static final String ZTABLE_LOCKS = "/table_locks"; - - // reserved keyspace is any row that begins with a tilde '~' character - public static final Key METADATA_RESERVED_KEYSPACE_START_KEY = new Key(new Text(new byte[] {'~'})); - public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[] {'~' + 1})); - public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false); - public static final String METADATA_DELETE_FLAG_PREFIX = "~del"; - public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX; - public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false); - public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false); - public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in progress - public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false); - - public static final Text METADATA_SERVER_COLUMN_FAMILY = new Text("srv"); - public static final Text METADATA_TABLET_COLUMN_FAMILY = new Text("~tab"); // this needs to sort after all other column families for that tablet - public static final Text METADATA_CURRENT_LOCATION_COLUMN_FAMILY = new Text("loc"); - public static final Text METADATA_FUTURE_LOCATION_COLUMN_FAMILY = new Text("future"); - public static final Text METADATA_LAST_LOCATION_COLUMN_FAMILY = new Text("last"); - public static final Text METADATA_BULKFILE_COLUMN_FAMILY = new Text("loaded"); // temporary marker that indicates a tablet loaded a bulk file - public static final Text METADATA_CLONED_COLUMN_FAMILY = new Text("!cloned"); // temporary marker that indicates a tablet was successfully cloned - - // README : very important that prevRow sort last to avoid race conditions between - // garbage collector and split - public static final ColumnFQ METADATA_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("~pr")); // this needs to sort after everything - // else for that tablet - public static final ColumnFQ METADATA_OLD_PREV_ROW_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("oldprevrow")); - public static final ColumnFQ METADATA_DIRECTORY_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("dir")); - public static final ColumnFQ METADATA_TIME_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("time")); - public static final ColumnFQ METADATA_FLUSH_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("flush")); - public static final ColumnFQ METADATA_COMPACT_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("compact")); - public static final ColumnFQ METADATA_SPLIT_RATIO_COLUMN = new ColumnFQ(METADATA_TABLET_COLUMN_FAMILY, new Text("splitRatio")); - public static final ColumnFQ METADATA_LOCK_COLUMN = new ColumnFQ(METADATA_SERVER_COLUMN_FAMILY, new Text("lock")); - - public static final Text METADATA_DATAFILE_COLUMN_FAMILY = new Text("file"); - public static final Text METADATA_SCANFILE_COLUMN_FAMILY = new Text("scan"); - public static final Text METADATA_LOG_COLUMN_FAMILY = new Text("log"); - public static final Text METADATA_CHOPPED_COLUMN_FAMILY = new Text("chopped"); - public static final ColumnFQ METADATA_CHOPPED_COLUMN = new ColumnFQ(METADATA_CHOPPED_COLUMN_FAMILY, new Text("chopped")); - - public static final Range NON_ROOT_METADATA_KEYSPACE = new Range( - new Key(KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null)).followingKey(PartialKey.ROW), true, METADATA_RESERVED_KEYSPACE_START_KEY, false); - public static final Range METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_TABLE_ID)), true, METADATA_RESERVED_KEYSPACE_START_KEY, false); - - public static final KeyExtent ROOT_TABLET_EXTENT = new KeyExtent(new Text(METADATA_TABLE_ID), KeyExtent.getMetadataEntry(new Text(METADATA_TABLE_ID), null), - null); - public static final Range METADATA_ROOT_TABLET_KEYSPACE = new Range(ROOT_TABLET_EXTENT.getMetadataEntry(), false, KeyExtent.getMetadataEntry(new Text( - METADATA_TABLE_ID), null), true); - + /** - * @deprecated since 1.5.3; not used, but specifies UTF-8 + * Initial tablet directory name for the default tablet in all tables */ - @Deprecated - public static final String VALUE_ENCODING = UTF_8.name(); - + public static final String DEFAULT_TABLET_LOCATION = "/default_tablet"; + + public static final String ZTABLE_LOCKS = "/table_locks"; + public static final String BULK_PREFIX = "b-"; - public static final String OLD_BULK_PREFIX = "bulk_"; - - // note: all times are in milliseconds - - public static final int SCAN_BATCH_SIZE = 1000; // this affects the table client caching of metadata - - public static final long MIN_MASTER_LOOP_TIME = 1000; - public static final int MASTER_TABLETSERVER_CONNECTION_TIMEOUT = 3000; - public static final long CLIENT_SLEEP_BEFORE_RECONNECT = 1000; - + + public static final String CLONE_PREFIX = "c-"; - public static final byte[] CLONE_PREFIX_BYTES = CLONE_PREFIX.getBytes(UTF8); ++ public static final byte[] CLONE_PREFIX_BYTES = CLONE_PREFIX.getBytes(UTF_8); + + // this affects the table client caching of metadata + public static final int SCAN_BATCH_SIZE = 1000; + + // Scanners will default to fetching 3 batches of Key/Value pairs before asynchronously + // fetching the next batch. + public static final long SCANNER_DEFAULT_READAHEAD_THRESHOLD = 3l; + // Security configuration public static final String PW_HASH_ALGORITHM = "SHA-256"; - - // Representation of an empty set of authorizations - // (used throughout the code, because scans of metadata table and many tests do not set record-level visibility) - public static final Authorizations NO_AUTHS = new Authorizations(); - - public static final int DEFAULT_MINOR_COMPACTION_MAX_SLEEP_TIME = 60 * 3; // in seconds - + + /** + * @deprecated since 1.6.0; Use {@link Authorizations#EMPTY} instead + */ + @Deprecated + public static final Authorizations NO_AUTHS = Authorizations.EMPTY; + public static final int MAX_DATA_TO_PRINT = 64; - public static final int CLIENT_RETRIES = 5; - public static final int TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER = 2; public static final String CORE_PACKAGE_NAME = "org.apache.accumulo.core"; - public static final String OLD_PACKAGE_NAME = "cloudbase"; - public static final String VALID_TABLE_NAME_REGEX = "^\\w+$"; public static final String MAPFILE_EXTENSION = "map"; public static final String GENERATED_TABLET_DIRECTORY_PREFIX = "t-"; - + public static final String EXPORT_METADATA_FILE = "metadata.bin"; public static final String EXPORT_TABLE_CONFIG_FILE = "table_config.txt"; public static final String EXPORT_FILE = "exportMetadata.zip"; http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java index 35a9067,5134521..041c9e6 --- a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java +++ b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java @@@ -16,6 -16,9 +16,8 @@@ */ package org.apache.accumulo.core.cli; + import static com.google.common.base.Charsets.UTF_8; + -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java index 6d35757,251d6a7..a9c3292 --- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java @@@ -16,6 -16,11 +16,8 @@@ */ package org.apache.accumulo.core.client; + import static com.google.common.base.Charsets.UTF_8; + -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java index 63e3497,0000000..6925f18 mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/InstanceOperationsImpl.java @@@ -1,210 -1,0 +1,212 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.client.impl; + ++import static com.google.common.base.Charsets.UTF_8; ++ +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.ActiveScan; +import org.apache.accumulo.core.client.admin.ActiveCompaction; +import org.apache.accumulo.core.client.admin.InstanceOperations; +import org.apache.accumulo.core.client.impl.thrift.ClientService; +import org.apache.accumulo.core.client.impl.thrift.ConfigurationType; +import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException; +import org.apache.accumulo.core.master.thrift.MasterClientService; +import org.apache.accumulo.core.security.Credentials; +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService; +import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client; +import org.apache.accumulo.core.util.AddressUtil; +import org.apache.accumulo.core.util.ArgumentChecker; +import org.apache.accumulo.core.util.ThriftUtil; +import org.apache.accumulo.core.zookeeper.ZooUtil; +import org.apache.accumulo.fate.zookeeper.ZooCache; +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory; +import org.apache.accumulo.trace.instrument.Tracer; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +/** + * Provides a class for administering the accumulo instance + */ +public class InstanceOperationsImpl implements InstanceOperations { + private Instance instance; + private Credentials credentials; + + /** + * @param instance + * the connection information for this instance + * @param credentials + * the Credential, containing principal and Authentication Token + */ + public InstanceOperationsImpl(Instance instance, Credentials credentials) { + ArgumentChecker.notNull(instance, credentials); + this.instance = instance; + this.credentials = credentials; + } + + @Override + public void setProperty(final String property, final String value) throws AccumuloException, AccumuloSecurityException { + ArgumentChecker.notNull(property, value); + MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() { + @Override + public void execute(MasterClientService.Client client) throws Exception { + client.setSystemProperty(Tracer.traceInfo(), credentials.toThrift(instance), property, value); + } + }); + } + + @Override + public void removeProperty(final String property) throws AccumuloException, AccumuloSecurityException { + ArgumentChecker.notNull(property); + MasterClient.execute(instance, new ClientExec<MasterClientService.Client>() { + @Override + public void execute(MasterClientService.Client client) throws Exception { + client.removeSystemProperty(Tracer.traceInfo(), credentials.toThrift(instance), property); + } + }); + } + + @Override + public Map<String,String> getSystemConfiguration() throws AccumuloException, AccumuloSecurityException { + return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() { + @Override + public Map<String,String> execute(ClientService.Client client) throws Exception { + return client.getConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), ConfigurationType.CURRENT); + } + }); + } + + @Override + public Map<String,String> getSiteConfiguration() throws AccumuloException, AccumuloSecurityException { + return ServerClient.execute(instance, new ClientExecReturn<Map<String,String>,ClientService.Client>() { + @Override + public Map<String,String> execute(ClientService.Client client) throws Exception { + return client.getConfiguration(Tracer.traceInfo(), credentials.toThrift(instance), ConfigurationType.SITE); + } + }); + } + + @Override + public List<String> getTabletServers() { + ZooCache cache = new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut()); + String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS; + List<String> results = new ArrayList<String>(); + for (String candidate : cache.getChildren(path)) { + List<String> children = cache.getChildren(path + "/" + candidate); + if (children != null && children.size() > 0) { + List<String> copy = new ArrayList<String>(children); + Collections.sort(copy); + byte[] data = cache.get(path + "/" + candidate + "/" + copy.get(0)); - if (data != null && !"master".equals(new String(data, Constants.UTF8))) { ++ if (data != null && !"master".equals(new String(data, UTF_8))) { + results.add(candidate); + } + } + } + return results; + } + + @Override + public List<ActiveScan> getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException { + Client client = null; + try { + client = ThriftUtil.getTServerClient(tserver, ServerConfigurationUtil.getConfiguration(instance)); + + List<ActiveScan> as = new ArrayList<ActiveScan>(); + for (org.apache.accumulo.core.tabletserver.thrift.ActiveScan activeScan : client.getActiveScans(Tracer.traceInfo(), credentials.toThrift(instance))) { + try { + as.add(new ActiveScanImpl(instance, activeScan)); + } catch (TableNotFoundException e) { + throw new AccumuloException(e); + } + } + return as; + } catch (TTransportException e) { + throw new AccumuloException(e); + } catch (ThriftSecurityException e) { + throw new AccumuloSecurityException(e.user, e.code, e); + } catch (TException e) { + throw new AccumuloException(e); + } finally { + if (client != null) + ThriftUtil.returnClient(client); + } + } + + @Override + public boolean testClassLoad(final String className, final String asTypeName) throws AccumuloException, AccumuloSecurityException { + return ServerClient.execute(instance, new ClientExecReturn<Boolean,ClientService.Client>() { + @Override + public Boolean execute(ClientService.Client client) throws Exception { + return client.checkClass(Tracer.traceInfo(), credentials.toThrift(instance), className, asTypeName); + } + }); + } + + @Override + public List<ActiveCompaction> getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException { + Client client = null; + try { + client = ThriftUtil.getTServerClient(tserver, ServerConfigurationUtil.getConfiguration(instance)); + + List<ActiveCompaction> as = new ArrayList<ActiveCompaction>(); + for (org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction activeCompaction : client.getActiveCompactions(Tracer.traceInfo(), + credentials.toThrift(instance))) { + as.add(new ActiveCompactionImpl(instance, activeCompaction)); + } + return as; + } catch (TTransportException e) { + throw new AccumuloException(e); + } catch (ThriftSecurityException e) { + throw new AccumuloSecurityException(e.user, e.code, e); + } catch (TException e) { + throw new AccumuloException(e); + } finally { + if (client != null) + ThriftUtil.returnClient(client); + } + } + + @Override + public void ping(String tserver) throws AccumuloException { + TTransport transport = null; + try { + transport = ThriftUtil.createTransport(AddressUtil.parseAddress(tserver, false), ServerConfigurationUtil.getConfiguration(instance)); + TabletClientService.Client client = ThriftUtil.createClient(new TabletClientService.Client.Factory(), transport); + client.getTabletServerStatus(Tracer.traceInfo(), credentials.toThrift(instance)); + } catch (TTransportException e) { + throw new AccumuloException(e); + } catch (ThriftSecurityException e) { + throw new AccumuloException(e); + } catch (TException e) { + throw new AccumuloException(e); + } finally { + if (transport != null) { + transport.close(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java index 30f0266,0000000..f32940c mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Namespaces.java @@@ -1,154 -1,0 +1,156 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.client.impl; + ++import static com.google.common.base.Charsets.UTF_8; ++ +import java.security.SecurityPermission; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.NamespaceNotFoundException; +import org.apache.accumulo.core.util.ArgumentChecker.Validator; +import org.apache.accumulo.core.zookeeper.ZooUtil; +import org.apache.accumulo.fate.zookeeper.ZooCache; +import org.apache.accumulo.fate.zookeeper.ZooCacheFactory; + +public class Namespaces { + public static final String VALID_NAME_REGEX = "^\\w*$"; + public static final Validator<String> VALID_NAME = new Validator<String>() { + @Override + public boolean isValid(String namespace) { + return namespace != null && namespace.matches(VALID_NAME_REGEX); + } + + @Override + public String invalidMessage(String namespace) { + if (namespace == null) + return "Namespace cannot be null"; + return "Namespaces must only contain word characters (letters, digits, and underscores): " + namespace; + } + }; + + public static final Validator<String> NOT_DEFAULT = new Validator<String>() { + @Override + public boolean isValid(String namespace) { + return !Namespaces.DEFAULT_NAMESPACE.equals(namespace); + } + + @Override + public String invalidMessage(String namespace) { + return "Namespace cannot be the reserved empty namespace"; + } + }; + + public static final Validator<String> NOT_ACCUMULO = new Validator<String>() { + @Override + public boolean isValid(String namespace) { + return !Namespaces.ACCUMULO_NAMESPACE.equals(namespace); + } + + @Override + public String invalidMessage(String namespace) { + return "Namespace cannot be the reserved namespace, " + Namespaces.ACCUMULO_NAMESPACE; + } + }; + + private static SecurityPermission TABLES_PERMISSION = new SecurityPermission("tablesPermission"); + + public static final String DEFAULT_NAMESPACE_ID = "+default"; + public static final String DEFAULT_NAMESPACE = ""; + public static final String ACCUMULO_NAMESPACE_ID = "+accumulo"; + public static final String ACCUMULO_NAMESPACE = "accumulo"; + + private static ZooCache getZooCache(Instance instance) { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(TABLES_PERMISSION); + } + return new ZooCacheFactory().getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut()); + } + + private static SortedMap<String,String> getMap(Instance instance, boolean nameAsKey) { + ZooCache zc = getZooCache(instance); + + List<String> namespaceIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES); + + TreeMap<String,String> namespaceMap = new TreeMap<String,String>(); + + for (String id : namespaceIds) { + byte[] path = zc.get(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES + "/" + id + Constants.ZNAMESPACE_NAME); + if (path != null) { + if (nameAsKey) - namespaceMap.put(new String(path, Constants.UTF8), id); ++ namespaceMap.put(new String(path, UTF_8), id); + else - namespaceMap.put(id, new String(path, Constants.UTF8)); ++ namespaceMap.put(id, new String(path, UTF_8)); + } + } + return namespaceMap; + } + + public static boolean exists(Instance instance, String namespaceId) { + ZooCache zc = getZooCache(instance); + List<String> namespaceIds = zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZNAMESPACES); + return namespaceIds.contains(namespaceId); + } + + public static String getNamespaceId(Instance instance, String namespace) throws NamespaceNotFoundException { + String id = getNameToIdMap(instance).get(namespace); + if (id == null) + throw new NamespaceNotFoundException(null, namespace, "getNamespaceId() failed to find namespace"); + return id; + } + + public static String getNamespaceName(Instance instance, String namespaceId) throws NamespaceNotFoundException { + String namespaceName = getIdToNameMap(instance).get(namespaceId); + if (namespaceName == null) + throw new NamespaceNotFoundException(namespaceId, null, "getNamespaceName() failed to find namespace"); + return namespaceName; + } + + public static SortedMap<String,String> getNameToIdMap(Instance instance) { + return getMap(instance, true); + } + + public static SortedMap<String,String> getIdToNameMap(Instance instance) { + return getMap(instance, false); + } + + public static List<String> getTableIds(Instance instance, String namespaceId) throws NamespaceNotFoundException { + String namespace = getNamespaceName(instance, namespaceId); + List<String> names = new LinkedList<String>(); + for (Entry<String,String> nameToId : Tables.getNameToIdMap(instance).entrySet()) + if (namespace.equals(Tables.qualify(nameToId.getKey()).getFirst())) + names.add(nameToId.getValue()); + return names; + } + + public static List<String> getTableNames(Instance instance, String namespaceId) throws NamespaceNotFoundException { + String namespace = getNamespaceName(instance, namespaceId); + List<String> names = new LinkedList<String>(); + for (String name : Tables.getNameToIdMap(instance).keySet()) + if (namespace.equals(Tables.qualify(name).getFirst())) + names.add(name); + return names; + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/9b20a9d4/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java index 8da530b,1fcebe7..70bdbb0 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ServerClient.java @@@ -16,7 -16,11 +16,9 @@@ */ package org.apache.accumulo.core.client.impl; + import static com.google.common.base.Charsets.UTF_8; + import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloException; @@@ -128,10 -142,13 +130,10 @@@ public class ServerClient for (String tserver : zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTSERVERS)) { String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + tserver; byte[] data = ZooUtil.getLockData(zc, path); - if (data != null && !new String(data, Constants.UTF8).equals("master")) - if (data != null) { - String tserverData = new String(data, UTF_8); - if (!tserverData.equals("master")) { - servers.add(new ThriftTransportKey(new ServerServices(tserverData).getAddressString(Service.TSERV_CLIENT), conf.getPort(Property.TSERV_CLIENTPORT), - rpcTimeout)); - } - } ++ if (data != null && !new String(data, UTF_8).equals("master")) + servers.add(new ThriftTransportKey( + new ServerServices(new String(data)).getAddressString(Service.TSERV_CLIENT), + rpcTimeout, SslConnectionParams.forClient(ServerConfigurationUtil.getConfiguration(instance)))); } boolean opened = false;
