AMBARI-21459. Add lucene index migration script to infra solr client package (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f072dd21 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f072dd21 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f072dd21 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: f072dd2156e83d8a487ce0c3229c1ae22788c6be Parents: e799f52 Author: oleewere <[email protected]> Authored: Wed Jul 12 21:04:54 2017 +0200 Committer: oleewere <[email protected]> Committed: Sat Jul 15 19:04:53 2017 +0200 ---------------------------------------------------------------------- ambari-infra/ambari-infra-solr-client/build.xml | 1 + ambari-infra/ambari-infra-solr-client/pom.xml | 10 ++ .../src/main/resources/solrIndexHelper.sh | 156 +++++++++++++++++++ 3 files changed, 167 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f072dd21/ambari-infra/ambari-infra-solr-client/build.xml ---------------------------------------------------------------------- diff --git a/ambari-infra/ambari-infra-solr-client/build.xml b/ambari-infra/ambari-infra-solr-client/build.xml index a54e336..9b8b6cc 100644 --- a/ambari-infra/ambari-infra-solr-client/build.xml +++ b/ambari-infra/ambari-infra-solr-client/build.xml @@ -35,6 +35,7 @@ </copy> <copy todir="target/package" includeEmptyDirs="no"> <fileset file="src/main/resources/solrCloudCli.sh"/> + <fileset file="src/main/resources/solrIndexHelper.sh"/> </copy> <copy todir="target/package" includeEmptyDirs="no"> <fileset file="src/main/resources/log4j.properties"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/f072dd21/ambari-infra/ambari-infra-solr-client/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-infra/ambari-infra-solr-client/pom.xml b/ambari-infra/ambari-infra-solr-client/pom.xml index d103003..3818aba 100644 --- a/ambari-infra/ambari-infra-solr-client/pom.xml +++ b/ambari-infra/ambari-infra-solr-client/pom.xml @@ -36,6 +36,16 @@ <version>${solr.version}</version> </dependency> <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + <version>${solr.version}</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-backward-codecs</artifactId> + <version>${solr.version}</version> + </dependency> + <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>3.4.9</version> http://git-wip-us.apache.org/repos/asf/ambari/blob/f072dd21/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh ---------------------------------------------------------------------- diff --git a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh new file mode 100755 index 0000000..12e6a77 --- /dev/null +++ b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +JVM="java" +sdir="`dirname \"$0\"`" +: ${JAVA_HOME:?"Please set the JAVA_HOME for lucene index migration!"} + +function print_help() { + cat << EOF + + Usage: solrIndexHelper.sh [<command>] [<arguments with flags>] + + commands: + upgrade-index Check and upgrade solr index data in core directories. + run-check-index-tool call 'java -cp ... org.apache.lucene.index.IndexUpgrader' directly + run-upgrade-index-tool call 'java -cp ... org.apache.lucene.index.CheckIndex' directly + help print usage + + + upgrade-index command arguments: + -d, --index-data-dir <DIRECTORY> Location of the solr cores (e.g.: /opt/ambari_infra_solr/data) + -c, --core-filter <FILTER1,FILTER2> Comma separated name filters of core directoies (default: hadoop_logs,audit_logs,history) + -f, --force Force to start index upgrade, even is the version is at least 6. + +EOF +} + +function upgrade_core() { + local INDEX_DIR=${1:?"usage: <index_base_dir> e.g.: /opt/ambari_infra_solr/data"} + local FORCE_UPDATE=${2:?"usage <force_update_flag> e.g.: true"} + local SOLR_CORE_FILTERS=${3:?"usage: <comma separated core filters> e.g.: hadoop_logs,audit_logs,history"} + + SOLR_CORE_FILTER_ARR=$(echo $SOLR_CORE_FILTERS | sed "s/,/ /g") + + for coll in $SOLR_CORE_FILTER_ARR; do + if [[ "$1" == *"$coll"* ]]; then + echo "Core '$1' dir name contains $coll (core filter)'"; + version=$(PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.CheckIndex -fast $1|grep " version="|sed -e 's/.*=//g'|head -1) + if [ -z $version ] ; then + echo "Core '$1' - Empty index?" + return + fi + majorVersion=$(echo $version|cut -c 1) + if [ $majorVersion -ge 6 ] && [ $FORCE_UPDATE == "false" ] ; then + echo "Core '$1' - Already on version $version, not upgrading. Use -f or --force option to run upgrade anyway." + else + echo "Core '$1' - Index version is $version, upgrading ..." + PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.IndexUpgrader -delete-prior-commits $1 + echo "Upgrading core '$1' has finished" + fi + fi + done +} + +function upgrade_index() { + while [[ $# -gt 0 ]] + do + key="$1" + case $key in + -c|--core-filters) + local SOLR_CORE_FILTERS="$2" + shift 2 + ;; + -f|--force) + local FORCE_UPDATE="true" + shift + ;; + -d|--index-data-dir) + local INDEX_DIR="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac + done + if [[ -z "$INDEX_DIR" ]] ; then + echo "Index data dirctory option is required (-d or --index-data-dir). Exiting..." + exit 1 + fi + + if [[ -z "$SOLR_CORE_FILTERS" ]] ; then + SOLR_CORE_FILTERS="hadoop_logs,audit_logs,history" + fi + + if [[ -z "$FORCE_UPDATE" ]] ; then + FORCE_UPDATE="false" + else + echo "NOTE: Forcing index upgrade is set." + fi + + CORES=$(for replica_dir in `find $INDEX_DIR -name data`; do dirname $replica_dir; done); + if [[ -z "$CORES" ]] ; then + echo "No indices found on path $INDEX_DIR" + else + for c in $CORES ; do + if find $c/data -maxdepth 1 -type d -name 'index*' 1> /dev/null 2>&1; then + name=$(echo $c | sed -e 's/.*\///g') + abspath=$(cd "$(dirname "$c")"; pwd)/$(basename "$c") + find $c/data -maxdepth 1 -type d -name 'index*' | while read indexDir; do + echo "Checking core $name - $abspath" + upgrade_core "$indexDir" "$FORCE_UPDATE" "$SOLR_CORE_FILTERS" + done + else + echo "No index folder found for $name" + fi + done + echo "DONE" + fi +} + +function upgrade_index_tool() { + # see: https://cwiki.apache.org/confluence/display/solr/IndexUpgrader+Tool + PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.IndexUpgrader ${@} +} + +function check_index_tool() { + PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar" org.apache.lucene.index.CheckIndex ${@} +} + +function main() { + command="$1" + case $command in + "upgrade-index") + upgrade_index "${@:2}" + ;; + "run-check-index-tool") + check_index_tool "${@:2}" + ;; + "run-upgrade-index-tool") + upgrade_index_tool "${@:2}" + ;; + "help") + print_help + ;; + *) + echo "Available commands: (upgrade-index | run-check-index-tool | run-upgrade-index-tool | help)" + ;; + esac +} + +main ${1+"$@"}
