This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8adcdd8 AMBARI-23182. Infra Solr - add custom Ambari commands for
backup/migrate/restore index. (#596)
8adcdd8 is described below
commit 8adcdd89c1c3757ea16ce03ef18f1b6b091262e6
Author: Olivér Szabó <[email protected]>
AuthorDate: Sun Mar 11 23:23:57 2018 +0100
AMBARI-23182. Infra Solr - add custom Ambari commands for
backup/migrate/restore index. (#596)
---
.../src/main/resources/solrIndexHelper.sh | 5 ++
.../AMBARI_INFRA_SOLR/0.1.0/metainfo.xml | 35 +++++++++
.../0.1.0/package/scripts/collection.py | 75 +++++++++++++++++++
.../0.1.0/package/scripts/command_commons.py | 84 ++++++++++++++++++++++
.../0.1.0/package/scripts/infra_solr.py | 15 +++-
.../0.1.0/package/scripts/migrate.py | 55 ++++++++++++++
6 files changed, 268 insertions(+), 1 deletion(-)
diff --git
a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
index d47f071..4ba342a 100755
---
a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
+++
b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
@@ -68,6 +68,11 @@ function upgrade_core() {
verbose="-verbose"
fi
+ if [[ -f "$INDEX_DIR/write.lock" ]]; then
+ echo "Deleting $INDEX_DIR/write.lock file..."
+ rm "$INDEX_DIR/write.lock"
+ fi
+
for coll in $SOLR_CORE_FILTER_ARR; do
if [[ "$1" == *"$coll"* ]]; then
echo "$core_str '$1' dir name contains $coll (core filter)'";
diff --git
a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/metainfo.xml
index 68787bf..f5d58f7 100644
---
a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/metainfo.xml
+++
b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/metainfo.xml
@@ -42,6 +42,41 @@
<primary>true</primary>
</log>
</logs>
+ <customCommands>
+ <customCommand>
+ <name>BACKUP</name>
+ <commandScript>
+ <script>scripts/infra_solr.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </customCommand>
+ <customCommand>
+ <name>RESTORE</name>
+ <commandScript>
+ <script>scripts/infra_solr.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </customCommand>
+ <customCommand>
+ <name>MIGRATE</name>
+ <commandScript>
+ <script>scripts/infra_solr.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>36000</timeout>
+ <background>true</background>
+ </commandScript>
+ </customCommand>
+ <customCommand>
+ <name>DELETE</name>
+ <commandScript>
+ <script>scripts/infra_solr.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
<dependencies>
<dependency>
<name>AMBARI_INFRA_SOLR/INFRA_SOLR_CLIENT</name>
diff --git
a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py
b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py
new file mode 100644
index 0000000..0abcbc9
--- /dev/null
+++
b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/collection.py
@@ -0,0 +1,75 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Directory, Execute, File
+from resource_management.libraries.functions.format import format
+
+def backup_collection(env):
+ """
+ Backup collections using replication API (as Solr Cloud Backup API is not
available in Solr 5)
+ """
+ import params, command_commons
+ env.set_params(command_commons)
+
+ Logger.info(format("Backup Solr Collection {collection} to
{index_location}"))
+
+ solr_request_path =
format("{collection}/replication?command=BACKUP&location={index_location}&name={backup_name}&wt=json")
+ backup_api_cmd =
command_commons.create_solr_api_request_command(solr_request_path)
+
+ Directory(command_commons.index_location,
+ mode=0755,
+ cd_access='a',
+ owner=params.infra_solr_user,
+ group=params.user_group
+ )
+
+ Execute(backup_api_cmd, user=params.infra_solr_user, logoutput=True)
+
+def restore_collection(env):
+ """
+ Restore collections using replication API (as Solr Cloud Backup API is not
available in Solr 5)
+ """
+ import params, command_commons
+ env.set_params(command_commons)
+
+ Logger.info(format("Remove write.lock files from folder
'{index_location}'"))
+ for write_lock_file in
command_commons.get_files_by_pattern(format("{index_location}"), 'write.lock'):
+ File(write_lock_file, action="delete")
+
+ Logger.info(format("Restore Solr Collection {collection} from
{index_location}"))
+
+ solr_request_path =
format("{collection}/replication?command=RESTORE&location={index_location}&name={backup_name}&wt=json")
+ restore_api_cmd =
command_commons.create_solr_api_request_command(solr_request_path)
+
+ Execute(restore_api_cmd, user=params.infra_solr_user, logoutput=True)
+
+def delete_collection(env):
+ """
+ Delete specific Solr collection - used on ranger_audits by default
+ """
+ import params, command_commons
+ env.set_params(command_commons)
+
+ Logger.info(format("Delete Solr Collection: {collection}"))
+
+ solr_request_path =
format("admin/collections?action=DELETE&name={collection}&wt=json")
+ delete_api_cmd =
command_commons.create_solr_api_request_command(solr_request_path)
+
+ Execute(delete_api_cmd, user=params.infra_solr_user, logoutput=True)
\ No newline at end of file
diff --git
a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
new file mode 100644
index 0000000..7f0b3aa
--- /dev/null
+++
b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/command_commons.py
@@ -0,0 +1,84 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import fnmatch
+import os
+import params
+
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.format import format
+
+index_helper_script = '/usr/lib/ambari-infra-solr-client/solrIndexHelper.sh'
+
+# folder location which contains the snapshot/core folder
+index_location = default("/commandParams/solr_index_location", None)
+
+# index version (available index versions: 6.6.2 and 7.2.1, second one is used
by default)
+index_version = default("/commandParams/solr_index_version", '6.6.2')
+
+# if this flag is false, skip upgrade if the version is proper, you can force
to re-run the tool with setting the flag to true
+force = default("/commandParams/solr_index_upgrade_force", False)
+
+# if this flag is true, then it will generate specific folder for every backup
with a hostname suffix
+# where "." chars replaced with "_"(e.g.:
/my/path/backup_locationc7301_ambari_apache_org), that can be useful if
different
+# hosts share the same filesystem where the backup is stored.
+shared_fs = default("/commandParams/solr_shared_fs", False)
+
+# set verbose log for index migration (default: true)
+debug = default("/commandParams/solr_migrate_debug", True)
+
+# used for filtering folders in backup location (like: if the filter is
ranger, that will include snapshot.ranger folder but won't include
snapshot.hadoop_logs)
+core_filter = default("/commandParams/solr_core_filter", None)
+
+# delete write.lock file at the start of lucene index migration process
+delete_lock_on_start = default("/commandParams/solr_delete_lock_on_start",
True)
+# if it used, then core filter will be used with snapshot.* folder pattern
+backup_mode = default("/commandParams/solr_migrate_backup", True)
+
+log_output = default("/commandParams/solr_migrate_logoutput", True)
+# Solr colleection name (used for DELETE/BACKUP/RESTORE)
+collection = default("/commandParams/solr_collection", "ranger_audits")
+# it will be used in the snapshot name, if it's ranger, the snapshot folder
will be snapshot.ranger
+backup_name = default("/commandParams/solr_backup_name", "ranger")
+
+solr_protocol = "https" if params.infra_solr_ssl_enabled else "http"
+solr_base_url =
format("{solr_protocol}://{params.hostname}:{params.infra_solr_port}/solr")
+
+keytab = params.infra_solr_kerberos_keytab
+principal = params.infra_solr_kerberos_principal
+
+hostname_suffix = params.hostname.replace(".", "_")
+
+if shared_fs:
+ index_location = format("{index_location}_{hostname_suffix}")
+
+
+def get_files_by_pattern(directory, pattern):
+ for root, dirs, files in os.walk(directory):
+ for basename in files:
+ try:
+ matched = pattern.match(basename)
+ except AttributeError:
+ matched = fnmatch.fnmatch(basename, pattern)
+ if matched:
+ yield os.path.join(root, basename)
+
+def create_solr_api_request_command(request_path):
+ solr_url = format("{solr_base_url}/{request_path}")
+ api_cmd = format("kinit -kt {keytab} {principal} && curl -k --negotiate -u :
'{solr_url}'") if params.security_enabled else format("curl -k '{solr_url}'")
+ return api_cmd
diff --git
a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py
b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py
index 84f5645..54f3713 100644
---
a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py
+++
b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/infra_solr.py
@@ -27,9 +27,10 @@ from
resource_management.libraries.functions.get_user_call_output import get_use
from resource_management.libraries.functions.show_logs import show_logs
from resource_management.libraries.script.script import Script
+from collection import backup_collection, restore_collection, delete_collection
+from migrate import migrate_index
from setup_infra_solr import setup_infra_solr
-
class InfraSolr(Script):
def install(self, env):
import params
@@ -122,5 +123,17 @@ class InfraSolr(Script):
user=params.infra_solr_user)
zkmigrator.set_acls(params.infra_solr_znode, 'world:anyone:crdwa')
+ def backup(self, env):
+ backup_collection(env)
+
+ def restore(self, env):
+ restore_collection(env)
+
+ def migrate(self, env):
+ migrate_index(env)
+
+ def delete(self, env):
+ delete_collection(env)
+
if __name__ == "__main__":
InfraSolr().execute()
\ No newline at end of file
diff --git
a/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/migrate.py
b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/migrate.py
new file mode 100644
index 0000000..1ff7d95
--- /dev/null
+++
b/ambari-server/src/main/resources/common-services/AMBARI_INFRA_SOLR/0.1.0/package/scripts/migrate.py
@@ -0,0 +1,55 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+from resource_management.libraries.functions.format import format
+
+def migrate_index(env):
+ """
+ Migrate lucene index in the background.
+ """
+ import params, command_commons
+ env.set_params(command_commons)
+
+ index_migrate_cmd = format("{index_helper_script} upgrade-index -d
{index_location} -v {index_version}")
+
+ if command_commons.force is True:
+ index_migrate_cmd+=" -f"
+
+ if command_commons.backup_mode is True:
+ index_migrate_cmd+=" -b"
+
+ if command_commons.debug is True:
+ index_migrate_cmd+=" -g"
+
+ if command_commons.core_filter is not None:
+ index_migrate_cmd+=format(" -c {core_filter}")
+
+ if command_commons.delete_lock_on_start:
+ Logger.info(format("Remove write.lock files from folder
'{index_location}'"))
+ for write_lock_file in
command_commons.get_files_by_pattern(format("{index_location}"), 'write.lock'):
+ File(write_lock_file, action="delete")
+ else:
+ Logger.info("Skip removing write.lock files")
+
+ Logger.info(format("Migrate index at location: {index_location}"))
+ # It can generate a write.lock file
+ Execute(index_migrate_cmd, user=params.infra_solr_user,
environment={'JAVA_HOME': params.java64_home},
logoutput=command_commons.log_output)
+
--
To stop receiving notification emails like this one, please contact
[email protected].
