AMBARI-20808. Create schema requests to upgrade solr schema (do not change it externally) (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bff50bd3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bff50bd3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bff50bd3 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: bff50bd3258112b3f5815af3a8ba50da70da07d2 Parents: 346df92 Author: oleewere <[email protected]> Authored: Thu Apr 20 21:32:07 2017 +0200 Committer: oleewere <[email protected]> Committed: Thu Apr 20 21:34:39 2017 +0200 ---------------------------------------------------------------------- .../configurer/SolrCollectionConfigurer.java | 4 +- .../handler/AbstractSolrConfigHandler.java | 116 ++++++++++++++++ .../logsearch/handler/UpgradeSchemaHandler.java | 139 +++++++++++++++++++ .../handler/UploadConfigurationHandler.java | 136 +++++------------- 4 files changed, 293 insertions(+), 102 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/bff50bd3/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java index 2a4c790..2f56812 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/SolrCollectionConfigurer.java @@ -25,6 +25,7 @@ import org.apache.ambari.logsearch.handler.ACLHandler; import org.apache.ambari.logsearch.handler.CreateCollectionHandler; import org.apache.ambari.logsearch.handler.ListCollectionHandler; import org.apache.ambari.logsearch.handler.ReloadCollectionHandler; +import org.apache.ambari.logsearch.handler.UpgradeSchemaHandler; import org.apache.ambari.logsearch.handler.UploadConfigurationHandler; import org.apache.commons.lang.StringUtils; import org.apache.solr.client.solrj.impl.CloudSolrClient; @@ -103,7 +104,8 @@ public class SolrCollectionConfigurer implements Configurer { if (!state.isConfigurationUploaded()) { state.setConfigurationUploaded(true); } - return reloadCollectionNeeded; + boolean upgradeSchema = new UpgradeSchemaHandler(cloudSolrClient, configSetFolder).handle(cloudSolrClient, solrPropsConfig); + return reloadCollectionNeeded || upgradeSchema; } public boolean stopSetupCondition(SolrCollectionState state) { http://git-wip-us.apache.org/repos/asf/ambari/blob/bff50bd3/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/AbstractSolrConfigHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/AbstractSolrConfigHandler.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/AbstractSolrConfigHandler.java new file mode 100644 index 0000000..8693157 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/AbstractSolrConfigHandler.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.handler; + +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.commons.io.FileUtils; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkConfigManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Paths; +import java.util.UUID; + +public abstract class AbstractSolrConfigHandler implements SolrZkRequestHandler<Boolean> { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractSolrConfigHandler.class); + + private File configSetFolder; + + public AbstractSolrConfigHandler(File configSetFolder) { + this.configSetFolder = configSetFolder; + } + + @Override + public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception { + boolean reloadCollectionNeeded = false; + String separator = FileSystems.getDefault().getSeparator(); + String downloadFolderLocation = String.format("%s%s%s%s%s", System.getProperty("java.io.tmpdir"), separator, + UUID.randomUUID().toString(), separator, solrPropsConfig.getConfigName()); + solrClient.connect(); + SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient(); + File tmpDir = new File(downloadFolderLocation); + try { + ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient); + boolean configExists = zkConfigManager.configExists(solrPropsConfig.getConfigName()); + if (configExists) { + reloadCollectionNeeded = doIfConfigExists(solrPropsConfig, zkClient, separator, downloadFolderLocation, tmpDir); + } else { + doIfConfigNotExist(solrPropsConfig, zkConfigManager); + } + } catch (Exception e) { + throw new RuntimeException(String.format("Cannot upload configurations to zk. (collection: %s, config set folder: %s)", + solrPropsConfig.getCollection(), solrPropsConfig.getConfigSetFolder()), e); + } finally { + if (tmpDir.exists()) { + try { + FileUtils.deleteDirectory(tmpDir); + } catch (IOException e){ + LOG.error("Cannot delete temp directory.", e); + } + } + } + return reloadCollectionNeeded; + } + + /** + * Update config file (like solrconfig.xml) to zookeeper znode of solr, contains a download location as well which can be + * used to determine that you need to update the configuration or not + */ + public abstract boolean updateConfigIfNeeded(SolrPropsConfig solrPropsConfig, SolrZkClient zkClient, File file, + String separator, String downloadFolderLocation) throws IOException; + + /** + * Config file name which should be uploaded to zookeeper + */ + public abstract String getConfigFileName(); + + public void doIfConfigNotExist(SolrPropsConfig solrPropsConfig, ZkConfigManager zkConfigManager) throws IOException { + // Do nothing + }; + + public boolean doIfConfigExists(SolrPropsConfig solrPropsConfig, SolrZkClient zkClient, String separator, String downloadFolderLocation, File tmpDir) throws IOException { + boolean result = false; + LOG.info("Config set exists for '{}' collection. Refreshing it if needed...", solrPropsConfig.getCollection()); + if (!tmpDir.mkdirs()) { + LOG.error("Cannot create directories for '{}'", tmpDir.getAbsolutePath()); + } + ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient); + zkConfigManager.downloadConfigDir(solrPropsConfig.getConfigName(), Paths.get(downloadFolderLocation)); + File[] listOfFiles = getConfigSetFolder().listFiles(); + if (listOfFiles != null) { + for (File file : listOfFiles) { + if (file.getName().equals(getConfigFileName()) && updateConfigIfNeeded(solrPropsConfig, zkClient, file, separator, downloadFolderLocation)) { + result = true; + break; + } + } + } + return result; + } + + protected File getConfigSetFolder() { + return configSetFolder; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/bff50bd3/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UpgradeSchemaHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UpgradeSchemaHandler.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UpgradeSchemaHandler.java new file mode 100644 index 0000000..0da2125 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UpgradeSchemaHandler.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.handler; + +import org.apache.ambari.logsearch.conf.SolrPropsConfig; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.configuration.HierarchicalConfiguration.Node; +import org.apache.commons.configuration.XMLConfiguration; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.schema.SchemaRequest; +import org.apache.solr.common.cloud.SolrZkClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class UpgradeSchemaHandler extends AbstractSolrConfigHandler { + + private static final Logger LOG = LoggerFactory.getLogger(UpgradeSchemaHandler.class); + + private static final String SCHEMA_FILE = "managed-schema"; + private static final String FIELD_NAME_PATH = "field[@name]"; + private static final String FIELD_TYPE_NAME_PATH = "fieldType[@name]"; + private static final String DYNAMIC_FIELD = "dynamicField"; + private static final String DYNAMIC_FIELD_NAME_PATH = DYNAMIC_FIELD + "[@name]"; + + private CloudSolrClient cloudSolrClient; + + private XMLConfiguration localFileXml; + private List<String> localDynamicFields; + + public UpgradeSchemaHandler(CloudSolrClient cloudSolrClient, File configSetFolder) { + super(configSetFolder); + this.cloudSolrClient = cloudSolrClient; + } + + @Override + public boolean updateConfigIfNeeded(SolrPropsConfig solrPropsConfig, SolrZkClient zkClient, File file, String separator, String downloadFolderLocation) throws IOException { + boolean result = false; + if (localSchemaFileHasMoreFields(file, new File(String.format("%s%s%s", downloadFolderLocation, separator, file.getName())))) { + LOG.info("Solr schema file differs ('{}'), update config schema...", file.getName()); + try { + upgradeDynamicFields(); + } catch (Exception e) { + throw new RuntimeException(e); + } + result = true; + } + return result; + } + + // for now we only upgrades dynamic fields, later we can extend this feature if needed + private void upgradeDynamicFields() throws IOException, SolrServerException { + if (localFileXml.getRoot() != null && CollectionUtils.isNotEmpty(localDynamicFields)) { + List<Node> children = localFileXml.getRoot().getChildren(DYNAMIC_FIELD); + for (Node dynamicFieldNode : children) { + List<Node> attributes = dynamicFieldNode.getAttributes(); + Map<String, Object> attributesMap = new HashMap<>(); + for (Node attribute : attributes) { + attributesMap.put(attribute.getName(), attribute.getValue()); + } + if (attributesMap.get("name") != null && localDynamicFields.contains(attributesMap.get("name").toString())) { + SchemaRequest.AddDynamicField addDynamicFieldRequest = new SchemaRequest.AddDynamicField(attributesMap); + addDynamicFieldRequest.process(cloudSolrClient); + LOG.info("Added dynamic field request sent. (field name: {})", attributesMap.get("name")); + } + } + } + } + + @Override + public String getConfigFileName() { + return SCHEMA_FILE; + } + + private boolean localSchemaFileHasMoreFields(File localFile, File downloadedFile) { + try { + localFileXml = new XMLConfiguration(localFile); + XMLConfiguration downloadedFileXml = new XMLConfiguration(downloadedFile); + + List<String> localFieldNames = (ArrayList<String>) localFileXml.getProperty(FIELD_NAME_PATH); + List<String> localFieldTypes = (ArrayList<String>) localFileXml.getProperty(FIELD_TYPE_NAME_PATH); + localDynamicFields = (ArrayList<String>) localFileXml.getProperty(DYNAMIC_FIELD_NAME_PATH); + + List<String> fieldNames = (ArrayList<String>) downloadedFileXml.getProperty(FIELD_NAME_PATH); + List<String> fieldTypes = (ArrayList<String>) downloadedFileXml.getProperty(FIELD_TYPE_NAME_PATH); + List<String> dynamicFields = (ArrayList<String>) downloadedFileXml.getProperty(DYNAMIC_FIELD_NAME_PATH); + + boolean fieldNameHasDiff = hasMoreFields(localFieldNames, fieldNames, FIELD_NAME_PATH); + boolean fieldTypeHasDiff = hasMoreFields(localFieldTypes, fieldTypes, FIELD_TYPE_NAME_PATH); + boolean dynamicFieldNameHasDiff = hasMoreFields(localDynamicFields, dynamicFields, DYNAMIC_FIELD_NAME_PATH); + + return fieldNameHasDiff || fieldTypeHasDiff || dynamicFieldNameHasDiff; + } catch (Exception e) { + throw new RuntimeException("Exception during schema xml parsing.", e); + } + } + + private boolean hasMoreFields(List<String> localFields, List<String> fields, String tag) { + boolean result = false; + if (localFields != null) { + if (fields == null) { + result = true; + } else { + localFields.removeAll(fields); + if (!localFields.isEmpty()) { + result = true; + } + } + } + if (result) { + LOG.info("Found new fields or field types in local schema file.: {} ({})", localFields.toString(), tag); + } + return result; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/bff50bd3/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java index 27a6705..6964951 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/UploadConfigurationHandler.java @@ -19,133 +19,67 @@ package org.apache.ambari.logsearch.handler; import org.apache.ambari.logsearch.conf.SolrPropsConfig; -import org.apache.commons.configuration.XMLConfiguration; import org.apache.commons.io.FileUtils; -import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.commons.io.IOUtils; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkConfigManager; +import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; -import java.nio.file.FileSystems; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import java.io.InputStream; -public class UploadConfigurationHandler implements SolrZkRequestHandler<Boolean> { +public class UploadConfigurationHandler extends AbstractSolrConfigHandler { private static final Logger LOG = LoggerFactory.getLogger(UploadConfigurationHandler.class); - private static final String SCHEMA_FILE = "managed-schema"; private static final String SOLR_CONFIG_FILE = "solrconfig.xml"; - private static final String FIELD_NAME_PATH = "field[@name]"; - private static final String FIELD_TYPE_NAME_PATH = "fieldType[@name]"; - private static final String DYNAMIC_FIELD_NAME_PATH = "dynamicField[@name]"; - - private File configSetFolder; public UploadConfigurationHandler(File configSetFolder) { - this.configSetFolder = configSetFolder; + super(configSetFolder); } @Override - public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception { - boolean reloadCollectionNeeded = false; - String separator = FileSystems.getDefault().getSeparator(); - String downloadFolderLocation = String.format("%s%s%s%s%s", System.getProperty("java.io.tmpdir"), separator, - UUID.randomUUID().toString(), separator, solrPropsConfig.getConfigName()); - solrClient.connect(); - SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient(); - File tmpDir = new File(downloadFolderLocation); - try { + public boolean updateConfigIfNeeded(SolrPropsConfig solrPropsConfig, SolrZkClient zkClient, File file, + String separator, String downloadFolderLocation) throws IOException { + boolean result = false; + if (!FileUtils.contentEquals(file, new File(String.format("%s%s%s", downloadFolderLocation, separator, file.getName())))) { + LOG.info("Solr config file differs ('{}'), upload config set to zookeeper", file.getName()); ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient); - boolean configExists = zkConfigManager.configExists(solrPropsConfig.getConfigName()); - if (configExists) { - LOG.info("Config set exists for '{}' collection. Refreshing it if needed...", solrPropsConfig.getCollection()); - if (!tmpDir.mkdirs()) { - LOG.error("Cannot create directories for '{}'", tmpDir.getAbsolutePath()); - } - zkConfigManager.downloadConfigDir(solrPropsConfig.getConfigName(), Paths.get(downloadFolderLocation)); - File[] listOfFiles = configSetFolder.listFiles(); - if (listOfFiles != null) { - for (File file : listOfFiles) { - if (file.getName().equals(SOLR_CONFIG_FILE) && !FileUtils.contentEquals(file, new File(String.format("%s%s%s", downloadFolderLocation, separator, file.getName())))) { - LOG.info("Solr config file differs ('{}'), upload config set to zookeeper", file.getName()); - zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName()); - reloadCollectionNeeded = true; - break; - } - if (file.getName().equals(SCHEMA_FILE) && localSchemaFileHasMoreFields(file, new File(String.format("%s%s%s", downloadFolderLocation, separator, file.getName())))) { - LOG.info("Solr schema file differs ('{}'), upload config set to zookeeper", file.getName()); - zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName()); - reloadCollectionNeeded = true; - break; - } - } - } - } else { - LOG.info("Config set does not exist for '{}' collection. Uploading it to zookeeper...", solrPropsConfig.getCollection()); - File[] listOfFiles = configSetFolder.listFiles(); - if (listOfFiles != null) { - zkConfigManager.uploadConfigDir(configSetFolder.toPath(), solrPropsConfig.getConfigName()); + zkConfigManager.uploadConfigDir(getConfigSetFolder().toPath(), solrPropsConfig.getConfigName()); + String filePath = String.format("%s%s%s", getConfigSetFolder(), separator, getConfigFileName()); + String configsPath = String.format("/%s/%s/%s", "configs", solrPropsConfig.getConfigName(), getConfigFileName()); + InputStream is = new FileInputStream(filePath); + try { + if (zkClient.exists(configsPath, true)) { + zkClient.setData(configsPath, IOUtils.toByteArray(is), true); + } else { + zkClient.create(configsPath, IOUtils.toByteArray(is), CreateMode.PERSISTENT, true); } + } catch (Exception e) { + throw new IllegalStateException(e); } - } catch (Exception e) { - throw new RuntimeException(String.format("Cannot upload configurations to zk. (collection: %s, config set folder: %s)", - solrPropsConfig.getCollection(), solrPropsConfig.getConfigSetFolder()), e); - } finally { - if (tmpDir.exists()) { - try { - FileUtils.deleteDirectory(tmpDir); - } catch (IOException e){ - LOG.error("Cannot delete temp directory.", e); - } + finally { + IOUtils.closeQuietly(is); } + result = true; } - return reloadCollectionNeeded; + return result; } - private boolean localSchemaFileHasMoreFields(File localFile, File downloadedFile) { - try { - XMLConfiguration localFileXml = new XMLConfiguration(localFile); - XMLConfiguration downloadedFileXml = new XMLConfiguration(downloadedFile); - - List<String> localFieldNames = (ArrayList<String>) localFileXml.getProperty(FIELD_NAME_PATH); - List<String> localFieldTypes = (ArrayList<String>) localFileXml.getProperty(FIELD_TYPE_NAME_PATH); - List<String> localDynamicFields = (ArrayList<String>) localFileXml.getProperty(DYNAMIC_FIELD_NAME_PATH); - - List<String> fieldNames = (ArrayList<String>) downloadedFileXml.getProperty(FIELD_NAME_PATH); - List<String> fieldTypes = (ArrayList<String>) downloadedFileXml.getProperty(FIELD_TYPE_NAME_PATH); - List<String> dynamicFields = (ArrayList<String>) downloadedFileXml.getProperty(DYNAMIC_FIELD_NAME_PATH); - - boolean fieldNameHasDiff = hasMoreFields(localFieldNames, fieldNames, FIELD_NAME_PATH); - boolean fieldTypeHasDiff = hasMoreFields(localFieldTypes, fieldTypes, FIELD_TYPE_NAME_PATH); - boolean dynamicFieldNameHasDiff = hasMoreFields(localDynamicFields, dynamicFields, DYNAMIC_FIELD_NAME_PATH); - - return fieldNameHasDiff || fieldTypeHasDiff || dynamicFieldNameHasDiff; - } catch (Exception e) { - throw new RuntimeException("Exception during schema xml parsing.", e); + @Override + public void doIfConfigNotExist(SolrPropsConfig solrPropsConfig, ZkConfigManager zkConfigManager) throws IOException { + LOG.info("Config set does not exist for '{}' collection. Uploading it to zookeeper...", solrPropsConfig.getCollection()); + File[] listOfFiles = getConfigSetFolder().listFiles(); + if (listOfFiles != null) { + zkConfigManager.uploadConfigDir(getConfigSetFolder().toPath(), solrPropsConfig.getConfigName()); } } - private boolean hasMoreFields(List<String> localFields, List<String> fields, String tag) { - boolean result = false; - if (localFields != null) { - if (fields == null) { - result = true; - } else { - localFields.removeAll(fields); - if (!localFields.isEmpty()) { - result = true; - } - } - } - if (result) { - LOG.info("Found new fields or field types in local schema file.: {} ({})", localFields.toString(), tag); - } - return result; + @Override + public String getConfigFileName() { + return SOLR_CONFIG_FILE; } - }
