amit-jain commented on code in PR #2689: URL: https://github.com/apache/jackrabbit-oak/pull/2689#discussion_r2757216991
########## oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java: ########## @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexName; +import org.apache.jackrabbit.oak.plugins.tree.TreeConstants; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Processing of diff indexes, that is nodes under "/oak:index/diff.index". A + * diff index contains differences to existing indexes, and possibly new + * (custom) indexes in the form of JSON. These changes can then be merged + * (applied) to the index definitions. This allows to simplify index management, + * because it allows to modify (add, update) indexes in a simple way. + */ +public class DiffIndex { + + private static final Logger LOG = LoggerFactory.getLogger(DiffIndex.class); + + /** + * Apply changes to the index definitions. That means merge the index diff with + * the existing indexes, creating new index versions. It might also mean to + * remove old (merged) indexes if the diff no longer contains them. + * + * @param store the node store + * @param indexDefinitions the /oak:index node + */ + public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefinitions) { + JsonObject newImageLuceneDefinitions = null; + for (String diffIndex : new String[] { DiffIndexMerger.DIFF_INDEX, DiffIndexMerger.DIFF_INDEX_OPTIMIZER }) { + if (!indexDefinitions.hasChildNode(diffIndex)) { + continue; + } + NodeBuilder diffIndexDefinition = indexDefinitions.child(diffIndex); + NodeBuilder diffContent = diffIndexDefinition.getChildNode("diff.json").getChildNode("jcr:content"); + if (!diffContent.exists()) { + continue; + } + PropertyState lastMod = diffContent.getProperty("jcr:lastModified"); + if (lastMod == null) { + continue; + } + String modified = lastMod.getValue(Type.DATE); + PropertyState lastProcessed = diffContent.getProperty(":lastProcessed"); + if (lastProcessed != null) { + if (modified.equals(lastProcessed.getValue(Type.STRING))) { + // already processed + continue; + } + } + // store now, so a change is only processed once + diffContent.setProperty(":lastProcessed", modified); + PropertyState jcrData = diffContent.getProperty("jcr:data"); + String diff = tryReadString(jcrData); + if (diff == null) { + continue; + } + try { + JsonObject diffObj = JsonObject.fromJson("{\"diff\": " + diff + "}", true); + diffIndexDefinition.removeProperty("error"); + if (newImageLuceneDefinitions == null) { + newImageLuceneDefinitions = new JsonObject(); + } + newImageLuceneDefinitions.getChildren().put("/oak:index/" + diffIndex, diffObj); + } catch (Exception e) { + String message = "Error parsing " + diffIndex; + LOG.warn("{}: {}", message, e.getMessage(), e); + diffIndexDefinition.setProperty("error", message + ": " + e.getMessage()); + } + } + if (newImageLuceneDefinitions == null) { + // not a valid diff index, or already processed + return; + } + LOG.info("Processing a new diff.index with node store {}", store); + JsonObject repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(indexDefinitions); + LOG.debug("Index list {}", repositoryDefinitions.toString()); + try { + DiffIndexMerger.instance().merge(newImageLuceneDefinitions, repositoryDefinitions, store); + for (String indexPath : newImageLuceneDefinitions.getChildren().keySet()) { + if (indexPath.startsWith("/oak:index/" + DiffIndexMerger.DIFF_INDEX)) { + continue; + } + JsonObject newDef = newImageLuceneDefinitions.getChildren().get(indexPath); + String indexName = PathUtils.getName(indexPath); + JsonNodeBuilder.addOrReplace(indexDefinitions, store, indexName, IndexConstants.INDEX_DEFINITIONS_NODE_TYPE, newDef.toString()); + updateNodetypeIndexForPath(indexDefinitions, indexName, true); + disableOrRemoveOldVersions(indexDefinitions, indexPath, indexName); + } + removeDisabledMergedIndexes(indexDefinitions); + sortIndexes(indexDefinitions); + } catch (Exception e) { + LOG.warn("Error merging diff.index: {}", e.getMessage(), e); + NodeBuilder diffIndexDefinition = indexDefinitions.child(DiffIndexMerger.DIFF_INDEX); + diffIndexDefinition.setProperty("error", e.getMessage()); + } + } + + /** + * Try to read a text from the (binary) jcr:data property. Edge cases such as + * "property does not exist" and IO exceptions (blob not found) do not throw an + * exception (IO exceptions are logged). + * + * @param jcrData the "jcr:data" property + * @return the string, or null if reading fails + */ + public static String tryReadString(PropertyState jcrData) { + if (jcrData == null) { + return null; + } + InputStream in = jcrData.getValue(Type.BINARY).getNewStream(); + try { Review Comment: ```suggestion try (InputStream in = jcrData.getValue(Type.BINARY).getNewStream()) { ``` ########## oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java: ########## @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexName; +import org.apache.jackrabbit.oak.plugins.tree.TreeConstants; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Processing of diff indexes, that is nodes under "/oak:index/diff.index". A + * diff index contains differences to existing indexes, and possibly new + * (custom) indexes in the form of JSON. These changes can then be merged + * (applied) to the index definitions. This allows to simplify index management, + * because it allows to modify (add, update) indexes in a simple way. + */ +public class DiffIndex { + + private static final Logger LOG = LoggerFactory.getLogger(DiffIndex.class); + + /** + * Apply changes to the index definitions. That means merge the index diff with + * the existing indexes, creating new index versions. It might also mean to + * remove old (merged) indexes if the diff no longer contains them. + * + * @param store the node store + * @param indexDefinitions the /oak:index node + */ + public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefinitions) { + JsonObject newImageLuceneDefinitions = null; + for (String diffIndex : new String[] { DiffIndexMerger.DIFF_INDEX, DiffIndexMerger.DIFF_INDEX_OPTIMIZER }) { + if (!indexDefinitions.hasChildNode(diffIndex)) { + continue; + } + NodeBuilder diffIndexDefinition = indexDefinitions.child(diffIndex); + NodeBuilder diffContent = diffIndexDefinition.getChildNode("diff.json").getChildNode("jcr:content"); + if (!diffContent.exists()) { + continue; + } + PropertyState lastMod = diffContent.getProperty("jcr:lastModified"); + if (lastMod == null) { + continue; + } + String modified = lastMod.getValue(Type.DATE); + PropertyState lastProcessed = diffContent.getProperty(":lastProcessed"); + if (lastProcessed != null) { + if (modified.equals(lastProcessed.getValue(Type.STRING))) { + // already processed + continue; + } + } + // store now, so a change is only processed once + diffContent.setProperty(":lastProcessed", modified); + PropertyState jcrData = diffContent.getProperty("jcr:data"); + String diff = tryReadString(jcrData); + if (diff == null) { + continue; + } + try { + JsonObject diffObj = JsonObject.fromJson("{\"diff\": " + diff + "}", true); + diffIndexDefinition.removeProperty("error"); + if (newImageLuceneDefinitions == null) { + newImageLuceneDefinitions = new JsonObject(); + } + newImageLuceneDefinitions.getChildren().put("/oak:index/" + diffIndex, diffObj); + } catch (Exception e) { + String message = "Error parsing " + diffIndex; + LOG.warn("{}: {}", message, e.getMessage(), e); + diffIndexDefinition.setProperty("error", message + ": " + e.getMessage()); + } + } + if (newImageLuceneDefinitions == null) { + // not a valid diff index, or already processed + return; + } + LOG.info("Processing a new diff.index with node store {}", store); + JsonObject repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(indexDefinitions); + LOG.debug("Index list {}", repositoryDefinitions.toString()); Review Comment: ```suggestion LOG.debug("Index list {}", repositoryDefinitions); ``` ########## oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java: ########## @@ -0,0 +1,833 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.jackrabbit.oak.commons.StringUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.commons.json.JsopBuilder; +import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; +import org.apache.jackrabbit.oak.json.Base64BlobSerializer; +import org.apache.jackrabbit.oak.json.JsonSerializer; +import org.apache.jackrabbit.oak.plugins.index.IndexName; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Index definition merge utility that uses the "diff" mode. + */ +public class DiffIndexMerger { + + private final static Logger LOG = LoggerFactory.getLogger(DiffIndexMerger.class); + + public final static String DIFF_INDEX = "diff.index"; + public final static String DIFF_INDEX_OPTIMIZER = "diff.index.optimizer"; + + private final static String MERGE_INFO = "This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html";; + + // the list of unsupported included paths, e.g. "/apps,/libs" + // by default all paths are supported + private final static String[] UNSUPPORTED_INCLUDED_PATHS = System.getProperty("oak.diffIndex.unsupportedPaths", "").split(","); + + // in case a custom index is removed, whether a dummy index is created + private final static boolean DELETE_CREATES_DUMMY = Boolean.getBoolean("oak.diffIndex.deleteCreatesDummy"); + + // in case a customization was removed, create a copy of the OOTB index + private final static boolean DELETE_COPIES_OOTB = Boolean.getBoolean("oak.diffIndex.deleteCopiesOOTB"); + + // whether to log at info level + private final static boolean LOG_AT_INFO_LEVEL = Boolean.getBoolean("oak.diffIndex.logAtInfoLevel"); + + private final String[] unsupportedIncludedPaths; + private final boolean deleteCreatesDummyIndex; + private final boolean deleteCopiesOutOfTheBoxIndex; + private final boolean logAtInfoLevel; + + static final DiffIndexMerger INSTANCE = new DiffIndexMerger(UNSUPPORTED_INCLUDED_PATHS, + DELETE_CREATES_DUMMY, DELETE_COPIES_OOTB, LOG_AT_INFO_LEVEL); + + public static DiffIndexMerger instance() { + return INSTANCE; + } + + DiffIndexMerger(String[] unsupportedIncludedPaths, + boolean deleteCreatesDummyIndex, boolean deleteCopiesOutOfTheBoxIndex, + boolean logAtInfoLevel) { + this.unsupportedIncludedPaths = unsupportedIncludedPaths; + this.deleteCreatesDummyIndex = deleteCreatesDummyIndex; + this.deleteCopiesOutOfTheBoxIndex = deleteCopiesOutOfTheBoxIndex; + this.logAtInfoLevel = logAtInfoLevel; + } + + /** + * If there is a diff index, that is an index with prefix "diff.", then try to merge it. + * + * @param newImageLuceneDefinitions + * the new indexes + * (input and output) + * @param repositoryDefinitions + * the indexes in the writable repository + * (input) + * @param repositoryNodeStore + */ + public void merge(JsonObject newImageLuceneDefinitions, JsonObject repositoryDefinitions, NodeStore repositoryNodeStore) { + // combine all definitions into one object + JsonObject combined = new JsonObject(); + + // index definitions in the repository + combined.getChildren().putAll(repositoryDefinitions.getChildren()); + + // read the diff.index.optimizer explicitly, + // because it's a not a regular index definition, + // and so in the repositoryDefinitions + if (repositoryNodeStore != null) { + Map<String, JsonObject> diffInRepo = readDiffIndex(repositoryNodeStore, DIFF_INDEX_OPTIMIZER); + combined.getChildren().putAll(diffInRepo); + } + + // overwrite with the provided definitions (if any) + combined.getChildren().putAll(newImageLuceneDefinitions.getChildren()); + + // check if there "diff.index" or "diff.index.optimizer" + boolean found = combined.getChildren().containsKey("/oak:index/" + DIFF_INDEX) + || combined.getChildren().containsKey("/oak:index/" + DIFF_INDEX_OPTIMIZER); + if (!found) { + // early exit, so that the risk of merging the PR + // is very small for customers that do not use this + log("No 'diff.index' definition"); + return; + } + mergeDiff(newImageLuceneDefinitions, combined); + } + + /** + * If there is a diff index (hardcoded node "/oak:index/diff.index" or + * "/oak:index/diff.index.optimizer"), then iterate over all entries and create new + * (merged) versions if needed. + * + * @param newImageLuceneDefinitions + * the new Lucene definitions + * (input + output) + * @param combined + * the definitions in the repository, + * including the one in the customer repo and new ones + * (input) + * @return whether a new version of an index was added + */ + boolean mergeDiff(JsonObject newImageLuceneDefinitions, JsonObject combined) { + // iterate again, this time process + + // collect the diff index(es) + HashMap<String, JsonObject> toProcess = new HashMap<>(); + tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX, toProcess); + tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX_OPTIMIZER, toProcess); + // if the diff index exists, but doesn't contain some of the previous indexes + // (indexes with mergeInfo), then we need to disable those (using /dummy includedPath) + extractExistingMergedIndexes(combined, toProcess); + if (toProcess.isEmpty()) { + log("No diff index definitions found."); + return false; + } + boolean hasChanges = false; + for (Entry<String, JsonObject> e : toProcess.entrySet()) { + String key = e.getKey(); + JsonObject value = e.getValue(); + if (key.startsWith("/oak:index/")) { + LOG.warn("The key should contains just the index name, without the '/oak:index' prefix for key {}", key); + key = key.substring("/oak:index/".length()); + } + log("Processing {}", key); + hasChanges |= processMerge(key, value, newImageLuceneDefinitions, combined); + } + return hasChanges; + } + + /** + * Extract a "diff.index" from the set of index definitions (if found), and if + * found, store the nested entries in the target map, merging them with previous + * entries if found. + * + * The diff.index may either have a file (a "jcr:content" child node with a + * "jcr:data" property), or a "diff" JSON object. For customers (in the git + * repository), the file is much easier to construct, but when running the + * indexing job, the nested JSON is much easier. + * + * @param indexDefs the set of index definitions (may be empty) + * @param name the name of the diff.index (either diff.index or + * diff.index.optimizer) + * @param target the target map of diff.index definitions + * @return the error message trying to parse the JSON file, or null + */ + public static String tryExtractDiffIndex(JsonObject indexDefs, String name, HashMap<String, JsonObject> target) { + JsonObject diffIndex = indexDefs.getChildren().get(name); + if (diffIndex == null) { + return null; + } + // extract either the file, or the nested json + JsonObject file = diffIndex.getChildren().get("diff.json"); + JsonObject diff; + if (file != null) { + // file + JsonObject jcrContent = file.getChildren().get("jcr:content"); + if (jcrContent == null) { + String message = "jcr:content child node is missing in diff.json"; + LOG.warn(message); + return message; + } + String jcrData = JsonNodeBuilder.oakStringValue(jcrContent, "jcr:data"); + try { + diff = JsonObject.fromJson(jcrData, true); + } catch (Exception e) { + LOG.warn("Illegal Json, ignoring: {}", jcrData, e); + String message = "Illegal Json, ignoring: " + e.getMessage(); + return message; + } + } else { + // nested json + diff = diffIndex.getChildren().get("diff"); + } + // store, if not empty + if (diff != null) { + for (Entry<String, JsonObject> e : diff.getChildren().entrySet()) { + String key = e.getKey(); + target.put(key, mergeDiffs(target.get(key), e.getValue())); + } + } + return null; + } + + /** + * Extract the indexes with a "mergeInfo" property and store them in the target + * object. This is needed so that indexes that were removed from the index.diff + * are detected (a new version is needed in this case with includedPaths + * "/dummy"). + * + * @param indexDefs the index definitions in the repository + * @param target the target map of "diff.index" definitions. for each entry + * found, an empty object is added + */ + private static void extractExistingMergedIndexes(JsonObject indexDefs, HashMap<String, JsonObject> target) { + for (Entry<String, JsonObject> e : indexDefs.getChildren().entrySet()) { + String key = e.getKey(); + JsonObject value = e.getValue(); + if (key.indexOf("-custom-") < 0 || !value.getProperties().containsKey("mergeInfo")) { + continue; + } + String baseName = IndexName.parse(key.substring("/oak:index/".length())).getBaseName(); + if (!target.containsKey(baseName)) { + // if there is no entry yet for this key, + // add a new empty object + target.put(baseName, new JsonObject()); + } + } + } + + /** + * Merge diff from "diff.index" and "diff.index.optimizer". + * The customer can define a diff (stored in "diff.index") + * and someone else (or the optimizer) can define one (stored in "diff.index.optimizer"). + * + * @param a the first diff + * @param b the second diff (overwrites entries in a) + * @return the merged entry + */ + public static JsonObject mergeDiffs(JsonObject a, JsonObject b) { + if (a == null) { + return b; + } else if (b == null) { + return a; + } + JsonObject result = JsonObject.fromJson(a.toString(), true); + result.getProperties().putAll(b.getProperties()); + HashSet<String> both = new HashSet<>(a.getChildren().keySet()); + both.addAll(b.getChildren().keySet()); + for (String k : both) { + result.getChildren().put(k, mergeDiffs(a.getChildren().get(k), b.getChildren().get(k))); + } + return result; + } + + /** + * Merge using the diff definition. + * + * If the latest customized index already matches, then + * newImageLuceneDefinitions will remain as is. Otherwise, a new customized + * index is added, with a "mergeInfo" property. + * + * Existing properties are never changed; only new properties/children are + * added. + * + * @param indexName the name, eg. "damAssetLucene" + * @param indexDiff the diff with the new properties + * @param newImageLuceneDefinitions the new Lucene definitions (input + output) + * @param combined the definitions in the repository, including + * the one in the customer repo and new ones + * (input) + * @return whether a new version of an index was added + */ + public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject newImageLuceneDefinitions, JsonObject combined) { + // extract the latest product index (eg. damAssetLucene-12) + // and customized index (eg. damAssetLucene-12-custom-3) - if any + IndexName latestProduct = null; + String latestProductKey = null; + IndexName latestCustomized = null; + String latestCustomizedKey = null; + String prefix = "/oak:index/"; + for (String key : combined.getChildren().keySet()) { + IndexName name = IndexName.parse(key.substring(prefix.length())); + if (!name.isVersioned()) { + log("Ignoring unversioned index {}", name); + continue; + } + if (!name.getBaseName().equals(indexName)) { + continue; + } + boolean isCustom = key.indexOf("-custom-") >= 0; + if (isCustom) { + if (latestCustomized == null || + name.compareTo(latestCustomized) > 0) { + latestCustomized = name; + latestCustomizedKey = key; + } + } else { + if (latestProduct == null || + name.compareTo(latestProduct) > 0) { + latestProduct = name; + latestProductKey = key; + } + } + } + log("Latest product: {}", latestProductKey); + log("Latest customized: {}", latestCustomizedKey); + if (latestProduct == null) { + if (indexName.indexOf('.') >= 0) { + // a fully custom index needs to contains a dot + log("Fully custom index {}", indexName); + } else { + log("No product version for {}", indexName); + return false; + } + } + JsonObject latestProductIndex = combined.getChildren().get(latestProductKey); + String[] includedPaths; + if (latestProductIndex == null) { + if (indexDiff.getProperties().isEmpty() && indexDiff.getChildren().isEmpty()) { + // there is no customization (any more), which means a dummy index may be needed + log("No customization for {}", indexName); + } else { + includedPaths = JsonNodeBuilder.oakStringArrayValue(indexDiff, "includedPaths"); + if (includesUnsupportedPaths(includedPaths)) { + LOG.warn("New custom index {} is not supported because it contains an unsupported path ({})", + indexName, Arrays.toString(unsupportedIncludedPaths)); + return false; + } + } + } else { + includedPaths = JsonNodeBuilder.oakStringArrayValue(latestProductIndex, "includedPaths"); + if (includesUnsupportedPaths(includedPaths)) { + LOG.warn("Customizing index {} is not supported because it contains an unsupported path ({})", + latestProductKey, Arrays.toString(unsupportedIncludedPaths)); + return false; + } + } + + // merge + JsonObject merged = null; + if (indexDiff == null) { + // no diff definition: use to the OOTB index + if (latestCustomized == null) { + log("Only a product index found, nothing to do"); + return false; + } + merged = latestProductIndex; + } else { + merged = processMerge(latestProductIndex, indexDiff); + } + + // compare to the latest version of the this index + JsonObject latestIndexVersion = new JsonObject(); + if (latestCustomized == null) { + latestIndexVersion = latestProductIndex; + } else { + latestIndexVersion = combined.getChildren().get(latestCustomizedKey); + } + JsonObject mergedDef = cleanedAndNormalized(switchToLucene(merged)); + // compute merge checksum for later, but do not yet add + String mergeChecksum = computeMergeChecksum(mergedDef); + // get the merge checksum before cleaning (cleaning removes it) - if available + String key; + if (latestIndexVersion == null) { + // new index + key = prefix + indexName + "-1-custom-1"; + } else { + String latestMergeChecksum = JsonNodeBuilder.oakStringValue(latestIndexVersion, "mergeChecksum"); + JsonObject latestDef = cleanedAndNormalized(switchToLucene(latestIndexVersion)); + if (isSameIgnorePropertyOrder(mergedDef, latestDef)) { + // normal case: no change + // (even if checksums do not match: checksums might be missing or manipulated) + log("Latest index matches"); + if (latestMergeChecksum != null && !latestMergeChecksum.equals(mergeChecksum)) { + LOG.warn("Indexes do match, but checksums do not. Possibly checksum was changed: {} vs {}", latestMergeChecksum, mergeChecksum); + LOG.warn("latest: {}\nmerged: {}", latestDef, mergedDef); + } + return false; + } + if (latestMergeChecksum != null && latestMergeChecksum.equals(mergeChecksum)) { + // checksum matches, but data does not match + // could be eg. due to numbers formatting issues (-0.0 vs 0.0, 0.001 vs 1e-3) + // but unexpected because we do not normally have such cases + LOG.warn("Indexes do not match, but checksums match. Possible normalization issue."); + LOG.warn("Index: {}, latest: {}\nmerged: {}", indexName, latestDef, mergedDef); + // if checksums match, we consider it a match + return false; + } + LOG.info("Indexes do not match, with"); + LOG.info("Index: {}, latest: {}\nmerged: {}", indexName, latestDef, mergedDef); + // a new merged index definition + if (latestProduct == null) { + // fully custom index: increment version + key = prefix + indexName + + "-" + latestCustomized.getProductVersion() + + "-custom-" + (latestCustomized.getCustomerVersion() + 1); + } else { + // customized OOTB index: use the latest product as the base + key = prefix + indexName + + "-" + latestProduct.getProductVersion() + + "-custom-"; + if (latestCustomized != null) { + key += (latestCustomized.getCustomerVersion() + 1); + } else { + key += "1"; + } + } + } + merged.getProperties().put("mergeInfo", JsopBuilder.encode(MERGE_INFO)); + merged.getProperties().put("mergeChecksum", JsopBuilder.encode(mergeChecksum)); + merged.getProperties().put("merges", "[" + JsopBuilder.encode("/oak:index/" + indexName) + "]"); + merged.getProperties().remove("reindexCount"); + merged.getProperties().remove("reindex"); + if (!deleteCopiesOutOfTheBoxIndex && indexDiff.toString().equals("{}")) { + merged.getProperties().put("type", "\"disabled\""); + merged.getProperties().put("mergeComment", "\"This index is superseeded and can be removed\""); + } + newImageLuceneDefinitions.getChildren().put(key, merged); + return true; + } + + /** + * Check whether the includedPaths covers unsupported paths, + * if there are any unsupported path (eg. "/apps" or "/libs"). + * In this case, simplified index management is not supported. + * + * @param includedPaths the includedPaths list + * @return true if any unsupported path is included + */ + public boolean includesUnsupportedPaths(String[] includedPaths) { + if (unsupportedIncludedPaths.length == 1 && "".equals(unsupportedIncludedPaths[0])) { + // set to an empty string + return false; + } + if (includedPaths == null) { + // not set means all entries + return true; + } + for (String path : includedPaths) { + if ("/".equals(path)) { + // all + return true; + } + for (String unsupported : unsupportedIncludedPaths) { + if (unsupported.isEmpty()) { + continue; + } + if (path.equals(unsupported) || path.startsWith(unsupported + "/")) { + // includedPaths matches, or starts with an unsupported path + return true; + } + } + } + return false; + } + + /** + * Compute the SHA-256 checksum of the JSON object. This is useful to detect + * that the JSON object was not "significantly" changed, even if stored + * somewhere and later read again. Insignificant changes include: rounding of + * floating point numbers, re-ordering properties, things like that. Without the + * checksum, we would risk creating a new version of a customized index each + * time the indexing job is run, even thought the customer didn't change + * anything. + * + * @param json the input + * @return the SHA-256 checksum + */ + private static String computeMergeChecksum(JsonObject json) { + byte[] bytes = json.toString().getBytes(StandardCharsets.UTF_8); + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + return StringUtils.convertBytesToHex(md.digest(bytes)); + } catch (NoSuchAlgorithmException e) { + // SHA-256 is guaranteed to be available in standard Java platforms + throw new RuntimeException("SHA-256 algorithm not available", e); Review Comment: Maybe IllegalStateException is better? ########## oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java: ########## @@ -0,0 +1,833 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.jackrabbit.oak.commons.StringUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.commons.json.JsopBuilder; +import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; +import org.apache.jackrabbit.oak.json.Base64BlobSerializer; +import org.apache.jackrabbit.oak.json.JsonSerializer; +import org.apache.jackrabbit.oak.plugins.index.IndexName; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Index definition merge utility that uses the "diff" mode. + */ +public class DiffIndexMerger { + + private final static Logger LOG = LoggerFactory.getLogger(DiffIndexMerger.class); + + public final static String DIFF_INDEX = "diff.index"; + public final static String DIFF_INDEX_OPTIMIZER = "diff.index.optimizer"; + + private final static String MERGE_INFO = "This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html";; + + // the list of unsupported included paths, e.g. "/apps,/libs" + // by default all paths are supported + private final static String[] UNSUPPORTED_INCLUDED_PATHS = System.getProperty("oak.diffIndex.unsupportedPaths", "").split(","); + + // in case a custom index is removed, whether a dummy index is created + private final static boolean DELETE_CREATES_DUMMY = Boolean.getBoolean("oak.diffIndex.deleteCreatesDummy"); + + // in case a customization was removed, create a copy of the OOTB index + private final static boolean DELETE_COPIES_OOTB = Boolean.getBoolean("oak.diffIndex.deleteCopiesOOTB"); + + // whether to log at info level + private final static boolean LOG_AT_INFO_LEVEL = Boolean.getBoolean("oak.diffIndex.logAtInfoLevel"); + + private final String[] unsupportedIncludedPaths; + private final boolean deleteCreatesDummyIndex; + private final boolean deleteCopiesOutOfTheBoxIndex; + private final boolean logAtInfoLevel; + + static final DiffIndexMerger INSTANCE = new DiffIndexMerger(UNSUPPORTED_INCLUDED_PATHS, + DELETE_CREATES_DUMMY, DELETE_COPIES_OOTB, LOG_AT_INFO_LEVEL); + + public static DiffIndexMerger instance() { + return INSTANCE; + } + + DiffIndexMerger(String[] unsupportedIncludedPaths, + boolean deleteCreatesDummyIndex, boolean deleteCopiesOutOfTheBoxIndex, + boolean logAtInfoLevel) { + this.unsupportedIncludedPaths = unsupportedIncludedPaths; + this.deleteCreatesDummyIndex = deleteCreatesDummyIndex; + this.deleteCopiesOutOfTheBoxIndex = deleteCopiesOutOfTheBoxIndex; + this.logAtInfoLevel = logAtInfoLevel; + } + + /** + * If there is a diff index, that is an index with prefix "diff.", then try to merge it. + * + * @param newImageLuceneDefinitions + * the new indexes + * (input and output) + * @param repositoryDefinitions + * the indexes in the writable repository + * (input) + * @param repositoryNodeStore + */ + public void merge(JsonObject newImageLuceneDefinitions, JsonObject repositoryDefinitions, NodeStore repositoryNodeStore) { + // combine all definitions into one object + JsonObject combined = new JsonObject(); + + // index definitions in the repository + combined.getChildren().putAll(repositoryDefinitions.getChildren()); + + // read the diff.index.optimizer explicitly, + // because it's a not a regular index definition, + // and so in the repositoryDefinitions + if (repositoryNodeStore != null) { + Map<String, JsonObject> diffInRepo = readDiffIndex(repositoryNodeStore, DIFF_INDEX_OPTIMIZER); + combined.getChildren().putAll(diffInRepo); + } + + // overwrite with the provided definitions (if any) + combined.getChildren().putAll(newImageLuceneDefinitions.getChildren()); + + // check if there "diff.index" or "diff.index.optimizer" + boolean found = combined.getChildren().containsKey("/oak:index/" + DIFF_INDEX) + || combined.getChildren().containsKey("/oak:index/" + DIFF_INDEX_OPTIMIZER); + if (!found) { + // early exit, so that the risk of merging the PR + // is very small for customers that do not use this + log("No 'diff.index' definition"); + return; + } + mergeDiff(newImageLuceneDefinitions, combined); + } + + /** + * If there is a diff index (hardcoded node "/oak:index/diff.index" or + * "/oak:index/diff.index.optimizer"), then iterate over all entries and create new + * (merged) versions if needed. + * + * @param newImageLuceneDefinitions + * the new Lucene definitions + * (input + output) + * @param combined + * the definitions in the repository, + * including the one in the customer repo and new ones + * (input) + * @return whether a new version of an index was added + */ + boolean mergeDiff(JsonObject newImageLuceneDefinitions, JsonObject combined) { + // iterate again, this time process + + // collect the diff index(es) + HashMap<String, JsonObject> toProcess = new HashMap<>(); + tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX, toProcess); + tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX_OPTIMIZER, toProcess); + // if the diff index exists, but doesn't contain some of the previous indexes + // (indexes with mergeInfo), then we need to disable those (using /dummy includedPath) + extractExistingMergedIndexes(combined, toProcess); + if (toProcess.isEmpty()) { + log("No diff index definitions found."); + return false; + } + boolean hasChanges = false; + for (Entry<String, JsonObject> e : toProcess.entrySet()) { + String key = e.getKey(); + JsonObject value = e.getValue(); + if (key.startsWith("/oak:index/")) { + LOG.warn("The key should contains just the index name, without the '/oak:index' prefix for key {}", key); + key = key.substring("/oak:index/".length()); + } + log("Processing {}", key); + hasChanges |= processMerge(key, value, newImageLuceneDefinitions, combined); + } + return hasChanges; + } + + /** + * Extract a "diff.index" from the set of index definitions (if found), and if + * found, store the nested entries in the target map, merging them with previous + * entries if found. + * + * The diff.index may either have a file (a "jcr:content" child node with a + * "jcr:data" property), or a "diff" JSON object. For customers (in the git + * repository), the file is much easier to construct, but when running the + * indexing job, the nested JSON is much easier. + * + * @param indexDefs the set of index definitions (may be empty) + * @param name the name of the diff.index (either diff.index or + * diff.index.optimizer) + * @param target the target map of diff.index definitions + * @return the error message trying to parse the JSON file, or null + */ + public static String tryExtractDiffIndex(JsonObject indexDefs, String name, HashMap<String, JsonObject> target) { + JsonObject diffIndex = indexDefs.getChildren().get(name); + if (diffIndex == null) { + return null; + } + // extract either the file, or the nested json + JsonObject file = diffIndex.getChildren().get("diff.json"); + JsonObject diff; + if (file != null) { + // file + JsonObject jcrContent = file.getChildren().get("jcr:content"); + if (jcrContent == null) { + String message = "jcr:content child node is missing in diff.json"; + LOG.warn(message); + return message; + } + String jcrData = JsonNodeBuilder.oakStringValue(jcrContent, "jcr:data"); + try { + diff = JsonObject.fromJson(jcrData, true); + } catch (Exception e) { + LOG.warn("Illegal Json, ignoring: {}", jcrData, e); + String message = "Illegal Json, ignoring: " + e.getMessage(); + return message; + } + } else { + // nested json + diff = diffIndex.getChildren().get("diff"); + } + // store, if not empty + if (diff != null) { + for (Entry<String, JsonObject> e : diff.getChildren().entrySet()) { + String key = e.getKey(); + target.put(key, mergeDiffs(target.get(key), e.getValue())); + } + } + return null; + } + + /** + * Extract the indexes with a "mergeInfo" property and store them in the target + * object. This is needed so that indexes that were removed from the index.diff + * are detected (a new version is needed in this case with includedPaths + * "/dummy"). + * + * @param indexDefs the index definitions in the repository + * @param target the target map of "diff.index" definitions. for each entry + * found, an empty object is added + */ + private static void extractExistingMergedIndexes(JsonObject indexDefs, HashMap<String, JsonObject> target) { + for (Entry<String, JsonObject> e : indexDefs.getChildren().entrySet()) { + String key = e.getKey(); + JsonObject value = e.getValue(); + if (key.indexOf("-custom-") < 0 || !value.getProperties().containsKey("mergeInfo")) { + continue; + } + String baseName = IndexName.parse(key.substring("/oak:index/".length())).getBaseName(); + if (!target.containsKey(baseName)) { + // if there is no entry yet for this key, + // add a new empty object + target.put(baseName, new JsonObject()); + } + } + } + + /** + * Merge diff from "diff.index" and "diff.index.optimizer". + * The customer can define a diff (stored in "diff.index") + * and someone else (or the optimizer) can define one (stored in "diff.index.optimizer"). + * + * @param a the first diff + * @param b the second diff (overwrites entries in a) + * @return the merged entry + */ + public static JsonObject mergeDiffs(JsonObject a, JsonObject b) { + if (a == null) { + return b; + } else if (b == null) { + return a; + } + JsonObject result = JsonObject.fromJson(a.toString(), true); + result.getProperties().putAll(b.getProperties()); + HashSet<String> both = new HashSet<>(a.getChildren().keySet()); + both.addAll(b.getChildren().keySet()); + for (String k : both) { + result.getChildren().put(k, mergeDiffs(a.getChildren().get(k), b.getChildren().get(k))); + } + return result; + } + + /** + * Merge using the diff definition. + * + * If the latest customized index already matches, then + * newImageLuceneDefinitions will remain as is. Otherwise, a new customized + * index is added, with a "mergeInfo" property. + * + * Existing properties are never changed; only new properties/children are + * added. + * + * @param indexName the name, eg. "damAssetLucene" + * @param indexDiff the diff with the new properties + * @param newImageLuceneDefinitions the new Lucene definitions (input + output) + * @param combined the definitions in the repository, including + * the one in the customer repo and new ones + * (input) + * @return whether a new version of an index was added + */ + public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject newImageLuceneDefinitions, JsonObject combined) { Review Comment: yes recommend breaking this method, example (from comments) - findLatestVersions - performMerge - isChanged - addMetadata ########## oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java: ########## @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexName; +import org.apache.jackrabbit.oak.plugins.tree.TreeConstants; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Processing of diff indexes, that is nodes under "/oak:index/diff.index". A + * diff index contains differences to existing indexes, and possibly new + * (custom) indexes in the form of JSON. These changes can then be merged + * (applied) to the index definitions. This allows to simplify index management, + * because it allows to modify (add, update) indexes in a simple way. + */ +public class DiffIndex { + + private static final Logger LOG = LoggerFactory.getLogger(DiffIndex.class); + + /** + * Apply changes to the index definitions. That means merge the index diff with + * the existing indexes, creating new index versions. It might also mean to + * remove old (merged) indexes if the diff no longer contains them. + * + * @param store the node store + * @param indexDefinitions the /oak:index node + */ + public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefinitions) { + JsonObject newImageLuceneDefinitions = null; + for (String diffIndex : new String[] { DiffIndexMerger.DIFF_INDEX, DiffIndexMerger.DIFF_INDEX_OPTIMIZER }) { + if (!indexDefinitions.hasChildNode(diffIndex)) { + continue; + } + NodeBuilder diffIndexDefinition = indexDefinitions.child(diffIndex); + NodeBuilder diffContent = diffIndexDefinition.getChildNode("diff.json").getChildNode("jcr:content"); + if (!diffContent.exists()) { + continue; + } + PropertyState lastMod = diffContent.getProperty("jcr:lastModified"); + if (lastMod == null) { + continue; + } + String modified = lastMod.getValue(Type.DATE); + PropertyState lastProcessed = diffContent.getProperty(":lastProcessed"); + if (lastProcessed != null) { + if (modified.equals(lastProcessed.getValue(Type.STRING))) { + // already processed + continue; + } + } + // store now, so a change is only processed once + diffContent.setProperty(":lastProcessed", modified); + PropertyState jcrData = diffContent.getProperty("jcr:data"); + String diff = tryReadString(jcrData); + if (diff == null) { + continue; + } + try { + JsonObject diffObj = JsonObject.fromJson("{\"diff\": " + diff + "}", true); + diffIndexDefinition.removeProperty("error"); + if (newImageLuceneDefinitions == null) { + newImageLuceneDefinitions = new JsonObject(); + } + newImageLuceneDefinitions.getChildren().put("/oak:index/" + diffIndex, diffObj); + } catch (Exception e) { + String message = "Error parsing " + diffIndex; + LOG.warn("{}: {}", message, e.getMessage(), e); + diffIndexDefinition.setProperty("error", message + ": " + e.getMessage()); + } + } + if (newImageLuceneDefinitions == null) { + // not a valid diff index, or already processed + return; + } + LOG.info("Processing a new diff.index with node store {}", store); + JsonObject repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(indexDefinitions); + LOG.debug("Index list {}", repositoryDefinitions.toString()); + try { + DiffIndexMerger.instance().merge(newImageLuceneDefinitions, repositoryDefinitions, store); + for (String indexPath : newImageLuceneDefinitions.getChildren().keySet()) { + if (indexPath.startsWith("/oak:index/" + DiffIndexMerger.DIFF_INDEX)) { + continue; + } + JsonObject newDef = newImageLuceneDefinitions.getChildren().get(indexPath); + String indexName = PathUtils.getName(indexPath); + JsonNodeBuilder.addOrReplace(indexDefinitions, store, indexName, IndexConstants.INDEX_DEFINITIONS_NODE_TYPE, newDef.toString()); + updateNodetypeIndexForPath(indexDefinitions, indexName, true); + disableOrRemoveOldVersions(indexDefinitions, indexPath, indexName); + } + removeDisabledMergedIndexes(indexDefinitions); + sortIndexes(indexDefinitions); + } catch (Exception e) { + LOG.warn("Error merging diff.index: {}", e.getMessage(), e); + NodeBuilder diffIndexDefinition = indexDefinitions.child(DiffIndexMerger.DIFF_INDEX); + diffIndexDefinition.setProperty("error", e.getMessage()); + } + } + + /** + * Try to read a text from the (binary) jcr:data property. Edge cases such as + * "property does not exist" and IO exceptions (blob not found) do not throw an + * exception (IO exceptions are logged). + * + * @param jcrData the "jcr:data" property + * @return the string, or null if reading fails + */ + public static String tryReadString(PropertyState jcrData) { + if (jcrData == null) { + return null; + } + InputStream in = jcrData.getValue(Type.BINARY).getNewStream(); + try { + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + LOG.warn("Can not read jcr:data", e); + return null; + } + } + + private static void sortIndexes(NodeBuilder builder) { + ArrayList<String> list = new ArrayList<>(); + for (String child : builder.getChildNodeNames()) { + list.add(child); + } + list.sort(Comparator.naturalOrder()); + builder.setProperty(TreeConstants.OAK_CHILD_ORDER, list, Type.NAMES); + } + + private static void removeDisabledMergedIndexes(NodeBuilder definitions) { + ArrayList<String> toRemove = new ArrayList<>(); + for (String child : definitions.getChildNodeNames()) { + if (!definitions.getChildNode(child).hasProperty("mergeChecksum")) { + continue; + } + if ("disabled".equals(definitions.getChildNode(child).getString("type"))) { + toRemove.add(child); + } + } + for (String r : toRemove) { + LOG.info("Removing disabled index {}", r); + definitions.child(r).remove(); + updateNodetypeIndexForPath(definitions, r, false); + } + } + + /** + * Try to remove or disable old version of merged indexes, if there are any. + * + * @param definitions the builder for /oak:index + * @param indexPath the path + * @param keep which index name (which version) to retain + */ + private static void disableOrRemoveOldVersions(NodeBuilder definitions, String indexPath, String keep) { + String indexName = indexPath; + if (indexPath.startsWith("/oak:index/")) { + indexName = indexPath.substring("/oak:index/".length()); + } + String baseName = IndexName.parse(indexName).getBaseName(); + ArrayList<String> toRemove = new ArrayList<>(); + for (String child : definitions.getChildNodeNames()) { + if (child.equals(keep) || child.indexOf("-custom-") < 0) { + // the one to keep, or not a customized or custom index + continue; + } + String childBaseName = IndexName.parse(child).getBaseName(); + if (baseName.equals(childBaseName)) { + if (indexName.equals(child)) { + if (!"disabled".equals(definitions.getChildNode(indexName).getString("type"))) { + continue; + } + } + toRemove.add(child); + } + } + for (String r : toRemove) { + LOG.info("Removing old index " + r); Review Comment: ```suggestion LOG.info("Removing old index {}", r); ``` ########## oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java: ########## @@ -0,0 +1,833 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.jackrabbit.oak.commons.StringUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.commons.json.JsopBuilder; +import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; +import org.apache.jackrabbit.oak.json.Base64BlobSerializer; +import org.apache.jackrabbit.oak.json.JsonSerializer; +import org.apache.jackrabbit.oak.plugins.index.IndexName; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Index definition merge utility that uses the "diff" mode. + */ +public class DiffIndexMerger { + + private final static Logger LOG = LoggerFactory.getLogger(DiffIndexMerger.class); + + public final static String DIFF_INDEX = "diff.index"; + public final static String DIFF_INDEX_OPTIMIZER = "diff.index.optimizer"; + + private final static String MERGE_INFO = "This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html";; + + // the list of unsupported included paths, e.g. "/apps,/libs" + // by default all paths are supported + private final static String[] UNSUPPORTED_INCLUDED_PATHS = System.getProperty("oak.diffIndex.unsupportedPaths", "").split(","); + + // in case a custom index is removed, whether a dummy index is created + private final static boolean DELETE_CREATES_DUMMY = Boolean.getBoolean("oak.diffIndex.deleteCreatesDummy"); + + // in case a customization was removed, create a copy of the OOTB index + private final static boolean DELETE_COPIES_OOTB = Boolean.getBoolean("oak.diffIndex.deleteCopiesOOTB"); + + // whether to log at info level + private final static boolean LOG_AT_INFO_LEVEL = Boolean.getBoolean("oak.diffIndex.logAtInfoLevel"); + + private final String[] unsupportedIncludedPaths; + private final boolean deleteCreatesDummyIndex; + private final boolean deleteCopiesOutOfTheBoxIndex; + private final boolean logAtInfoLevel; + + static final DiffIndexMerger INSTANCE = new DiffIndexMerger(UNSUPPORTED_INCLUDED_PATHS, + DELETE_CREATES_DUMMY, DELETE_COPIES_OOTB, LOG_AT_INFO_LEVEL); + + public static DiffIndexMerger instance() { + return INSTANCE; + } + + DiffIndexMerger(String[] unsupportedIncludedPaths, + boolean deleteCreatesDummyIndex, boolean deleteCopiesOutOfTheBoxIndex, + boolean logAtInfoLevel) { + this.unsupportedIncludedPaths = unsupportedIncludedPaths; + this.deleteCreatesDummyIndex = deleteCreatesDummyIndex; + this.deleteCopiesOutOfTheBoxIndex = deleteCopiesOutOfTheBoxIndex; + this.logAtInfoLevel = logAtInfoLevel; + } + + /** + * If there is a diff index, that is an index with prefix "diff.", then try to merge it. + * + * @param newImageLuceneDefinitions + * the new indexes + * (input and output) + * @param repositoryDefinitions + * the indexes in the writable repository + * (input) + * @param repositoryNodeStore + */ + public void merge(JsonObject newImageLuceneDefinitions, JsonObject repositoryDefinitions, NodeStore repositoryNodeStore) { + // combine all definitions into one object + JsonObject combined = new JsonObject(); + + // index definitions in the repository + combined.getChildren().putAll(repositoryDefinitions.getChildren()); + + // read the diff.index.optimizer explicitly, + // because it's a not a regular index definition, + // and so in the repositoryDefinitions + if (repositoryNodeStore != null) { + Map<String, JsonObject> diffInRepo = readDiffIndex(repositoryNodeStore, DIFF_INDEX_OPTIMIZER); + combined.getChildren().putAll(diffInRepo); + } + + // overwrite with the provided definitions (if any) + combined.getChildren().putAll(newImageLuceneDefinitions.getChildren()); + + // check if there "diff.index" or "diff.index.optimizer" + boolean found = combined.getChildren().containsKey("/oak:index/" + DIFF_INDEX) + || combined.getChildren().containsKey("/oak:index/" + DIFF_INDEX_OPTIMIZER); + if (!found) { + // early exit, so that the risk of merging the PR + // is very small for customers that do not use this + log("No 'diff.index' definition"); + return; + } + mergeDiff(newImageLuceneDefinitions, combined); + } + + /** + * If there is a diff index (hardcoded node "/oak:index/diff.index" or + * "/oak:index/diff.index.optimizer"), then iterate over all entries and create new + * (merged) versions if needed. + * + * @param newImageLuceneDefinitions + * the new Lucene definitions + * (input + output) + * @param combined + * the definitions in the repository, + * including the one in the customer repo and new ones + * (input) + * @return whether a new version of an index was added + */ + boolean mergeDiff(JsonObject newImageLuceneDefinitions, JsonObject combined) { + // iterate again, this time process + + // collect the diff index(es) + HashMap<String, JsonObject> toProcess = new HashMap<>(); + tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX, toProcess); + tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX_OPTIMIZER, toProcess); + // if the diff index exists, but doesn't contain some of the previous indexes + // (indexes with mergeInfo), then we need to disable those (using /dummy includedPath) + extractExistingMergedIndexes(combined, toProcess); + if (toProcess.isEmpty()) { + log("No diff index definitions found."); + return false; + } + boolean hasChanges = false; + for (Entry<String, JsonObject> e : toProcess.entrySet()) { + String key = e.getKey(); + JsonObject value = e.getValue(); + if (key.startsWith("/oak:index/")) { + LOG.warn("The key should contains just the index name, without the '/oak:index' prefix for key {}", key); + key = key.substring("/oak:index/".length()); + } + log("Processing {}", key); + hasChanges |= processMerge(key, value, newImageLuceneDefinitions, combined); + } + return hasChanges; + } + + /** + * Extract a "diff.index" from the set of index definitions (if found), and if + * found, store the nested entries in the target map, merging them with previous + * entries if found. + * + * The diff.index may either have a file (a "jcr:content" child node with a + * "jcr:data" property), or a "diff" JSON object. For customers (in the git + * repository), the file is much easier to construct, but when running the + * indexing job, the nested JSON is much easier. + * + * @param indexDefs the set of index definitions (may be empty) + * @param name the name of the diff.index (either diff.index or + * diff.index.optimizer) + * @param target the target map of diff.index definitions + * @return the error message trying to parse the JSON file, or null + */ + public static String tryExtractDiffIndex(JsonObject indexDefs, String name, HashMap<String, JsonObject> target) { + JsonObject diffIndex = indexDefs.getChildren().get(name); + if (diffIndex == null) { + return null; + } + // extract either the file, or the nested json + JsonObject file = diffIndex.getChildren().get("diff.json"); + JsonObject diff; + if (file != null) { + // file + JsonObject jcrContent = file.getChildren().get("jcr:content"); + if (jcrContent == null) { + String message = "jcr:content child node is missing in diff.json"; + LOG.warn(message); + return message; + } + String jcrData = JsonNodeBuilder.oakStringValue(jcrContent, "jcr:data"); + try { + diff = JsonObject.fromJson(jcrData, true); + } catch (Exception e) { + LOG.warn("Illegal Json, ignoring: {}", jcrData, e); + String message = "Illegal Json, ignoring: " + e.getMessage(); + return message; + } + } else { + // nested json + diff = diffIndex.getChildren().get("diff"); + } + // store, if not empty + if (diff != null) { + for (Entry<String, JsonObject> e : diff.getChildren().entrySet()) { + String key = e.getKey(); + target.put(key, mergeDiffs(target.get(key), e.getValue())); + } + } + return null; + } + + /** + * Extract the indexes with a "mergeInfo" property and store them in the target + * object. This is needed so that indexes that were removed from the index.diff + * are detected (a new version is needed in this case with includedPaths + * "/dummy"). + * + * @param indexDefs the index definitions in the repository + * @param target the target map of "diff.index" definitions. for each entry + * found, an empty object is added + */ + private static void extractExistingMergedIndexes(JsonObject indexDefs, HashMap<String, JsonObject> target) { + for (Entry<String, JsonObject> e : indexDefs.getChildren().entrySet()) { + String key = e.getKey(); + JsonObject value = e.getValue(); + if (key.indexOf("-custom-") < 0 || !value.getProperties().containsKey("mergeInfo")) { + continue; + } + String baseName = IndexName.parse(key.substring("/oak:index/".length())).getBaseName(); + if (!target.containsKey(baseName)) { + // if there is no entry yet for this key, + // add a new empty object + target.put(baseName, new JsonObject()); + } + } + } + + /** + * Merge diff from "diff.index" and "diff.index.optimizer". + * The customer can define a diff (stored in "diff.index") + * and someone else (or the optimizer) can define one (stored in "diff.index.optimizer"). + * + * @param a the first diff + * @param b the second diff (overwrites entries in a) + * @return the merged entry + */ + public static JsonObject mergeDiffs(JsonObject a, JsonObject b) { + if (a == null) { + return b; + } else if (b == null) { + return a; + } + JsonObject result = JsonObject.fromJson(a.toString(), true); + result.getProperties().putAll(b.getProperties()); + HashSet<String> both = new HashSet<>(a.getChildren().keySet()); + both.addAll(b.getChildren().keySet()); + for (String k : both) { + result.getChildren().put(k, mergeDiffs(a.getChildren().get(k), b.getChildren().get(k))); + } + return result; + } + + /** + * Merge using the diff definition. + * + * If the latest customized index already matches, then + * newImageLuceneDefinitions will remain as is. Otherwise, a new customized + * index is added, with a "mergeInfo" property. + * + * Existing properties are never changed; only new properties/children are + * added. + * + * @param indexName the name, eg. "damAssetLucene" + * @param indexDiff the diff with the new properties + * @param newImageLuceneDefinitions the new Lucene definitions (input + output) + * @param combined the definitions in the repository, including + * the one in the customer repo and new ones + * (input) + * @return whether a new version of an index was added + */ + public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject newImageLuceneDefinitions, JsonObject combined) { + // extract the latest product index (eg. damAssetLucene-12) + // and customized index (eg. damAssetLucene-12-custom-3) - if any + IndexName latestProduct = null; + String latestProductKey = null; + IndexName latestCustomized = null; + String latestCustomizedKey = null; + String prefix = "/oak:index/"; + for (String key : combined.getChildren().keySet()) { + IndexName name = IndexName.parse(key.substring(prefix.length())); + if (!name.isVersioned()) { + log("Ignoring unversioned index {}", name); + continue; + } + if (!name.getBaseName().equals(indexName)) { + continue; + } + boolean isCustom = key.indexOf("-custom-") >= 0; + if (isCustom) { + if (latestCustomized == null || + name.compareTo(latestCustomized) > 0) { + latestCustomized = name; + latestCustomizedKey = key; + } + } else { + if (latestProduct == null || + name.compareTo(latestProduct) > 0) { + latestProduct = name; + latestProductKey = key; + } + } + } + log("Latest product: {}", latestProductKey); + log("Latest customized: {}", latestCustomizedKey); + if (latestProduct == null) { + if (indexName.indexOf('.') >= 0) { + // a fully custom index needs to contains a dot + log("Fully custom index {}", indexName); + } else { + log("No product version for {}", indexName); + return false; + } + } + JsonObject latestProductIndex = combined.getChildren().get(latestProductKey); + String[] includedPaths; + if (latestProductIndex == null) { + if (indexDiff.getProperties().isEmpty() && indexDiff.getChildren().isEmpty()) { + // there is no customization (any more), which means a dummy index may be needed + log("No customization for {}", indexName); + } else { + includedPaths = JsonNodeBuilder.oakStringArrayValue(indexDiff, "includedPaths"); + if (includesUnsupportedPaths(includedPaths)) { + LOG.warn("New custom index {} is not supported because it contains an unsupported path ({})", + indexName, Arrays.toString(unsupportedIncludedPaths)); + return false; + } + } + } else { + includedPaths = JsonNodeBuilder.oakStringArrayValue(latestProductIndex, "includedPaths"); + if (includesUnsupportedPaths(includedPaths)) { + LOG.warn("Customizing index {} is not supported because it contains an unsupported path ({})", + latestProductKey, Arrays.toString(unsupportedIncludedPaths)); + return false; + } + } + + // merge + JsonObject merged = null; + if (indexDiff == null) { + // no diff definition: use to the OOTB index + if (latestCustomized == null) { + log("Only a product index found, nothing to do"); + return false; + } + merged = latestProductIndex; + } else { + merged = processMerge(latestProductIndex, indexDiff); + } + + // compare to the latest version of the this index + JsonObject latestIndexVersion = new JsonObject(); + if (latestCustomized == null) { + latestIndexVersion = latestProductIndex; + } else { + latestIndexVersion = combined.getChildren().get(latestCustomizedKey); + } + JsonObject mergedDef = cleanedAndNormalized(switchToLucene(merged)); + // compute merge checksum for later, but do not yet add + String mergeChecksum = computeMergeChecksum(mergedDef); + // get the merge checksum before cleaning (cleaning removes it) - if available + String key; + if (latestIndexVersion == null) { + // new index + key = prefix + indexName + "-1-custom-1"; + } else { + String latestMergeChecksum = JsonNodeBuilder.oakStringValue(latestIndexVersion, "mergeChecksum"); + JsonObject latestDef = cleanedAndNormalized(switchToLucene(latestIndexVersion)); + if (isSameIgnorePropertyOrder(mergedDef, latestDef)) { + // normal case: no change + // (even if checksums do not match: checksums might be missing or manipulated) + log("Latest index matches"); + if (latestMergeChecksum != null && !latestMergeChecksum.equals(mergeChecksum)) { + LOG.warn("Indexes do match, but checksums do not. Possibly checksum was changed: {} vs {}", latestMergeChecksum, mergeChecksum); + LOG.warn("latest: {}\nmerged: {}", latestDef, mergedDef); + } + return false; + } + if (latestMergeChecksum != null && latestMergeChecksum.equals(mergeChecksum)) { + // checksum matches, but data does not match + // could be eg. due to numbers formatting issues (-0.0 vs 0.0, 0.001 vs 1e-3) + // but unexpected because we do not normally have such cases + LOG.warn("Indexes do not match, but checksums match. Possible normalization issue."); + LOG.warn("Index: {}, latest: {}\nmerged: {}", indexName, latestDef, mergedDef); + // if checksums match, we consider it a match + return false; + } + LOG.info("Indexes do not match, with"); + LOG.info("Index: {}, latest: {}\nmerged: {}", indexName, latestDef, mergedDef); + // a new merged index definition + if (latestProduct == null) { + // fully custom index: increment version + key = prefix + indexName + + "-" + latestCustomized.getProductVersion() + + "-custom-" + (latestCustomized.getCustomerVersion() + 1); + } else { + // customized OOTB index: use the latest product as the base + key = prefix + indexName + + "-" + latestProduct.getProductVersion() + + "-custom-"; + if (latestCustomized != null) { + key += (latestCustomized.getCustomerVersion() + 1); + } else { + key += "1"; + } + } + } + merged.getProperties().put("mergeInfo", JsopBuilder.encode(MERGE_INFO)); + merged.getProperties().put("mergeChecksum", JsopBuilder.encode(mergeChecksum)); + merged.getProperties().put("merges", "[" + JsopBuilder.encode("/oak:index/" + indexName) + "]"); + merged.getProperties().remove("reindexCount"); + merged.getProperties().remove("reindex"); + if (!deleteCopiesOutOfTheBoxIndex && indexDiff.toString().equals("{}")) { + merged.getProperties().put("type", "\"disabled\""); + merged.getProperties().put("mergeComment", "\"This index is superseeded and can be removed\""); Review Comment: ```suggestion merged.getProperties().put("mergeComment", "\"This index is superseded and can be removed\""); ``` ########## oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java: ########## @@ -234,6 +234,23 @@ public static Collection<String> filterReplacedIndexes(Collection<String> indexP return result; } + public static Collection<String> filterNewestIndexes(Collection<String> indexPaths, NodeState rootState) { Review Comment: rootState ununsed parameter, consider removing -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
