Author: ngupta
Date: Thu Jan 21 06:48:34 2021
New Revision: 1885742
URL: http://svn.apache.org/viewvc?rev=1885742&view=rev
Log:
OAK-9312|Modify oak-run to support creating indexes of type Elasticsearch
Added:
jackrabbit/oak/trunk/oak-run-elastic/
jackrabbit/oak/trunk/oak-run-elastic/pom.xml
jackrabbit/oak/trunk/oak-run-elastic/src/
jackrabbit/oak/trunk/oak-run-elastic/src/main/
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticDocumentStoreIndexer.java
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticIndexCommand.java
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticIndexOptions.java
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticOutOfBandIndexer.java
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexer.java
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexerProvider.java
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/AvailableElasticModes.java
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/Main.java
jackrabbit/oak/trunk/oak-run-elastic/src/test/
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/jackrabbit/
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/jackrabbit/oak/
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/jackrabbit/oak/index/
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/jackrabbit/oak/index/indexer/
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexerTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocumentMaker.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriterFactory.java
jackrabbit/oak/trunk/pom.xml
Added: jackrabbit/oak/trunk/oak-run-elastic/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-elastic/pom.xml?rev=1885742&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run-elastic/pom.xml (added)
+++ jackrabbit/oak/trunk/oak-run-elastic/pom.xml Thu Jan 21 06:48:34 2021
@@ -0,0 +1,187 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd ">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-parent</artifactId>
+ <version>1.37-SNAPSHOT</version>
+ <relativePath>../oak-parent/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>oak-run-elastic</artifactId>
+ <name>Oak Elastic Runnable Jar</name>
+
+ <properties>
+ <jetty.version>8.2.0.v20160908</jetty.version>
+ <groovy.version>2.4.17</groovy.version>
+ <max.jar.size>27000000</max.jar.size>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.gmavenplus</groupId>
+ <artifactId>gmavenplus-plugin</artifactId>
+ <version>1.5</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>addSources</goal>
+ <goal>addTestSources</goal>
+ <goal>generateStubs</goal>
+ <goal>compile</goal>
+ <goal>testGenerateStubs</goal>
+ <goal>testCompile</goal>
+ <goal>removeStubs</goal>
+ <goal>removeTestStubs</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>3.0.0-M2</version>
+ <executions>
+ <execution>
+ <id>enforce-file-size</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <rules>
+
+ <requireFilesSize>
+ <maxsize>${max.jar.size}</maxsize>
+ <files>
+
<file>${project.build.directory}/${project.build.finalName}.jar</file>
+ </files>
+ </requireFilesSize>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-run-commons</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-jcr</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-query-spi</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-core-spi</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-blob-cloud</artifactId>
+ <version>${project.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-segment-tar</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-segment</artifactId>
+ <version>1.6.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-store-document</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-search-elastic</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-run</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- Nullability annotations -->
+ <dependency>
+ <groupId>org.jetbrains</groupId>
+ <artifactId>annotations</artifactId>
+ </dependency>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+
+
+</project>
\ No newline at end of file
Added:
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticDocumentStoreIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticDocumentStoreIndexer.java?rev=1885742&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticDocumentStoreIndexer.java
(added)
+++
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticDocumentStoreIndexer.java
Thu Jan 21 06:48:34 2021
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.jackrabbit.oak.index.indexer.document.DocumentStoreIndexer;
+import org.apache.jackrabbit.oak.index.indexer.document.ElasticIndexerProvider;
+import
org.apache.jackrabbit.oak.index.indexer.document.NodeStateIndexerProvider;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticConnection;
+
+import java.io.IOException;
+import java.util.List;
+
+/*
+Out of band indexer for Elasticsearch. Provides support to index document
store for given index definitions or reindex existing indexes
+ */
+public class ElasticDocumentStoreIndexer extends DocumentStoreIndexer {
+ private final IndexHelper indexHelper;
+ private final List<NodeStateIndexerProvider> indexerProviders;
+ private final IndexerSupport indexerSupport;
+
+ private final String indexPrefix;
+ private final String scheme;
+ private final String host;
+ private final int port;
+ private final String apiKeyId;
+ private final String apiSecretId;
+
+ public ElasticDocumentStoreIndexer(IndexHelper indexHelper, IndexerSupport
indexerSupport,
+ String indexPrefix, String scheme,
+ String host, int port,
+ String apiKeyId, String apiSecretId)
throws IOException {
+ super(indexHelper, indexerSupport);
+ this.indexHelper = indexHelper;
+ this.indexerSupport = indexerSupport;
+ this.indexerProviders = createProviders();
+
+ this.indexPrefix = indexPrefix;
+ this.scheme = scheme;
+ this.host = host;
+ this.port = port;
+ this.apiKeyId = apiKeyId;
+ this.apiSecretId = apiSecretId;
+ }
+
+ protected List<NodeStateIndexerProvider> createProviders() {
+ List<NodeStateIndexerProvider> providers = ImmutableList.of(
+ createElasticIndexerProvider()
+ );
+
+ providers.forEach(closer::register);
+ return providers;
+ }
+
+ private NodeStateIndexerProvider createElasticIndexerProvider() {
+ final ElasticConnection.Builder.BuildStep buildStep =
ElasticConnection.newBuilder()
+ .withIndexPrefix(indexPrefix)
+ .withConnectionParameters(
+ scheme,
+ host,
+ port
+ );
+ final ElasticConnection coordinate;
+ if (apiKeyId != null && apiSecretId != null) {
+ coordinate = buildStep.withApiKeys(apiKeyId, apiSecretId).build();
+ } else {
+ coordinate = buildStep.build();
+ }
+ closer.register(coordinate);
+ return new ElasticIndexerProvider(indexHelper, coordinate);
+ }
+
+}
Added:
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticIndexCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticIndexCommand.java?rev=1885742&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticIndexCommand.java
(added)
+++
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticIndexCommand.java
Thu Jan 21 06:48:34 2021
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Sets;
+import com.google.common.io.Closer;
+import joptsimple.OptionParser;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.importer.IndexDefinitionUpdater;
+import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.run.cli.Options;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.run.commons.LoggingInitializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/*
+Command file for Elastic index operation.
+ */
+public class ElasticIndexCommand implements Command {
+ private static final Logger log =
LoggerFactory.getLogger(ElasticIndexCommand.class);
+ private Options opts;
+ private ElasticIndexOptions indexOpts;
+ public static final String NAME = "index";
+
+ private final String summary = "Provides elastic index management related
operations";
+ private static boolean disableExitOnError;
+
+
+ @Override
+ public void execute(String... args) throws Exception {
+ OptionParser parser = new OptionParser();
+
+ opts = new Options();
+ opts.setCommandName(NAME);
+ opts.setSummary(summary);
+ opts.setConnectionString(CommonOptions.DEFAULT_CONNECTION_STRING);
+ opts.registerOptionsFactory(ElasticIndexOptions.FACTORY);
+ opts.parseAndConfigure(parser, args);
+
+ indexOpts = opts.getOptionBean(ElasticIndexOptions.class);
+
+ //Clean up before setting up NodeStore as the temp
+ //directory might be used by NodeStore for cache stuff like
persistentCache
+ //setupDirectories(indexOpts);
+ //setupLogging(indexOpts);
+
+ logCliArgs(args);
+
+ boolean success = false;
+ try {
+ try (Closer closer = Closer.create()) {
+ //configureCustomizer(opts, closer, true);
+ NodeStoreFixture fixture =
NodeStoreFixtureProvider.create(opts);
+ closer.register(fixture);
+ execute(fixture, indexOpts, closer);
+ //tellReportPaths();
+ }
+
+ success = true;
+ } catch (Throwable e) {
+ log.error("Error occurred while performing index tasks", e);
+ if (disableExitOnError) {
+ throw e;
+ }
+ } finally {
+ shutdownLogging();
+ }
+
+ if (!success) {
+ //Needed for changes after OAK-6409
+ System.exit(1);
+ }
+ }
+
+ private void execute(NodeStoreFixture fixture, IndexOptions indexOpts,
Closer closer)
+ throws IOException, CommitFailedException {
+ IndexHelper indexHelper = createIndexHelper(fixture, indexOpts,
closer);
+
+ // TODO : See if we need to support dumpIndexStats and index defs for
elastic - not needed for now
+ //dumpIndexStats(indexOpts, indexHelper);
+ //dumpIndexDefinitions(indexOpts, indexHelper);
+ reindexOperation(indexOpts, indexHelper);
+ }
+
+ private IndexHelper createIndexHelper(NodeStoreFixture fixture,
+ IndexOptions indexOpts, Closer
closer) throws IOException {
+ IndexHelper indexHelper = new IndexHelper(fixture.getStore(),
fixture.getBlobStore(), fixture.getWhiteboard(),
+ indexOpts.getOutDir(), indexOpts.getWorkDir(),
computeIndexPaths(indexOpts));
+
+ // TODO : See if pre text extraction is needed for elastic
+ //configurePreExtractionSupport(indexOpts, indexHelper);
+
+ closer.register(indexHelper);
+ return indexHelper;
+ }
+
+ private List<String> computeIndexPaths(IndexOptions indexOpts) throws
IOException {
+ //Combine the indexPaths from json and cli args
+ Set<String> indexPaths = new
LinkedHashSet<>(indexOpts.getIndexPaths());
+ File definitions = indexOpts.getIndexDefinitionsFile();
+ if (definitions != null) {
+ IndexDefinitionUpdater updater = new
IndexDefinitionUpdater(definitions);
+ Set<String> indexPathsFromJson = updater.getIndexPaths();
+ Set<String> diff = Sets.difference(indexPathsFromJson, indexPaths);
+ if (!diff.isEmpty()) {
+ log.info("Augmenting the indexPaths with {} which are present
in {}", diff, definitions);
+ }
+ indexPaths.addAll(indexPathsFromJson);
+ }
+ return new ArrayList<>(indexPaths);
+ }
+
+ private void reindexOperation(IndexOptions indexOpts, IndexHelper
indexHelper) throws IOException, CommitFailedException {
+ if (!indexOpts.isReindex()) {
+ return;
+ }
+
+ String checkpoint = indexOpts.getCheckpoint();
+ reindex(indexOpts, indexHelper, checkpoint);
+ }
+
+ private void reindex(IndexOptions idxOpts, IndexHelper indexHelper, String
checkpoint) throws IOException, CommitFailedException {
+ Preconditions.checkNotNull(checkpoint, "Checkpoint value is required
for reindexing done in read only mode");
+
+ Stopwatch w = Stopwatch.createStarted();
+ IndexerSupport indexerSupport = createIndexerSupport(indexHelper,
checkpoint);
+ log.info("Proceeding to index {} upto checkpoint {} {}",
indexHelper.getIndexPaths(), checkpoint,
+ indexerSupport.getCheckpointInfo());
+
+ if (opts.getCommonOpts().isMongo() && idxOpts.isDocTraversalMode()) {
+ log.info("Using Document order traversal to perform reindexing");
+ try (ElasticDocumentStoreIndexer indexer = new
ElasticDocumentStoreIndexer(indexHelper, indexerSupport,
indexOpts.getIndexPrefix(),
+ indexOpts.getElasticScheme(), indexOpts.getElasticHost(),
+ indexOpts.getElasticPort(), indexOpts.getApiKeyId(),
indexOpts.getApiKeySecret())) {
+ indexer.reindex();
+ // Wait for default flush interval before exiting the try block
+ // to make sure the client is not closed before the last flush
+ // TODO : See if this can be handled in a better manner
+
Thread.sleep(ElasticIndexDefinition.BULK_FLUSH_INTERVAL_MS_DEFAULT * 2);
+ } catch (InterruptedException e) {
+ log.debug("Exception while waiting for Elastic connection to
close", e);
+ }
+ } else {
+ try (ElasticOutOfBandIndexer indexer = new
ElasticOutOfBandIndexer(indexHelper, indexerSupport, indexOpts.getIndexPrefix(),
+ indexOpts.getElasticScheme(), indexOpts.getElasticHost(),
+ indexOpts.getElasticPort(), indexOpts.getApiKeyId(),
indexOpts.getApiKeySecret())) {
+
+ indexer.reindex();
+ // Wait for default flush interval before exiting the try block
+ // to make sure the client is not closed before the last flush
+
Thread.sleep(ElasticIndexDefinition.BULK_FLUSH_INTERVAL_MS_DEFAULT * 2);
+ } catch (InterruptedException e) {
+ log.debug("Exception while waiting for Elastic connection to
close", e);
+ }
+ }
+ indexerSupport.writeMetaInfo(checkpoint);
+ log.info("Indexing completed for indexes {} in {} ({} ms)",
+ indexHelper.getIndexPaths(), w,
w.elapsed(TimeUnit.MILLISECONDS));
+ }
+
+ private IndexerSupport createIndexerSupport(IndexHelper indexHelper,
String checkpoint) {
+ IndexerSupport indexerSupport = new IndexerSupport(indexHelper,
checkpoint);
+
+ File definitions = indexOpts.getIndexDefinitionsFile();
+ if (definitions != null) {
+ Preconditions.checkArgument(definitions.exists(), "Index
definitions file [%s] not found", getPath(definitions));
+ indexerSupport.setIndexDefinitions(definitions);
+ }
+ return indexerSupport;
+ }
+
+ static Path getPath(File file) {
+ return file.toPath().normalize().toAbsolutePath();
+ }
+
+ private void shutdownLogging() {
+ LoggingInitializer.shutdownLogging();
+ }
+
+ private static void logCliArgs(String[] args) {
+ log.info("Command line arguments used for indexing [{}]", Joiner.on('
').join(args));
+ List<String> inputArgs =
ManagementFactory.getRuntimeMXBean().getInputArguments();
+ if (!inputArgs.isEmpty()) {
+ log.info("System properties and vm options passed {}", inputArgs);
+ }
+ }
+
+ public static void setDisableExitOnError(boolean disableExitOnError) {
+ ElasticIndexCommand.disableExitOnError = disableExitOnError;
+ }
+
+}
Added:
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticIndexOptions.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticIndexOptions.java?rev=1885742&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticIndexOptions.java
(added)
+++
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticIndexOptions.java
Thu Jan 21 06:48:34 2021
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSpec;
+import org.apache.jackrabbit.oak.run.cli.OptionsBeanFactory;
+
+/*
+Oak Run options for Elasticsearch indexing. These are primarily used to create
the connection to elasticsearch.
+ */
+public class ElasticIndexOptions extends IndexOptions {
+
+ public static final OptionsBeanFactory FACTORY = ElasticIndexOptions::new;
+
+ private final OptionSpec<String> scheme;
+ private final OptionSpec<String> host;
+ private final OptionSpec<String> apiKeyId;
+ private final OptionSpec<String> apiKeySecret;
+ private final OptionSpec<Integer> port;
+ private final OptionSpec<String> indexPrefix;
+
+
+ public ElasticIndexOptions(OptionParser parser) {
+ super(parser);
+ scheme = parser.accepts("scheme", "Elastic scheme")
+ .withRequiredArg().ofType(String.class);
+ host = parser.accepts("host", "Elastic host")
+ .withRequiredArg().ofType(String.class);
+ port = parser.accepts("port", "Elastic port")
+ .withRequiredArg().ofType(Integer.class);
+ apiKeyId = parser.accepts("apiKeyId", "Elastic api key id")
+ .withRequiredArg().ofType(String.class);
+ apiKeySecret = parser.accepts("apiKeySecret", "Elastic api key host")
+ .withRequiredArg().ofType(String.class);
+ indexPrefix = parser.accepts("indexPrefix", "Elastic indexPrefix")
+ .withRequiredArg().ofType(String.class);
+ }
+
+ public String getElasticScheme() {
+ return scheme.value(options);
+ }
+
+ public int getElasticPort() {
+ return port.value(options);
+ }
+
+ public String getElasticHost() {
+ return host.value(options);
+ }
+
+ public String getApiKeyId() {
+ return apiKeyId.value(options);
+ }
+
+ public String getApiKeySecret() {
+ return apiKeySecret.value(options);
+ }
+
+ public String getIndexPrefix() {
+ return indexPrefix.value(options);
+ }
+}
Added:
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticOutOfBandIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticOutOfBandIndexer.java?rev=1885742&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticOutOfBandIndexer.java
(added)
+++
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/ElasticOutOfBandIndexer.java
Thu Jan 21 06:48:34 2021
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index;
+
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticConnection;
+import
org.apache.jackrabbit.oak.plugins.index.elastic.index.ElasticIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+
+
+import static java.util.Arrays.asList;
+
+/*
+Out of band indexer for Elasticsearch. Provides support to index segment store
for given index definitions or reindex existing indexes
+ */
+public class ElasticOutOfBandIndexer extends OutOfBandIndexer {
+ private final String indexPrefix;
+ private final String scheme;
+ private final String host;
+ private final int port;
+ private final String apiKeyId;
+ private final String apiSecretId;
+
+ public ElasticOutOfBandIndexer(IndexHelper indexHelper, IndexerSupport
indexerSupport,
+ String indexPrefix, String scheme,
+ String host, int port,
+ String apiKeyId, String apiSecretId) {
+ super(indexHelper, indexerSupport);
+ this.indexPrefix = indexPrefix;
+ this.scheme = scheme;
+ this.host = host;
+ this.port = port;
+ this.apiKeyId = apiKeyId;
+ this.apiSecretId = apiSecretId;
+ }
+
+ @Override
+ protected IndexEditorProvider createIndexEditorProvider() {
+ IndexEditorProvider elastic = createElasticEditorProvider();
+ return CompositeIndexEditorProvider.compose(asList(elastic));
+ }
+
+ private IndexEditorProvider createElasticEditorProvider() {
+ final ElasticConnection.Builder.BuildStep buildStep =
ElasticConnection.newBuilder()
+ .withIndexPrefix(indexPrefix)
+ .withConnectionParameters(
+ scheme,
+ host,
+ port
+ );
+ final ElasticConnection coordinate;
+ if (apiKeyId != null && apiSecretId != null) {
+ coordinate = buildStep.withApiKeys(apiKeyId, apiSecretId).build();
+ } else {
+ coordinate = buildStep.build();
+ }
+ closer.register(coordinate);
+ ElasticIndexEditorProvider editorProvider = new
ElasticIndexEditorProvider(coordinate,
+ new ExtractedTextCache(10 * FileUtils.ONE_MB, 100));
+ return editorProvider;
+ }
+}
Added:
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexer.java?rev=1885742&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexer.java
(added)
+++
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexer.java
Thu Jan 21 06:48:34 2021
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.indexer.document;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
+import org.apache.jackrabbit.oak.plugins.index.elastic.index.ElasticDocument;
+import
org.apache.jackrabbit.oak.plugins.index.elastic.index.ElasticDocumentMaker;
+import
org.apache.jackrabbit.oak.plugins.index.progress.IndexingProgressReporter;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import
org.apache.jackrabbit.oak.plugins.index.search.spi.binary.FulltextBinaryTextExtractor;
+import
org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriter;
+import org.apache.jackrabbit.oak.spi.filter.PathFilter;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+
+import java.io.IOException;
+import java.util.Set;
+
+/*
+NodeStateIndexer for Elastic. Indexes entries from a given nodestate.
+ */
+public class ElasticIndexer implements NodeStateIndexer {
+
+ private final IndexDefinition definition;
+ private final FulltextBinaryTextExtractor binaryTextExtractor;
+ private final NodeBuilder definitionBuilder;
+ private final IndexingProgressReporter progressReporter;
+ private final FulltextIndexWriter<ElasticDocument> indexWriter;
+
+ public ElasticIndexer(IndexDefinition definition,
FulltextBinaryTextExtractor binaryTextExtractor,
+ NodeBuilder definitionBuilder,
IndexingProgressReporter progressReporter,
+ FulltextIndexWriter<ElasticDocument> indexWriter) {
+ this.definition = definition;
+ this.binaryTextExtractor = binaryTextExtractor;
+ this.definitionBuilder = definitionBuilder;
+ this.progressReporter = progressReporter;
+ this.indexWriter = indexWriter;
+ }
+
+ @Override
+ public boolean shouldInclude(String path) {
+ return getFilterResult(path) != PathFilter.Result.EXCLUDE;
+ }
+
+ @Override
+ public boolean shouldInclude(NodeDocument doc) {
+ //TODO possible optimization for NodeType based filtering
+ return true;
+ }
+
+ @Override
+ public boolean index(NodeStateEntry entry) throws IOException,
CommitFailedException {
+ if (getFilterResult(entry.getPath()) != PathFilter.Result.INCLUDE) {
+ return false;
+ }
+
+ IndexDefinition.IndexingRule indexingRule =
definition.getApplicableIndexingRule(entry.getNodeState());
+
+ if (indexingRule == null) {
+ return false;
+ }
+ ElasticDocumentMaker maker = newDocumentMaker(indexingRule,
entry.getPath());
+
+ ElasticDocument doc = maker.makeDocument(entry.getNodeState());
+
+ if (doc != null) {
+ writeToIndex(doc, entry.getPath());
+ progressReporter.indexUpdate(definition.getIndexPath());
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public boolean indexesRelativeNodes() {
+ return definition.indexesRelativeNodes();
+ }
+
+ @Override
+ public Set<String> getRelativeIndexedNodeNames() {
+ return definition.getRelativeNodeNames();
+ }
+
+ @Override
+ public void close() throws IOException {
+ indexWriter.close(System.currentTimeMillis());
+ }
+
+ private PathFilter.Result getFilterResult(String path) {
+ return definition.getPathFilter().filter(path);
+ }
+
+ private void writeToIndex(ElasticDocument doc, String path) throws
IOException {
+ indexWriter.updateDocument(path, doc);
+ }
+
+ private ElasticDocumentMaker newDocumentMaker(IndexDefinition.IndexingRule
indexingRule, String path) {
+ return new ElasticDocumentMaker(binaryTextExtractor, definition,
+ indexingRule,
+ path);
+ }
+}
Added:
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexerProvider.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexerProvider.java?rev=1885742&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexerProvider.java
(added)
+++
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexerProvider.java
Thu Jan 21 06:48:34 2021
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.indexer.document;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.index.IndexHelper;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticConnection;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.elastic.index.ElasticDocument;
+import
org.apache.jackrabbit.oak.plugins.index.elastic.index.ElasticIndexWriterFactory;
+import
org.apache.jackrabbit.oak.plugins.index.progress.IndexingProgressReporter;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+import
org.apache.jackrabbit.oak.plugins.index.search.spi.binary.FulltextBinaryTextExtractor;
+import
org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriter;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.jetbrains.annotations.NotNull;
+
+import org.jetbrains.annotations.Nullable;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+
+public class ElasticIndexerProvider implements NodeStateIndexerProvider {
+ private final ExtractedTextCache textCache =
+ new ExtractedTextCache(FileUtils.ONE_MB * 5,
TimeUnit.HOURS.toSeconds(5));
+ private final IndexHelper indexHelper;
+ private final ElasticIndexWriterFactory indexWriterFactory;
+ private final ElasticConnection coordinate;
+
+ public ElasticIndexerProvider(IndexHelper indexHelper, ElasticConnection
coordinate) {
+ this.indexHelper = indexHelper;
+ this.indexWriterFactory = new ElasticIndexWriterFactory(coordinate);
+ this.coordinate = coordinate;
+ }
+
+
+ @Override
+ public @Nullable NodeStateIndexer getIndexer(@NotNull String type,
@NotNull String indexPath, @NotNull NodeBuilder definition, @NotNull NodeState
root, IndexingProgressReporter progressReporter) {
+ if
(!ElasticIndexDefinition.TYPE_ELASTICSEARCH.equals(definition.getString(TYPE_PROPERTY_NAME)))
{
+ return null;
+ }
+ ElasticIndexDefinition idxDefinition = (ElasticIndexDefinition) new
ElasticIndexDefinition.Builder(coordinate.getIndexPrefix()).
+
root(root).indexPath(indexPath).defn(definition.getNodeState()).reindex().build();
+
+ FulltextIndexWriter<ElasticDocument> indexWriter =
indexWriterFactory.newInstance(idxDefinition, definition, CommitInfo.EMPTY,
true);
+ FulltextBinaryTextExtractor textExtractor = new
FulltextBinaryTextExtractor(textCache, idxDefinition, true);
+ return new ElasticIndexer(idxDefinition, textExtractor, definition,
progressReporter, indexWriter);
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+}
Added:
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/AvailableElasticModes.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/AvailableElasticModes.java?rev=1885742&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/AvailableElasticModes.java
(added)
+++
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/AvailableElasticModes.java
Thu Jan 21 06:48:34 2021
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.jackrabbit.oak.index.ElasticIndexCommand;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.run.commons.Modes;
+
+/*
+Avaialble modes for elastic. Add new elastic operations/commands to be
supported here.
+ */
+public final class AvailableElasticModes {
+ // list of available Modes for the tool
+ public static final Modes MODES = new Modes(
+ ImmutableMap.<String, Command>builder()
+ .put("index", new ElasticIndexCommand())
+ .build());
+}
Added:
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1885742&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/Main.java
(added)
+++
jackrabbit/oak/trunk/oak-run-elastic/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Thu Jan 21 06:48:34 2021
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.run.commons.Utils;
+
+import java.util.Locale;
+
+import static java.util.Arrays.copyOfRange;
+import static org.apache.jackrabbit.oak.run.AvailableElasticModes.MODES;
+
+/*
+Entry point for the oak run elastic utility
+ */
+public final class Main {
+ private Main() {
+ // Prevent instantiation.
+ }
+
+ public static void main(String... args) throws Exception {
+ Utils.printProductInfo(
+ args,
+
Main.class.getResourceAsStream("/META-INF/maven/org.apache.jackrabbit/oak-run-elastic/pom.properties"));
+
+ Command command = MODES.getCommand("help");
+
+ if (args.length > 0) {
+ command = MODES.getCommand(args[0].toLowerCase(Locale.ENGLISH));
+
+ if (command == null) {
+ command = MODES.getCommand("help");
+ }
+
+ args = copyOfRange(args, 1, args.length);
+ }
+
+ command.execute(args);
+ }
+}
Added:
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexerTest.java?rev=1885742&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexerTest.java
(added)
+++
jackrabbit/oak/trunk/oak-run-elastic/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/ElasticIndexerTest.java
Thu Jan 21 06:48:34 2021
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.index.indexer.document;
+
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticConnection;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexDefinition;
+import
org.apache.jackrabbit.oak.plugins.index.elastic.index.ElasticIndexWriterFactory;
+import
org.apache.jackrabbit.oak.plugins.index.elastic.util.ElasticIndexDefinitionBuilder;
+import
org.apache.jackrabbit.oak.plugins.index.progress.IndexingProgressReporter;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import
org.apache.jackrabbit.oak.plugins.index.search.spi.binary.FulltextBinaryTextExtractor;
+import
org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriter;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT;
+
+public class ElasticIndexerTest {
+
+ private NodeState root = INITIAL_CONTENT;
+
+ @Test
+ public void nodeIndexed_WithIncludedPaths() throws Exception {
+ ElasticIndexDefinitionBuilder idxb = new
ElasticIndexDefinitionBuilder();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ idxb.includedPaths("/content");
+
+ NodeState defn = idxb.build();
+ IndexDefinition idxDefn = new ElasticIndexDefinition(root, defn,
"/oak:index/testIndex", "testPrefix");
+
+ NodeBuilder builder = root.builder();
+
+ FulltextIndexWriter indexWriter = new
ElasticIndexWriterFactory(mock(ElasticConnection.class)).newInstance(idxDefn,
defn.builder(), CommitInfo.EMPTY, true);
+ ElasticIndexer indexer = new ElasticIndexer(idxDefn,
mock(FulltextBinaryTextExtractor.class), builder,
+ mock(IndexingProgressReporter.class), indexWriter);
+
+ NodeState testNode = EMPTY_NODE.builder().setProperty("foo",
"bar").getNodeState();
+
+ assertTrue(indexer.index(new NodeStateEntry(testNode, "/content/x")));
+ assertFalse(indexer.index(new NodeStateEntry(testNode, "/x")));
+ assertFalse(indexer.index(new NodeStateEntry(testNode, "/")));
+ }
+
+}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java?rev=1885742&r1=1885741&r2=1885742&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
Thu Jan 21 06:48:34 2021
@@ -53,8 +53,8 @@ public class IndexOptions implements Opt
private final OptionSpec<Void> importIndex;
private final OptionSpec<Void> docTraversal;
private final OptionSpec<Integer> consistencyCheck;
- private OptionSet options;
- private final Set<OptionSpec> actionOpts;
+ protected OptionSet options;
+ protected final Set<OptionSpec> actionOpts;
private final OptionSpec<String> indexPaths;
private final OptionSpec<String> checkpoint;
private final Set<String> operationNames;
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java?rev=1885742&r1=1885741&r2=1885742&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
Thu Jan 21 06:48:34 2021
@@ -66,7 +66,7 @@ public class OutOfBandIndexer implements
*/
public static final String LOCAL_INDEX_ROOT_DIR = "indexes";
- private final Closer closer = Closer.create();
+ protected final Closer closer = Closer.create();
private final IndexHelper indexHelper;
private NodeStore copyOnWriteStore;
private IndexerSupport indexerSupport;
@@ -145,7 +145,7 @@ public class OutOfBandIndexer implements
copyOnWriteStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
- private IndexEditorProvider createIndexEditorProvider() throws IOException
{
+ protected IndexEditorProvider createIndexEditorProvider() throws
IOException {
IndexEditorProvider lucene = createLuceneEditorProvider();
IndexEditorProvider property = createPropertyEditorProvider();
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java?rev=1885742&r1=1885741&r2=1885742&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
Thu Jan 21 06:48:34 2021
@@ -62,7 +62,7 @@ import static org.apache.jackrabbit.oak.
public class DocumentStoreIndexer implements Closeable{
private final Logger log = LoggerFactory.getLogger(getClass());
private final Logger traversalLog =
LoggerFactory.getLogger(DocumentStoreIndexer.class.getName()+".traversal");
- private final Closer closer = Closer.create();
+ protected final Closer closer = Closer.create();
private final IndexHelper indexHelper;
private final List<NodeStateIndexerProvider> indexerProviders;
private final IndexerSupport indexerSupport;
@@ -204,7 +204,7 @@ public class DocumentStoreIndexer implem
return new CompositeIndexer(indexers);
}
- private List<NodeStateIndexerProvider> createProviders() throws
IOException {
+ protected List<NodeStateIndexerProvider> createProviders() throws
IOException {
List<NodeStateIndexerProvider> providers = ImmutableList.of(
createLuceneIndexProvider()
);
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java?rev=1885742&r1=1885741&r2=1885742&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
Thu Jan 21 06:48:34 2021
@@ -37,7 +37,7 @@ import java.util.Set;
import static
org.apache.jackrabbit.oak.plugins.index.elastic.util.ElasticIndexUtils.toDoubles;
-class ElasticDocument {
+public class ElasticDocument {
private static final Logger LOG =
LoggerFactory.getLogger(ElasticDocument.class);
private final String path;
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocumentMaker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocumentMaker.java?rev=1885742&r1=1885741&r2=1885742&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocumentMaker.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocumentMaker.java
Thu Jan 21 06:48:34 2021
@@ -34,9 +34,9 @@ import org.jetbrains.annotations.Nullabl
import java.io.IOException;
import java.util.List;
-class ElasticDocumentMaker extends FulltextDocumentMaker<ElasticDocument> {
+public class ElasticDocumentMaker extends
FulltextDocumentMaker<ElasticDocument> {
- ElasticDocumentMaker(@Nullable FulltextBinaryTextExtractor textExtractor,
+ public ElasticDocumentMaker(@Nullable FulltextBinaryTextExtractor
textExtractor,
@NotNull IndexDefinition definition,
IndexDefinition.IndexingRule indexingRule, @NotNull
String path) {
super(textExtractor, definition, indexingRule, path);
Modified:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriterFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriterFactory.java?rev=1885742&r1=1885741&r2=1885742&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriterFactory.java
(original)
+++
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriterFactory.java
Thu Jan 21 06:48:34 2021
@@ -24,10 +24,10 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.jetbrains.annotations.NotNull;
-class ElasticIndexWriterFactory implements
FulltextIndexWriterFactory<ElasticDocument> {
+public class ElasticIndexWriterFactory implements
FulltextIndexWriterFactory<ElasticDocument> {
private final ElasticConnection elasticConnection;
- ElasticIndexWriterFactory(@NotNull ElasticConnection elasticConnection) {
+ public ElasticIndexWriterFactory(@NotNull ElasticConnection
elasticConnection) {
this.elasticConnection = elasticConnection;
}
Modified: jackrabbit/oak/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/pom.xml?rev=1885742&r1=1885741&r2=1885742&view=diff
==============================================================================
--- jackrabbit/oak/trunk/pom.xml (original)
+++ jackrabbit/oak/trunk/pom.xml Thu Jan 21 06:48:34 2021
@@ -81,6 +81,7 @@
<module>oak-benchmarks-lucene</module>
<module>oak-benchmarks-solr</module>
<module>oak-benchmarks-elastic</module>
+ <module>oak-run-elastic</module>
</modules>
<scm>
