This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-4519
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/TIKA-4519 by this push:
new 57c4cf7e1 TIKA-4519 -- checkpoint moving into factories as plugins
57c4cf7e1 is described below
commit 57c4cf7e16e827cca804ea637678e65e1dd20a28
Author: tallison <[email protected]>
AuthorDate: Mon Nov 3 17:39:21 2025 -0500
TIKA-4519 -- checkpoint moving into factories as plugins
---
.../apache/tika/plugins/AbstractTikaPlugin.java | 14 +++++++
.../java/org/apache/tika/plugins/PluginConfig.java | 2 +-
.../org/apache/tika/plugins/PluginConfigs.java | 15 ++++---
.../java/org/apache/tika/plugins/TikaPlugin.java | 8 ++++
.../tika/pipes/emitter/fs/FileSystemEmitter.java | 16 ++++----
.../pipes/emitter/fs/FileSystemEmitterFactory.java | 21 ++++++++++
.../tika/pipes/fetcher/fs/FileSystemFetcher.java | 44 +++++++++++++-------
.../pipes/fetcher/fs/FileSystemFetcherFactory.java | 19 +++++++++
.../fetcher/fs/config/FileSystemFetcherConfig.java | 12 +++++-
.../src/main/resources/fetcher-plugin.properties | 21 ----------
.../pipes/fetcher/fs/FileSystemFetcherTest.java | 4 +-
.../apache/tika/pipes/api/TikaPluginFactory.java | 14 +++++++
.../tika/pipes/api/emitter/AbstractEmitter.java | 28 +++----------
.../pipes/api/emitter/AbstractStreamEmitter.java | 24 +++--------
.../org/apache/tika/pipes/api/emitter/Emitter.java | 11 +----
.../tika/pipes/api/emitter/EmitterFactory.java | 8 ++++
.../tika/pipes/api/fetcher/AbstractFetcher.java | 47 ----------------------
.../org/apache/tika/pipes/api/fetcher/Fetcher.java | 11 +----
.../tika/pipes/api/fetcher/FetcherFactory.java | 7 ++++
.../tika/pipes/api/fetcher/RangeFetcher.java | 1 -
.../apache/tika/pipes/core/PassbackFilterTest.java | 8 ++--
.../apache/tika/pipes/core/PipesServerTest.java | 2 +-
.../tika/pipes/core/TikaPipesConfigTest.java | 4 +-
.../apache/tika/pipes/core/async/MockFetcher.java | 13 +++---
.../tika/pipes/core/emitter/MockEmitter.java | 16 +++-----
.../tika/pipes/core/fetcher/MockFetcher.java | 13 +++---
.../test/resources/configs/fetchers-emitters.json | 14 +++++--
.../apache/tika/pipes/core/PipesPluginsConfig.java | 26 ++++++++----
.../org/apache/tika/pipes/core/PipesServer.java | 14 +++----
.../apache/tika/pipes/core/async/AsyncEmitter.java | 2 +-
.../tika/pipes/core/async/AsyncProcessor.java | 2 +-
.../apache/tika/pipes/core/async/EmitDataPair.java | 2 +-
.../apache/tika/pipes/core/emitter/EmitKey.java | 16 ++++----
.../tika/pipes/core/emitter/EmitterManager.java | 26 ++++++------
.../tika/pipes/core/emitter/EmptyEmitter.java | 13 +-----
.../tika/pipes/core/fetcher/EmptyFetcher.java | 13 ++----
.../apache/tika/pipes/core/fetcher/FetchKey.java | 22 +++++-----
.../tika/pipes/core/fetcher/FetcherManager.java | 27 +++++++------
.../pipes/core/pipesiterator/PipesIterator.java | 20 ++++-----
.../serialization/FetchEmitTupleDeserializer.java | 4 +-
.../serialization/FetchEmitTupleSerializer.java | 4 +-
.../filelist/FileListPipesIterator.java | 8 ++--
.../pipesiterator/fs/FileSystemPipesIterator.java | 6 +--
.../tika/pipes/core/TikaPipesConfigTest.java | 4 +-
.../filelist/FileListPipesIteratorTest.java | 16 ++++----
.../fs/FileSystemPipesIteratorTest.java | 2 +-
.../src/test/resources/configs/fetchers.json | 7 +++-
.../test/resources/configs/tika-config-broken.xml | 2 +-
.../apache/tika/config/pipes-iterator-config.xml | 4 +-
.../tika/config/pipes-iterator-multiple-config.xml | 8 ++--
.../serialization/PluginsConfigDeserializer.java | 6 +--
.../serialization/PluginsConfigSerializer.java | 2 +-
.../tika/serialization/PluginsConfigTest.java | 12 +++---
53 files changed, 336 insertions(+), 329 deletions(-)
diff --git
a/tika-core/src/main/java/org/apache/tika/plugins/AbstractTikaPlugin.java
b/tika-core/src/main/java/org/apache/tika/plugins/AbstractTikaPlugin.java
new file mode 100644
index 000000000..8762ee861
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/plugins/AbstractTikaPlugin.java
@@ -0,0 +1,14 @@
+package org.apache.tika.plugins;
+
+public class AbstractTikaPlugin implements TikaPlugin {
+
+ protected final PluginConfig pluginConfig;
+
+ public AbstractTikaPlugin(PluginConfig pluginConfig) {
+ this.pluginConfig = pluginConfig;
+ }
+
+ public PluginConfig getPluginConfig() {
+ return pluginConfig;
+ }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/plugins/PluginConfig.java
b/tika-core/src/main/java/org/apache/tika/plugins/PluginConfig.java
index 532ee9d38..f4e178cd0 100644
--- a/tika-core/src/main/java/org/apache/tika/plugins/PluginConfig.java
+++ b/tika-core/src/main/java/org/apache/tika/plugins/PluginConfig.java
@@ -16,6 +16,6 @@
*/
package org.apache.tika.plugins;
-public record PluginConfig(String pluginId, String jsonConfig) {
+public record PluginConfig(String id, String factoryPluginId, String
jsonConfig) {
}
diff --git a/tika-core/src/main/java/org/apache/tika/plugins/PluginConfigs.java
b/tika-core/src/main/java/org/apache/tika/plugins/PluginConfigs.java
index fbed67343..d73b2302c 100644
--- a/tika-core/src/main/java/org/apache/tika/plugins/PluginConfigs.java
+++ b/tika-core/src/main/java/org/apache/tika/plugins/PluginConfigs.java
@@ -19,6 +19,7 @@ package org.apache.tika.plugins;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
public class PluginConfigs {
@@ -33,14 +34,18 @@ public class PluginConfigs {
}
public void add(PluginConfig pluginConfig) {
- if (pluginConfigs.containsKey(pluginConfig.pluginId())) {
- throw new IllegalArgumentException("Can't overwrite existing
plugin for id: " + pluginConfig.pluginId());
+ if (pluginConfigs.containsKey(pluginConfig.id())) {
+ throw new IllegalArgumentException("Can't overwrite existing
plugin for id: " + pluginConfig.factoryPluginId());
}
- pluginConfigs.put(pluginConfig.pluginId(), pluginConfig);
+ pluginConfigs.put(pluginConfig.factoryPluginId(), pluginConfig);
}
- public Optional<PluginConfig> get(String pluginId) {
- return Optional.ofNullable(pluginConfigs.get(pluginId));
+ public Optional<PluginConfig> get(String id) {
+ return Optional.ofNullable(pluginConfigs.get(id));
+ }
+
+ public Set<String> ids() {
+ return pluginConfigs.keySet();
}
}
diff --git a/tika-core/src/main/java/org/apache/tika/plugins/TikaPlugin.java
b/tika-core/src/main/java/org/apache/tika/plugins/TikaPlugin.java
new file mode 100644
index 000000000..aa1d6b330
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/plugins/TikaPlugin.java
@@ -0,0 +1,8 @@
+package org.apache.tika.plugins;
+
+/**
+ * This is an interface for plugins created by TikaPluginFactory
+ */
+public interface TikaPlugin {
+ PluginConfig getPluginConfig();
+}
diff --git
a/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
b/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
index 9b874ef1c..66ea2d76b 100644
---
a/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
+++
b/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitter.java
@@ -51,21 +51,23 @@ import org.apache.tika.utils.StringUtils;
* <pre class="prettyprint">
* </pre>
*/
-@Extension
public class FileSystemEmitter extends AbstractStreamEmitter {
private static final Logger LOG =
LoggerFactory.getLogger(FileSystemEmitter.class);
+ public static FileSystemEmitter build(PluginConfig pluginConfig) throws
TikaConfigException, IOException {
+ FileSystemEmitter emitter = new FileSystemEmitter(pluginConfig);
+ emitter.configure();
+ return emitter;
+ }
private FileSystemEmitterConfig fileSystemEmitterConfig;
- public FileSystemEmitter() throws IOException {
- super();
+ public FileSystemEmitter(PluginConfig pluginConfig) {
+ super(pluginConfig);
}
- @Override
- public void configure(PluginConfig pluginConfig) throws
TikaConfigException, IOException {
- checkPluginId(pluginConfig.pluginId());
+ private void configure() throws TikaConfigException, IOException {
fileSystemEmitterConfig =
FileSystemEmitterConfig.load(pluginConfig.jsonConfig());
checkConfig(fileSystemEmitterConfig);
}
@@ -155,7 +157,7 @@ public class FileSystemEmitter extends
AbstractStreamEmitter {
FileSystemEmitterConfig config = fileSystemEmitterConfig;
PluginConfigs pluginConfigs = parseContext.get(PluginConfigs.class);
if (pluginConfigs != null) {
- Optional<PluginConfig> pluginConfigOpt =
pluginConfigs.get(getPluginId());
+ Optional<PluginConfig> pluginConfigOpt =
pluginConfigs.get(getPluginConfig().id());
if (pluginConfigOpt.isPresent()) {
config =
FileSystemEmitterConfig.load(pluginConfigOpt.get().jsonConfig());
checkConfig(config);
diff --git
a/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitterFactory.java
b/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitterFactory.java
new file mode 100644
index 000000000..baca9e5a3
--- /dev/null
+++
b/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitterFactory.java
@@ -0,0 +1,21 @@
+package org.apache.tika.pipes.emitter.fs;
+
+import java.io.IOException;
+
+import org.pf4j.Extension;
+
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.pipes.api.emitter.Emitter;
+import org.apache.tika.pipes.api.emitter.EmitterFactory;
+import org.apache.tika.pipes.api.fetcher.Fetcher;
+import org.apache.tika.pipes.api.fetcher.FetcherFactory;
+import org.apache.tika.plugins.PluginConfig;
+
+@Extension
+public class FileSystemEmitterFactory implements EmitterFactory {
+
+ @Override
+ public Emitter buildPlugin(PluginConfig pluginConfig) throws IOException,
TikaConfigException {
+ return FileSystemEmitter.build(pluginConfig);
+ }
+}
diff --git
a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcher.java
b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcher.java
index 3f6d0ffde..880011f26 100644
---
a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcher.java
+++
b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcher.java
@@ -27,7 +27,6 @@ import java.nio.file.attribute.FileTime;
import java.util.Date;
import java.util.Optional;
-import org.pf4j.Extension;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +38,8 @@ import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.api.fetcher.AbstractFetcher;
+import org.apache.tika.plugins.AbstractTikaPlugin;
+import org.apache.tika.pipes.api.fetcher.Fetcher;
import org.apache.tika.pipes.fetcher.fs.config.FileSystemFetcherConfig;
import org.apache.tika.plugins.PluginConfig;
import org.apache.tika.plugins.PluginConfigs;
@@ -57,25 +57,27 @@ import org.apache.tika.utils.StringUtils;
* </pre>
*/
-@Extension
-public class FileSystemFetcher extends AbstractFetcher {
+public class FileSystemFetcher extends AbstractTikaPlugin implements Fetcher {
- public FileSystemFetcher() throws IOException {
- super();
+
+
+ public static FileSystemFetcher build(PluginConfig pluginConfig) throws
TikaConfigException, IOException {
+ FileSystemFetcher fetcher = new FileSystemFetcher(pluginConfig);
+ fetcher.configure();
+ return fetcher;
}
+ private static final String PLUGIN_ID = "file-system-fetcher";
+
private static final Logger LOG =
LoggerFactory.getLogger(FileSystemFetcher.class);
- private FileSystemFetcherConfig defaultFileSystemFetcherConfig = new
FileSystemFetcherConfig();
+ private FileSystemFetcherConfig defaultFileSystemFetcherConfig;
- static boolean isDescendant(Path root, Path descendant) {
- return descendant.toAbsolutePath().normalize()
- .startsWith(root.toAbsolutePath().normalize());
+ public FileSystemFetcher(PluginConfig pluginConfig) {
+ super(pluginConfig);
}
- @Override
- public void configure(PluginConfig pluginConfig) throws IOException,
TikaConfigException {
- checkPluginId(pluginConfig.pluginId());
+ private void configure() throws IOException, TikaConfigException {
defaultFileSystemFetcherConfig =
FileSystemFetcherConfig.load(pluginConfig.jsonConfig());
checkConfig(defaultFileSystemFetcherConfig);
}
@@ -90,10 +92,10 @@ public class FileSystemFetcher extends AbstractFetcher {
FileSystemFetcherConfig config = defaultFileSystemFetcherConfig;
PluginConfigs pluginConfigManager =
parseContext.get(PluginConfigs.class);
if (pluginConfigManager != null) {
- Optional<PluginConfig> pluginConfigOpt =
pluginConfigManager.get(getPluginId());
+ Optional<PluginConfig> pluginConfigOpt =
pluginConfigManager.get(getPluginConfig().id());
if (pluginConfigOpt.isPresent()) {
PluginConfig pluginConfig = pluginConfigOpt.get();
- checkPluginId(pluginConfig.pluginId());
+ checkPluginId(pluginConfig.factoryPluginId());
config =
FileSystemFetcherConfig.load(pluginConfig.jsonConfig());
checkConfig(config);
}
@@ -121,6 +123,12 @@ public class FileSystemFetcher extends AbstractFetcher {
return TikaInputStream.get(p, metadata);
}
+ private void checkPluginId(String pluginId) {
+ if (! PLUGIN_ID.equals(pluginId)) {
+ throw new IllegalStateException("Plugin id=" + pluginId + " needs
to =" + PLUGIN_ID);
+ }
+ }
+
private void updateFileSystemMetadata(Path p, Metadata metadata,
FileSystemFetcherConfig config) throws IOException {
if (! config.isExtractFileSystemMetadata()) {
return;
@@ -165,4 +173,10 @@ public class FileSystemFetcher extends AbstractFetcher {
"base path must not contain \u0000. " + "Seriously, what
were you thinking?");
}
}
+
+ static boolean isDescendant(Path root, Path descendant) {
+ return descendant.toAbsolutePath().normalize()
+ .startsWith(root.toAbsolutePath().normalize());
+ }
+
}
diff --git
a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherFactory.java
b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherFactory.java
new file mode 100644
index 000000000..fa8bb433d
--- /dev/null
+++
b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherFactory.java
@@ -0,0 +1,19 @@
+package org.apache.tika.pipes.fetcher.fs;
+
+import java.io.IOException;
+
+import org.pf4j.Extension;
+
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.pipes.api.fetcher.Fetcher;
+import org.apache.tika.pipes.api.fetcher.FetcherFactory;
+import org.apache.tika.plugins.PluginConfig;
+
+@Extension
+public class FileSystemFetcherFactory implements FetcherFactory {
+
+ @Override
+ public Fetcher buildPlugin(PluginConfig pluginConfig) throws IOException,
TikaConfigException {
+ return FileSystemFetcher.build(pluginConfig);
+ }
+}
diff --git
a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/config/FileSystemFetcherConfig.java
b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/config/FileSystemFetcherConfig.java
index defaebf7e..ff1e06608 100644
---
a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/config/FileSystemFetcherConfig.java
+++
b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/config/FileSystemFetcherConfig.java
@@ -18,14 +18,22 @@ package org.apache.tika.pipes.fetcher.fs.config;
import java.io.IOException;
+import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.tika.exception.TikaConfigException;
+
public class FileSystemFetcherConfig {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- public static FileSystemFetcherConfig load(String json) throws IOException
{
- return OBJECT_MAPPER.readValue(json, FileSystemFetcherConfig.class);
+ public static FileSystemFetcherConfig load(String json) throws
IOException, TikaConfigException {
+ try {
+ return OBJECT_MAPPER.readValue(json,
FileSystemFetcherConfig.class);
+ } catch (JacksonException e) {
+ throw new TikaConfigException("problem w json", e);
+ }
}
private String basePath;
diff --git
a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/resources/fetcher-plugin.properties
b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/resources/fetcher-plugin.properties
deleted file mode 100644
index b2488f75d..000000000
---
a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/resources/fetcher-plugin.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-plugin.id=file-system-fetcher
-plugin.class=org.apache.tika.pipes.fetcher.fs.FileSystemFetcherPlugin
-plugin.version=4.0.0-SNAPSHOT
-plugin.provider=Local File System Fetcher
-plugin.description=Capable of fetching the local file system
diff --git
a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/test/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherTest.java
b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/test/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherTest.java
index e87372a23..0e4767bb1 100644
---
a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/test/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherTest.java
+++
b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/test/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherTest.java
@@ -47,9 +47,9 @@ public class FileSystemFetcherTest {
@Test
public void testNullByte() throws Exception {
- FileSystemFetcher f = new FileSystemFetcher();
assertThrows(TikaConfigException.class, () -> {
- f.configure(new PluginConfig("test", "{
\"basePath\":\"bad\u0000path\"}"));
+ PluginConfig pluginConfig = new PluginConfig("test", "test", "{
\"basePath\":\"bad\u0000path\"}");
+ FileSystemFetcher f = FileSystemFetcher.build(pluginConfig);
});
}
}
diff --git
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/TikaPluginFactory.java
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/TikaPluginFactory.java
new file mode 100644
index 000000000..b98b0830d
--- /dev/null
+++
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/TikaPluginFactory.java
@@ -0,0 +1,14 @@
+package org.apache.tika.pipes.api;
+
+import java.io.IOException;
+
+import org.pf4j.ExtensionPoint;
+
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.plugins.PluginConfig;
+import org.apache.tika.plugins.TikaPlugin;
+
+public interface TikaPluginFactory<T extends TikaPlugin> extends
ExtensionPoint {
+
+ T buildPlugin(PluginConfig pluginConfig) throws IOException,
TikaConfigException;
+}
diff --git
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/AbstractEmitter.java
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/AbstractEmitter.java
index 46fa7c92e..73f556df2 100644
---
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/AbstractEmitter.java
+++
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/AbstractEmitter.java
@@ -17,26 +17,15 @@
package org.apache.tika.pipes.api.emitter;
import java.io.IOException;
-import java.io.InputStream;
import java.util.List;
-import java.util.Properties;
-import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.plugins.AbstractTikaPlugin;
+import org.apache.tika.plugins.PluginConfig;
-public abstract class AbstractEmitter implements Emitter {
+public abstract class AbstractEmitter extends AbstractTikaPlugin implements
Emitter {
- private final String pluginId;
- public AbstractEmitter() throws IOException {
- Properties properties = new Properties();
- try (InputStream is =
this.getClass().getResourceAsStream("/emitter-plugin.properties")) {
- properties.load(is);
- }
- pluginId = (String) properties.get("plugin.id");
- }
-
- @Override
- public String getPluginId() {
- return pluginId;
+ public AbstractEmitter(PluginConfig pluginConfig) throws IOException {
+ super(pluginConfig);
}
@Override
@@ -45,11 +34,4 @@ public abstract class AbstractEmitter implements Emitter {
emit(item.getEmitKey(), item.getMetadataList(),
item.getParseContext());
}
}
-
-
- protected void checkPluginId(String pluginId) throws TikaConfigException {
- if (! getPluginId().equals(pluginId)) {
- throw new TikaConfigException("Plugin id mismatch: " +
getPluginId() + " <> " + pluginId);
- }
- }
}
diff --git
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/AbstractStreamEmitter.java
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/AbstractStreamEmitter.java
index 0f12535d3..5011ed01e 100644
---
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/AbstractStreamEmitter.java
+++
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/AbstractStreamEmitter.java
@@ -22,27 +22,13 @@ import java.util.List;
import java.util.Properties;
import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.plugins.AbstractTikaPlugin;
+import org.apache.tika.plugins.PluginConfig;
-public abstract class AbstractStreamEmitter implements StreamEmitter {
+public abstract class AbstractStreamEmitter extends AbstractTikaPlugin
implements StreamEmitter {
- private final String pluginId;
- public AbstractStreamEmitter() throws IOException {
- Properties properties = new Properties();
- try (InputStream is =
this.getClass().getResourceAsStream("/emitter-plugin.properties")) {
- properties.load(is);
- }
- pluginId = (String) properties.get("plugin.id");
- }
-
- @Override
- public String getPluginId() {
- return pluginId;
- }
-
- protected void checkPluginId(String pluginId) throws TikaConfigException {
- if (! getPluginId().equals(pluginId)) {
- throw new TikaConfigException("Plugin id mismatch: " +
getPluginId() + " <> " + pluginId);
- }
+ public AbstractStreamEmitter(PluginConfig pluginConfig) {
+ super(pluginConfig);
}
@Override
diff --git
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/Emitter.java
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/Emitter.java
index dad43467a..76fee4754 100644
---
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/Emitter.java
+++
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/Emitter.java
@@ -19,18 +19,11 @@ package org.apache.tika.pipes.api.emitter;
import java.io.IOException;
import java.util.List;
-import org.pf4j.ExtensionPoint;
-
-import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.plugins.PluginConfig;
-
-public interface Emitter extends ExtensionPoint {
-
- void configure(PluginConfig pluginConfig) throws TikaConfigException,
IOException;
+import org.apache.tika.plugins.TikaPlugin;
- String getPluginId();
+public interface Emitter extends TikaPlugin {
void emit(String emitKey, List<Metadata> metadataList, ParseContext
parseContext) throws IOException;
diff --git
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/EmitterFactory.java
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/EmitterFactory.java
new file mode 100644
index 000000000..1b1768abe
--- /dev/null
+++
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/emitter/EmitterFactory.java
@@ -0,0 +1,8 @@
+package org.apache.tika.pipes.api.emitter;
+
+import org.apache.tika.pipes.api.TikaPluginFactory;
+import org.apache.tika.pipes.api.fetcher.Fetcher;
+
+public interface EmitterFactory extends TikaPluginFactory<Emitter> {
+
+}
diff --git
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/AbstractFetcher.java
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/AbstractFetcher.java
deleted file mode 100644
index 6d796b584..000000000
---
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/AbstractFetcher.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.tika.pipes.api.fetcher;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Properties;
-
-import org.apache.tika.exception.TikaConfigException;
-
-public abstract class AbstractFetcher implements Fetcher {
-
- private final String pluginId;
-
- public AbstractFetcher() throws IOException {
- Properties properties = new Properties();
- try (InputStream is =
this.getClass().getResourceAsStream("/fetcher-plugin.properties")) {
- properties.load(is);
- }
- pluginId = (String) properties.get("plugin.id");
- }
-
- @Override
- public String getPluginId() {
- return pluginId;
- }
-
- protected void checkPluginId(String pluginId) throws TikaConfigException {
- if (! getPluginId().equals(pluginId)) {
- throw new TikaConfigException("Plugin id mismatch: " +
getPluginId() + " <> " + pluginId);
- }
- }
-}
diff --git
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/Fetcher.java
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/Fetcher.java
index 07469022e..70c66fd5c 100644
---
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/Fetcher.java
+++
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/Fetcher.java
@@ -19,13 +19,10 @@ package org.apache.tika.pipes.api.fetcher;
import java.io.IOException;
import java.io.InputStream;
-import org.pf4j.ExtensionPoint;
-
-import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.plugins.PluginConfig;
+import org.apache.tika.plugins.TikaPlugin;
/**
* Interface for an object that will fetch an InputStream given
@@ -34,11 +31,7 @@ import org.apache.tika.plugins.PluginConfig;
* <p>
* Implementations of Fetcher must be thread safe.
*/
-public interface Fetcher extends ExtensionPoint {
-
- void configure(PluginConfig fetcherConfig) throws TikaConfigException,
IOException;
-
- String getPluginId();
+public interface Fetcher extends TikaPlugin {
InputStream fetch(String fetchKey, Metadata metadata, ParseContext
parseContext) throws TikaException, IOException;
}
diff --git
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/FetcherFactory.java
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/FetcherFactory.java
new file mode 100644
index 000000000..35d0f930c
--- /dev/null
+++
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/FetcherFactory.java
@@ -0,0 +1,7 @@
+package org.apache.tika.pipes.api.fetcher;
+
+import org.apache.tika.pipes.api.TikaPluginFactory;
+
+public interface FetcherFactory extends TikaPluginFactory<Fetcher> {
+
+}
diff --git
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/RangeFetcher.java
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/RangeFetcher.java
index 025b4cd82..a1f011b48 100644
---
a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/RangeFetcher.java
+++
b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/RangeFetcher.java
@@ -27,7 +27,6 @@ import org.apache.tika.parser.ParseContext;
* This class extracts a range of bytes from a given fetch key.
*/
public interface RangeFetcher extends Fetcher {
- //At some point, Tika 3.x?, we may want to add optional ranges to the
fetchKey?
default InputStream fetch(String fetchKey, long startOffset, long
endOffset, Metadata metadata)
throws TikaException, IOException {
diff --git
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/PassbackFilterTest.java
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/PassbackFilterTest.java
index a4eea1ce5..bd71194a5 100644
---
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/PassbackFilterTest.java
+++
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/PassbackFilterTest.java
@@ -47,8 +47,8 @@ import org.apache.tika.utils.StringUtils;
public class PassbackFilterTest {
private Path tmpDir;
- String fetcherPluginId = "file-system-fetcher";
- String emitterPluginId = "file-system-emitter";
+ String fetcherId = "fsf";
+ String emitterId = "fse";
String testPdfFile = "testOverlappingText.pdf";
private PipesClient pipesClient;
@@ -78,8 +78,8 @@ public class PassbackFilterTest {
ParseContext parseContext = new ParseContext();
parseContext.set(PassbackFilter.class, new MyPassbackFilter());
PipesResult pipesResult = pipesClient.process(
- new FetchEmitTuple(testPdfFile, new FetchKey(fetcherPluginId,
testPdfFile),
- new EmitKey(emitterPluginId, emitFileBase), new
Metadata(), parseContext,
+ new FetchEmitTuple(testPdfFile, new FetchKey(fetcherId,
testPdfFile),
+ new EmitKey(emitterId, emitFileBase), new Metadata(),
parseContext,
FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP));
assertEquals(PipesResult.STATUS.EMIT_SUCCESS_PASSBACK,
pipesResult.getStatus());
Assertions.assertNotNull(pipesResult
diff --git
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/PipesServerTest.java
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/PipesServerTest.java
index ccbfa4b5f..ba788e02c 100644
---
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/PipesServerTest.java
+++
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/PipesServerTest.java
@@ -73,7 +73,7 @@ public class PipesServerTest extends TikaTest {
pipesServer.initializeResources();
FetchEmitTuple fetchEmitTuple = new FetchEmitTuple("id",
- new FetchKey("file-system-fetcher", testDoc),
+ new FetchKey("fsf", testDoc),
new EmitKey("", ""));
Fetcher fetcher = FetcherManager.load(pipesConfig).getFetcher();
PipesServer.MetadataListAndEmbeddedBytes
diff --git
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
index 6f049652e..5a2ebf7d6 100644
---
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
+++
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
@@ -87,7 +87,7 @@ public class TikaPipesConfigTest extends
AbstractTikaConfigTest {
public void testPipesIterator() throws Exception {
PipesIterator it =
PipesIterator.build(getConfigFilePath("pipes-iterator-config.xml"));
- assertEquals("fs1", it.getFetcherName());
+ assertEquals("fs1", it.getFetcherId());
}
@Test
@@ -95,7 +95,7 @@ public class TikaPipesConfigTest extends
AbstractTikaConfigTest {
assertThrows(TikaConfigException.class, () -> {
PipesIterator it =
PipesIterator.build(getConfigFilePath("pipes-iterator-multiple-config.xml"));
- assertEquals("fs1", it.getFetcherName());
+ assertEquals("fs1", it.getFetcherId());
});
}
diff --git
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/async/MockFetcher.java
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/async/MockFetcher.java
index a6f05cbbd..6d28a44d2 100644
---
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/async/MockFetcher.java
+++
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/async/MockFetcher.java
@@ -25,10 +25,10 @@ import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.api.fetcher.AbstractFetcher;
+import org.apache.tika.pipes.api.fetcher.Fetcher;
import org.apache.tika.plugins.PluginConfig;
-public class MockFetcher extends AbstractFetcher {
+public class MockFetcher implements Fetcher {
private static final byte[] BYTES = ("<?xml version=\"1.0\"
encoding=\"UTF-8\" ?>" + "<mock>" +
"<metadata action=\"add\" name=\"dc:creator\">Nikolai
Lobachevsky</metadata>" +
@@ -38,13 +38,14 @@ public class MockFetcher extends AbstractFetcher {
super();
}
- @Override
- public void configure(PluginConfig fetcherConfig) throws
TikaConfigException, IOException {
- //no-op
- }
@Override
public InputStream fetch(String fetchKey, Metadata metadata, ParseContext
parseContext) throws TikaException, IOException {
return new ByteArrayInputStream(BYTES);
}
+
+ @Override
+ public PluginConfig getPluginConfig() {
+ return null;
+ }
}
diff --git
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/emitter/MockEmitter.java
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/emitter/MockEmitter.java
index 86fa68250..290163743 100644
---
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/emitter/MockEmitter.java
+++
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/emitter/MockEmitter.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.pf4j.Extension;
import org.apache.tika.config.Initializable;
@@ -36,7 +35,6 @@ import org.apache.tika.pipes.api.emitter.EmitData;
import org.apache.tika.pipes.api.emitter.Emitter;
import org.apache.tika.plugins.PluginConfig;
-@Extension
public class MockEmitter implements Initializable, Emitter {
public static ArrayBlockingQueue<EmitData> EMIT_DATA = new
ArrayBlockingQueue<>(10000);
@@ -48,6 +46,11 @@ public class MockEmitter implements Initializable, Emitter {
public MockEmitter() throws IOException {
}
+ @Override
+ public PluginConfig getPluginConfig() {
+ return null;
+ }
+
private static record MockEmitterConfig(boolean throwOnCheck) {
}
@@ -69,15 +72,6 @@ public class MockEmitter implements Initializable, Emitter {
}
- @Override
- public void configure(PluginConfig pluginConfig) throws
TikaConfigException, IOException {
- config = new ObjectMapper().readValue(pluginConfig.jsonConfig(),
MockEmitterConfig.class);
- }
-
- @Override
- public String getPluginId() {
- return "mock-emitter";
- }
@Override
public void emit(String emitKey, List<Metadata> metadataList, ParseContext
parseContext)
diff --git
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/fetcher/MockFetcher.java
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/fetcher/MockFetcher.java
index 9741d35d5..b9a94de05 100644
---
a/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/fetcher/MockFetcher.java
+++
b/tika-pipes/tika-pipes-core-tests/src/test/java/org/apache/tika/pipes/core/fetcher/MockFetcher.java
@@ -28,10 +28,10 @@ import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.api.fetcher.AbstractFetcher;
+import org.apache.tika.pipes.api.fetcher.Fetcher;
import org.apache.tika.plugins.PluginConfig;
-public class MockFetcher extends AbstractFetcher {
+public class MockFetcher implements Fetcher {
private Map<String, Param> params;
@@ -54,14 +54,15 @@ public class MockFetcher extends AbstractFetcher {
this.byteString = byteString;
}
- @Override
- public void configure(PluginConfig fetcherConfig) throws
TikaConfigException, IOException {
- //no-op
- }
@Override
public InputStream fetch(String fetchKey, Metadata metadata, ParseContext
parseContext) throws TikaException, IOException {
return byteString == null ? new ByteArrayInputStream(new byte[0]) :
new
ByteArrayInputStream(byteString.getBytes(StandardCharsets.UTF_8));
}
+
+ @Override
+ public PluginConfig getPluginConfig() {
+ return null;
+ }
}
diff --git
a/tika-pipes/tika-pipes-core-tests/src/test/resources/configs/fetchers-emitters.json
b/tika-pipes/tika-pipes-core-tests/src/test/resources/configs/fetchers-emitters.json
index 3e2409761..2c4c5a5e6 100644
---
a/tika-pipes/tika-pipes-core-tests/src/test/resources/configs/fetchers-emitters.json
+++
b/tika-pipes/tika-pipes-core-tests/src/test/resources/configs/fetchers-emitters.json
@@ -1,15 +1,21 @@
{
"plugins" : {
"fetchers": {
- "file-system-fetcher": {
+ "fsf": {
+ "factoryPluginId": "file-system-fetcher",
+ "config": {
"basePath": "FETCHERS_BASE_PATH",
"extractFileSystemMetadata": false
}
+ }
},
"emitters": {
- "file-system-emitter": {
- "basePath": "EMITTERS_BASE_PATH",
- "fileExtension": "json"
+ "fse": {
+ "factoryPluginId": "file-system-emitter",
+ "config": {
+ "basePath": "EMITTERS_BASE_PATH",
+ "fileExtension": "json"
+ }
}
}
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesPluginsConfig.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesPluginsConfig.java
index 6fa63b7a6..ec491cf19 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesPluginsConfig.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesPluginsConfig.java
@@ -25,6 +25,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
import java.util.Optional;
+import java.util.Set;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -68,9 +69,11 @@ public class PipesPluginsConfig {
Iterator<String> it = pluginsNode.fieldNames();
manager = new PluginConfigs();
while (it.hasNext()) {
- String pluginId = it.next();
- JsonNode jsonConfig = pluginsNode.get(pluginId);
- manager.add(new PluginConfig(pluginId, jsonConfig.toString()));
+ String id = it.next();
+ JsonNode configNode = pluginsNode.get(id);
+ String pluginId = configNode.get("factoryPluginId").asText();
+ JsonNode config = configNode.get("config");
+ manager.add(new PluginConfig(id, pluginId, config.toString()));
}
return manager;
}
@@ -92,18 +95,27 @@ public class PipesPluginsConfig {
this.pluginsDir = pluginsDir;
}
- public Optional<PluginConfig> getFetcherConfig(String pluginId) {
+
+ public PluginConfigs getFetcherConfig() {
+ return fetchers;
+ }
+
+ public PluginConfigs getEmitterConfig() {
+ return emitters;
+ }
+
+ public Optional<PluginConfig> getFetcherConfig(String id) {
if (fetchers == null) {
throw new IllegalArgumentException("fetchers element was not
loaded");
}
- return fetchers.get(pluginId);
+ return fetchers.get(id);
}
- public Optional<PluginConfig> getEmitterConfig(String pluginId) {
+ public Optional<PluginConfig> getEmitterConfig(String id) {
if (emitters == null) {
throw new IllegalArgumentException("emitters element was not
loaded");
}
- return emitters.get(pluginId);
+ return emitters.get(id);
}
public Optional<PluginConfig> getIteratorConfig(String pluginId) {
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesServer.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesServer.java
index ff520877b..747e39983 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesServer.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesServer.java
@@ -306,9 +306,9 @@ public class PipesServer implements Runnable {
Emitter emitter = null;
try {
- emitter = emitterManager.getEmitter(emitKey.getEmitterPluginId());
+ emitter = emitterManager.getEmitter(emitKey.getEmitterId());
} catch (IllegalArgumentException e) {
- String noEmitterMsg = getNoEmitterMsg(taskId,
emitKey.getEmitterPluginId());
+ String noEmitterMsg = getNoEmitterMsg(taskId,
emitKey.getEmitterId());
LOG.warn(noEmitterMsg);
write(STATUS.EMITTER_NOT_FOUND, noEmitterMsg);
return;
@@ -486,7 +486,7 @@ public class PipesServer implements Runnable {
injectUserMetadata(t.getMetadata(), parseData.getMetadataList());
EmitKey emitKey = t.getEmitKey();
if (StringUtils.isBlank(emitKey.getEmitKey())) {
- emitKey = new EmitKey(emitKey.getEmitterPluginId(),
t.getFetchKey().getFetchKey());
+ emitKey = new EmitKey(emitKey.getEmitterId(),
t.getFetchKey().getFetchKey());
t.setEmitKey(emitKey);
}
EmitDataImpl emitDataTuple = new
EmitDataImpl(t.getEmitKey().getEmitKey(), parseData.getMetadataList(), stack);
@@ -552,9 +552,9 @@ public class PipesServer implements Runnable {
private Fetcher getFetcher(FetchEmitTuple t) {
try {
- return
fetcherManager.getFetcher(t.getFetchKey().getFetcherPluginId());
+ return fetcherManager.getFetcher(t.getFetchKey().getFetcherId());
} catch (IllegalArgumentException e) {
- String noFetcherMsg =
getNoFetcherMsg(t.getFetchKey().getFetcherPluginId());
+ String noFetcherMsg =
getNoFetcherMsg(t.getFetchKey().getFetcherId());
LOG.warn(noFetcherMsg);
write(STATUS.FETCHER_NOT_FOUND, noFetcherMsg);
return null;
@@ -581,9 +581,9 @@ public class PipesServer implements Runnable {
return null;
}
- private String getNoFetcherMsg(String fetcherPluginId) {
+ private String getNoFetcherMsg(String fetcherId) {
StringBuilder sb = new StringBuilder();
- sb.append("Fetcher '").append(fetcherPluginId).append("'");
+ sb.append("Fetcher '").append(fetcherId).append("'");
sb.append(" not found.");
sb.append("\nThe configured FetcherManager supports:");
int i = 0;
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncEmitter.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncEmitter.java
index ab8e617e2..ccffd03de 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncEmitter.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncEmitter.java
@@ -109,7 +109,7 @@ public class AsyncEmitter implements Callable<Integer> {
(estimatedSize + sz), maxBytes);
emitAll();
}
- List<EmitData> cached =
map.computeIfAbsent(emitDataPair.emitterPluginId(), k -> new ArrayList<>());
+ List<EmitData> cached =
map.computeIfAbsent(emitDataPair.emitterId(), k -> new ArrayList<>());
updateEstimatedSize(sz);
cached.add(emitDataPair.emitData());
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java
index 3782572c4..0cc4889fc 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/AsyncProcessor.java
@@ -305,7 +305,7 @@ public class AsyncProcessor implements Closeable {
if (shouldEmit(result)) {
LOG.trace("adding result to emitter queue: " +
result.getEmitData());
boolean offered = emitDataTupleQueue.offer(
- new
EmitDataPair(t.getEmitKey().getEmitterPluginId(), result.getEmitData()),
MAX_OFFER_WAIT_MS,
+ new
EmitDataPair(t.getEmitKey().getEmitterId(), result.getEmitData()),
MAX_OFFER_WAIT_MS,
TimeUnit.MILLISECONDS);
if (! offered) {
throw new RuntimeException("Couldn't offer
emit data to queue " +
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/EmitDataPair.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/EmitDataPair.java
index 25bc71971..36e37280d 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/EmitDataPair.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/async/EmitDataPair.java
@@ -18,5 +18,5 @@ package org.apache.tika.pipes.core.async;
import org.apache.tika.pipes.api.emitter.EmitData;
-public record EmitDataPair(String emitterPluginId, EmitData emitData) {
+public record EmitDataPair(String emitterId, EmitData emitData) {
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitKey.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitKey.java
index 64de8ff33..6f0c26529 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitKey.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitKey.java
@@ -28,20 +28,20 @@ public class EmitKey implements Serializable {
*/
private static final long serialVersionUID = -3861669115439125268L;
- private String emitterPluginId;
+ private String emitterId;
private String emitKey;
//for serialization only...yuck.
public EmitKey() {
}
- public EmitKey(String emitterPluginId, String emitKey) {
- this.emitterPluginId = emitterPluginId;
+ public EmitKey(String emitterId, String emitKey) {
+ this.emitterId = emitterId;
this.emitKey = emitKey;
}
- public String getEmitterPluginId() {
- return emitterPluginId;
+ public String getEmitterId() {
+ return emitterId;
}
public String getEmitKey() {
@@ -50,7 +50,7 @@ public class EmitKey implements Serializable {
@Override
public String toString() {
- return "EmitterKey{" + "emitterPluginId='" + emitterPluginId + '\'' +
", emitterKey='" + emitKey +
+ return "EmitterKey{" + "emitterId='" + emitterId + '\'' + ",
emitterKey='" + emitKey +
'\'' + '}';
}
@@ -65,7 +65,7 @@ public class EmitKey implements Serializable {
EmitKey emitKey1 = (EmitKey) o;
- if (!Objects.equals(emitterPluginId, emitKey1.emitterPluginId)) {
+ if (!Objects.equals(emitterId, emitKey1.emitterId)) {
return false;
}
return Objects.equals(emitKey, emitKey1.emitKey);
@@ -73,7 +73,7 @@ public class EmitKey implements Serializable {
@Override
public int hashCode() {
- int result = emitterPluginId != null ? emitterPluginId.hashCode() : 0;
+ int result = emitterId != null ? emitterId.hashCode() : 0;
result = 31 * result + (emitKey != null ? emitKey.hashCode() : 0);
return result;
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitterManager.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitterManager.java
index 6549a0df8..e2686bda0 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitterManager.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmitterManager.java
@@ -31,10 +31,9 @@ import org.pf4j.PluginManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.pipes.api.emitter.Emitter;
+import org.apache.tika.pipes.api.emitter.EmitterFactory;
import org.apache.tika.pipes.core.PipesPluginsConfig;
import org.apache.tika.plugins.PluginConfig;
@@ -68,18 +67,17 @@ public class EmitterManager {
pluginManager.loadPlugins();
pluginManager.startPlugins();
Map<String, Emitter> emitterMap = new HashMap<>();
- for (Emitter emitter : pluginManager.getExtensions(Emitter.class)) {
- LOG.warn("EMITTER PLUGIN ID: " + emitter.getPluginId() + " : " +
emitter.getClass());
- Optional<PluginConfig> emitterConfig =
pluginsConfig.getEmitterConfig(emitter.getPluginId());
- if (emitterConfig.isPresent()) {
- emitter.configure(emitterConfig.get());
- if (emitter instanceof Initializable) {
- ((Initializable)
emitter).checkInitialization(InitializableProblemHandler.THROW);
+ for (EmitterFactory emitterFactory :
pluginManager.getExtensions(EmitterFactory.class)) {
+ for (String id : pluginsConfig.getEmitterConfig().ids()) {
+ Optional<PluginConfig> pluginConfigOpt =
pluginsConfig.getFetcherConfig(id);
+ if (pluginConfigOpt.isEmpty()) {
+ LOG.warn("Couldn't find config for id={}", id);
+ } else {
+ PluginConfig pluginConfig = pluginConfigOpt.get();
+ Emitter emitter = emitterFactory.buildPlugin(pluginConfig);
+ emitterMap.put(pluginConfig.id(), emitter);
}
- } else {
- LOG.warn("no configuration found for emitter pluginId={}",
emitter.getPluginId());
}
- emitterMap.put(emitter.getPluginId(), emitter);
}
return new EmitterManager(emitterMap);
}
@@ -112,11 +110,11 @@ public class EmitterManager {
* @return
*/
public Emitter getEmitter() {
- if (emitterMap.size() == 0) {
+ if (emitterMap.isEmpty()) {
throw new IllegalArgumentException("emitters size must == 1 for
the no arg call");
}
if (emitterMap.size() > 1) {
- throw new IllegalArgumentException("need to specify 'emitterName'
if > 1 emitters are" +
+ throw new IllegalArgumentException("need to specify 'emitterId' if
> 1 emitters are" +
" available");
}
for (Emitter emitter : emitterMap.values()) {
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmptyEmitter.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmptyEmitter.java
index 1149e6fe3..93c7d5165 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmptyEmitter.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/emitter/EmptyEmitter.java
@@ -19,7 +19,6 @@ package org.apache.tika.pipes.core.emitter;
import java.io.IOException;
import java.util.List;
-import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.api.emitter.AbstractEmitter;
@@ -27,17 +26,9 @@ import org.apache.tika.plugins.PluginConfig;
public class EmptyEmitter extends AbstractEmitter {
- public EmptyEmitter() throws IOException {
- }
- @Override
- public void configure(PluginConfig pluginConfig) throws
TikaConfigException, IOException {
- //no-op
- }
-
- @Override
- public String getPluginId() {
- return "";
+ public EmptyEmitter(PluginConfig pluginConfig) throws IOException {
+ super(pluginConfig);
}
@Override
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/EmptyFetcher.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/EmptyFetcher.java
index a6c576ea6..36375efc3 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/EmptyFetcher.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/EmptyFetcher.java
@@ -29,17 +29,12 @@ import org.apache.tika.plugins.PluginConfig;
public class EmptyFetcher implements Fetcher {
@Override
- public void configure(PluginConfig pluginConfig) throws
TikaConfigException {
- //no-op
- }
-
- @Override
- public String getPluginId() {
- return "empty";
+ public InputStream fetch(String fetchKey, Metadata metadata, ParseContext
parseContext) throws TikaException, IOException {
+ return null;
}
@Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext
parseContext) throws TikaException, IOException {
- return null;
+ public PluginConfig getPluginConfig() {
+ return new PluginConfig("empty", "empty-fetcher", "");
}
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/FetchKey.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/FetchKey.java
index 635e029eb..dbb1f904c 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/FetchKey.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/FetchKey.java
@@ -20,7 +20,7 @@ import java.io.Serializable;
import java.util.Objects;
/**
- * Pair of fetcherPluginId (which fetcher to call) and the key
+ * Pair of fetcherId (which fetcher to call) and the key
* to send to that fetcher to retrieve a specific file.
*/
public class FetchKey implements Serializable {
@@ -29,7 +29,7 @@ public class FetchKey implements Serializable {
*/
private static final long serialVersionUID = -3861669115439125268L;
- private String fetcherPluginId;
+ private String fetcherId;
private String fetchKey;
private long rangeStart = -1;
private long rangeEnd = -1;
@@ -39,19 +39,19 @@ public class FetchKey implements Serializable {
}
- public FetchKey(String fetcherPluginId, String fetchKey) {
- this(fetcherPluginId, fetchKey, -1, -1);
+ public FetchKey(String fetcherId, String fetchKey) {
+ this(fetcherId, fetchKey, -1, -1);
}
- public FetchKey(String fetcherPluginId, String fetchKey, long rangeStart,
long rangeEnd) {
- this.fetcherPluginId = fetcherPluginId;
+ public FetchKey(String fetcherId, String fetchKey, long rangeStart, long
rangeEnd) {
+ this.fetcherId = fetcherId;
this.fetchKey = fetchKey;
this.rangeStart = rangeStart;
this.rangeEnd = rangeEnd;
}
- public String getFetcherPluginId() {
- return fetcherPluginId;
+ public String getFetcherId() {
+ return fetcherId;
}
public String getFetchKey() {
@@ -80,18 +80,18 @@ public class FetchKey implements Serializable {
}
FetchKey fetchKey1 = (FetchKey) o;
return rangeStart == fetchKey1.rangeStart && rangeEnd ==
fetchKey1.rangeEnd &&
- Objects.equals(fetcherPluginId, fetchKey1.fetcherPluginId) &&
+ Objects.equals(fetcherId, fetchKey1.fetcherId) &&
Objects.equals(fetchKey, fetchKey1.fetchKey);
}
@Override
public int hashCode() {
- return Objects.hash(fetcherPluginId, fetchKey, rangeStart, rangeEnd);
+ return Objects.hash(fetcherId, fetchKey, rangeStart, rangeEnd);
}
@Override
public String toString() {
- return "FetchKey{" + "fetcherPluginId='" + fetcherPluginId + '\'' + ",
fetchKey='" + fetchKey +
+ return "FetchKey{" + "fetcherId='" + fetcherId + '\'' + ", fetchKey='"
+ fetchKey +
'\'' + ", rangeStart=" + rangeStart + ", rangeEnd=" + rangeEnd
+ '}';
}
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/FetcherManager.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/FetcherManager.java
index dc38202a4..fae86dbf0 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/FetcherManager.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/fetcher/FetcherManager.java
@@ -36,6 +36,7 @@ import org.apache.tika.config.InitializableProblemHandler;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.pipes.api.fetcher.Fetcher;
+import org.apache.tika.pipes.api.fetcher.FetcherFactory;
import org.apache.tika.pipes.core.PipesPluginsConfig;
import org.apache.tika.plugins.PluginConfig;
@@ -65,17 +66,17 @@ public class FetcherManager {
pluginManager.loadPlugins();
pluginManager.startPlugins();
Map<String, Fetcher> fetcherMap = new HashMap<>();
- for (Fetcher fetcher : pluginManager.getExtensions(Fetcher.class)) {
- Optional<PluginConfig> fetcherConfig =
pluginsConfig.getFetcherConfig(fetcher.getPluginId());
- if (fetcherConfig.isPresent()) {
- fetcher.configure(fetcherConfig.get());
- if (fetcher instanceof Initializable) {
- ((Initializable)
fetcher).checkInitialization(InitializableProblemHandler.THROW);
+ for (FetcherFactory fetcherFactory :
pluginManager.getExtensions(FetcherFactory.class)) {
+ for (String id : pluginsConfig.getFetcherConfig().ids()) {
+ Optional<PluginConfig> pluginConfigOpt =
pluginsConfig.getFetcherConfig(id);
+ if (pluginConfigOpt.isEmpty()) {
+ LOG.warn("Couldn't find config for id={}", id);
+ } else {
+ PluginConfig pluginConfig = pluginConfigOpt.get();
+ Fetcher fetcher = fetcherFactory.buildPlugin(pluginConfig);
+ fetcherMap.put(pluginConfig.id(), fetcher);
}
- } else {
- LOG.warn("no configuration found for fetcher pluginId={}",
fetcher.getPluginId());
}
- fetcherMap.put(fetcher.getPluginId(), fetcher);
}
return new FetcherManager(fetcherMap);
}
@@ -87,11 +88,11 @@ public class FetcherManager {
}
- public Fetcher getFetcher(String pluginId) throws IOException,
TikaException {
- Fetcher fetcher = fetcherMap.get(pluginId);
+ public Fetcher getFetcher(String id) throws IOException, TikaException {
+ Fetcher fetcher = fetcherMap.get(id);
if (fetcher == null) {
throw new IllegalArgumentException(
- "Can't find fetcher for fetcherPluginId: " + pluginId + ".
I've loaded: " +
+ "Can't find fetcher for id=" + id + ". I've loaded: " +
fetcherMap.keySet());
}
return fetcher;
@@ -112,7 +113,7 @@ public class FetcherManager {
throw new IllegalArgumentException("fetchers size must == 1 for
the no arg call");
}
if (fetcherMap.size() > 1) {
- throw new IllegalArgumentException("need to specify
'fetcherPluginId' if > 1 fetchers are" +
+ throw new IllegalArgumentException("need to specify 'fetcherId' if
> 1 fetchers are" +
" available");
}
for (Fetcher fetcher : fetcherMap.values()) {
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/pipesiterator/PipesIterator.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/pipesiterator/PipesIterator.java
index 5bde0a20f..2ec086701 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/pipesiterator/PipesIterator.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/pipesiterator/PipesIterator.java
@@ -64,8 +64,8 @@ public abstract class PipesIterator extends ConfigBase
private long maxWaitMs = DEFAULT_MAX_WAIT_MS;
private ArrayBlockingQueue<FetchEmitTuple> queue = null;
private int queueSize = DEFAULT_QUEUE_SIZE;
- private String fetcherPluginId;
- private String emitterPluginId;
+ private String fetcherId;
+ private String emitterId;
private FetchEmitTuple.ON_PARSE_EXCEPTION onParseException =
FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT;
private BasicContentHandlerFactory.HANDLER_TYPE handlerType =
@@ -89,22 +89,22 @@ public abstract class PipesIterator extends ConfigBase
}
}
- public String getFetcherPluginId() {
- return fetcherPluginId;
+ public String getFetcherId() {
+ return fetcherId;
}
@Field
- public void setFetcherPluginId(String fetcherPluginId) {
- this.fetcherPluginId = fetcherPluginId;
+ public void setFetcherId(String fetcherId) {
+ this.fetcherId = fetcherId;
}
- public String getEmitterPluginId() {
- return emitterPluginId;
+ public String getEmitterId() {
+ return emitterId;
}
@Field
- public void setEmitterPluginId(String emitterPluginId) {
- this.emitterPluginId = emitterPluginId;
+ public void setEmitterId(String emitterId) {
+ this.emitterId = emitterId;
}
@Field
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java
index 17bfbce28..fa65f8759 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleDeserializer.java
@@ -51,7 +51,7 @@ public class FetchEmitTupleDeserializer extends
JsonDeserializer<FetchEmitTuple>
JsonNode root = jsonParser.readValueAsTree();
String id = readVal(ID, root, null, true);
- String fetcherPluginId = readVal(FETCHER, root, null, true);
+ String fetcherId = readVal(FETCHER, root, null, true);
String fetchKey = readVal(FETCH_KEY, root, null, true);
String emitterName = readVal(EMITTER, root, "", false);
String emitKey = readVal(EMIT_KEY, root, "", false);
@@ -62,7 +62,7 @@ public class FetchEmitTupleDeserializer extends
JsonDeserializer<FetchEmitTuple>
ParseContext parseContext = parseContextNode == null ? new
ParseContext() : ParseContextDeserializer.readParseContext(parseContextNode);
FetchEmitTuple.ON_PARSE_EXCEPTION onParseException =
readOnParseException(root);
- return new FetchEmitTuple(id, new FetchKey(fetcherPluginId, fetchKey,
fetchRangeStart, fetchRangeEnd),
+ return new FetchEmitTuple(id, new FetchKey(fetcherId, fetchKey,
fetchRangeStart, fetchRangeEnd),
new EmitKey(emitterName, emitKey), metadata, parseContext,
onParseException);
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleSerializer.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleSerializer.java
index 45eb74257..2e46793e5 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleSerializer.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/serialization/FetchEmitTupleSerializer.java
@@ -43,13 +43,13 @@ public class FetchEmitTupleSerializer extends
JsonSerializer<FetchEmitTuple> {
jsonGenerator.writeStartObject();
jsonGenerator.writeStringField(ID, t.getId());
- jsonGenerator.writeStringField(FETCHER,
t.getFetchKey().getFetcherPluginId());
+ jsonGenerator.writeStringField(FETCHER,
t.getFetchKey().getFetcherId());
jsonGenerator.writeStringField(FETCH_KEY,
t.getFetchKey().getFetchKey());
if (t.getFetchKey().hasRange()) {
jsonGenerator.writeNumberField(FETCH_RANGE_START,
t.getFetchKey().getRangeStart());
jsonGenerator.writeNumberField(FETCH_RANGE_END,
t.getFetchKey().getRangeEnd());
}
- jsonGenerator.writeStringField(EMITTER,
t.getEmitKey().getEmitterPluginId());
+ jsonGenerator.writeStringField(EMITTER, t.getEmitKey().getEmitterId());
if (!StringUtils.isBlank(t.getEmitKey().getEmitKey())) {
jsonGenerator.writeStringField(EMIT_KEY,
t.getEmitKey().getEmitKey());
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/pipesiterator/filelist/FileListPipesIterator.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/pipesiterator/filelist/FileListPipesIterator.java
index 3d97845fd..cac6afb8f 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/pipesiterator/filelist/FileListPipesIterator.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/pipesiterator/filelist/FileListPipesIterator.java
@@ -67,8 +67,8 @@ public class FileListPipesIterator extends PipesIterator
implements Initializabl
String line = reader.readLine();
while (line != null) {
if (! line.startsWith("#") && !StringUtils.isBlank(line)) {
- FetchKey fetchKey = new FetchKey(getFetcherPluginId(),
line);
- EmitKey emitKey = new EmitKey(getEmitterPluginId(), line);
+ FetchKey fetchKey = new FetchKey(getFetcherId(), line);
+ EmitKey emitKey = new EmitKey(getEmitterId(), line);
ParseContext parseContext = new ParseContext();
parseContext.set(HandlerConfig.class, getHandlerConfig());
tryToAdd(new FetchEmitTuple(line, fetchKey, emitKey,
@@ -95,8 +95,8 @@ public class FileListPipesIterator extends PipesIterator
implements Initializabl
throws TikaConfigException {
//these should all be fatal
TikaConfig.mustNotBeEmpty("fileList", fileList);
- TikaConfig.mustNotBeEmpty("fetcherPluginId", getFetcherPluginId());
- TikaConfig.mustNotBeEmpty("emitterPluginId", getEmitterPluginId());
+ TikaConfig.mustNotBeEmpty("fetcherId", getFetcherId());
+ TikaConfig.mustNotBeEmpty("emitterId", getEmitterId());
fileListPath = Paths.get(fileList);
if (!Files.isRegularFile(fileListPath)) {
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIterator.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIterator.java
index 69a75f97a..8c029f18d 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIterator.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIterator.java
@@ -79,7 +79,7 @@ public class FileSystemPipesIterator extends PipesIterator
}
try {
- Files.walkFileTree(basePath, new
FSFileVisitor(getFetcherPluginId(), getEmitterPluginId()));
+ Files.walkFileTree(basePath, new FSFileVisitor(getFetcherId(),
getEmitterId()));
} catch (IOException e) {
Throwable cause = e.getCause();
if (cause != null && cause instanceof TimeoutException) {
@@ -95,8 +95,8 @@ public class FileSystemPipesIterator extends PipesIterator
throws TikaConfigException {
//these should all be fatal
TikaConfig.mustNotBeEmpty("basePath", basePath);
- TikaConfig.mustNotBeEmpty("fetcherPluginId", getFetcherPluginId());
- TikaConfig.mustNotBeEmpty("emitterPluginId", getEmitterPluginId());
+ TikaConfig.mustNotBeEmpty("fetcherId", getFetcherId());
+ TikaConfig.mustNotBeEmpty("emitterId", getEmitterId());
}
@Override
diff --git
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
index ae59bf64d..bcfcdbbf0 100644
---
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
+++
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/TikaPipesConfigTest.java
@@ -86,7 +86,7 @@ public class TikaPipesConfigTest extends
AbstractTikaConfigTest {
public void testPipesIterator() throws Exception {
PipesIterator it =
PipesIterator.build(getConfigFilePath("pipes-iterator-config.xml"));
- assertEquals("fsf1", it.getFetcherPluginId());
+ assertEquals("fsf1", it.getFetcherId());
}
@Test
@@ -94,7 +94,7 @@ public class TikaPipesConfigTest extends
AbstractTikaConfigTest {
assertThrows(TikaConfigException.class, () -> {
PipesIterator it =
PipesIterator.build(getConfigFilePath("pipes-iterator-multiple-config.xml"));
- assertEquals("fsf1", it.getFetcherPluginId());
+ assertEquals("fsf1", it.getFetcherId());
});
}
}
diff --git
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/pipesiterator/filelist/FileListPipesIteratorTest.java
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/pipesiterator/filelist/FileListPipesIteratorTest.java
index 2cf38ef51..e9cf5dd64 100644
---
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/pipesiterator/filelist/FileListPipesIteratorTest.java
+++
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/pipesiterator/filelist/FileListPipesIteratorTest.java
@@ -36,8 +36,8 @@ public class FileListPipesIteratorTest {
public void testBasic() throws Exception {
Path p =
Paths.get(this.getClass().getResource("/test-documents/file-list.txt").toURI());
FileListPipesIterator it = new FileListPipesIterator();
- it.setFetcherPluginId("f");
- it.setEmitterPluginId("e");
+ it.setFetcherId("f");
+ it.setEmitterId("e");
it.setFileList(p.toAbsolutePath().toString());
it.setHasHeader(false);
it.checkInitialization(InitializableProblemHandler.DEFAULT);
@@ -46,8 +46,8 @@ public class FileListPipesIteratorTest {
for (FetchEmitTuple t : it) {
assertEquals(t.getFetchKey().getFetchKey(),
t.getEmitKey().getEmitKey());
assertEquals(t.getId(), t.getEmitKey().getEmitKey());
- assertEquals("f", t.getFetchKey().getFetcherPluginId());
- assertEquals("e", t.getEmitKey().getEmitterPluginId());
+ assertEquals("f", t.getFetchKey().getFetcherId());
+ assertEquals("e", t.getEmitKey().getEmitterId());
lines.add(t.getId());
}
assertEquals("the", lines.get(0));
@@ -59,8 +59,8 @@ public class FileListPipesIteratorTest {
public void testHasHeader() throws Exception {
Path p =
Paths.get(this.getClass().getResource("/test-documents/file-list.txt").toURI());
FileListPipesIterator it = new FileListPipesIterator();
- it.setFetcherPluginId("f");
- it.setEmitterPluginId("e");
+ it.setFetcherId("f");
+ it.setEmitterId("e");
it.setFileList(p.toAbsolutePath().toString());
it.setHasHeader(true);
it.checkInitialization(InitializableProblemHandler.DEFAULT);
@@ -69,8 +69,8 @@ public class FileListPipesIteratorTest {
for (FetchEmitTuple t : it) {
assertEquals(t.getFetchKey().getFetchKey(),
t.getEmitKey().getEmitKey());
assertEquals(t.getId(), t.getEmitKey().getEmitKey());
- assertEquals("f", t.getFetchKey().getFetcherPluginId());
- assertEquals("e", t.getEmitKey().getEmitterPluginId());
+ assertEquals("f", t.getFetchKey().getFetcherId());
+ assertEquals("e", t.getEmitKey().getEmitterId());
lines.add(t.getId());
}
assertEquals("brown", lines.get(0));
diff --git
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorTest.java
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorTest.java
index 9178aeb2b..4328aabb0 100644
---
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorTest.java
+++
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorTest.java
@@ -63,7 +63,7 @@ public class FileSystemPipesIteratorTest {
String fetcherName = "file-system-fetcher";
PipesIterator it = new FileSystemPipesIterator(root);
- it.setFetcherPluginId(fetcherName);
+ it.setFetcherId(fetcherName);
it.setQueueSize(2);
Set<String> iteratorSet = new HashSet<>();
diff --git
a/tika-pipes/tika-pipes-core/src/test/resources/configs/fetchers.json
b/tika-pipes/tika-pipes-core/src/test/resources/configs/fetchers.json
index b0e9e0ff0..453505d07 100644
--- a/tika-pipes/tika-pipes-core/src/test/resources/configs/fetchers.json
+++ b/tika-pipes/tika-pipes-core/src/test/resources/configs/fetchers.json
@@ -2,8 +2,11 @@
"plugins" : {
"fetchers": {
"file-system-fetcher": {
- "basePath": "{BASE_PATH}",
- "extractFileSystemMetadata": false
+ "id": "extract-false",
+ "config": {
+ "basePath": "{BASE_PATH}",
+ "extractFileSystemMetadata": false
+ }
}
}
}
diff --git
a/tika-pipes/tika-pipes-core/src/test/resources/configs/tika-config-broken.xml
b/tika-pipes/tika-pipes-core/src/test/resources/configs/tika-config-broken.xml
index 6aa305e34..112f40725 100644
---
a/tika-pipes/tika-pipes-core/src/test/resources/configs/tika-config-broken.xml
+++
b/tika-pipes/tika-pipes-core/src/test/resources/configs/tika-config-broken.xml
@@ -26,7 +26,7 @@
</fetcher>
</fetchers>
<pipesIterator
class="org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator">
- <fetcherPluginId>fs</fetcherPluginId>
+ <fetcherId>fs</fetcherId>
<basePath>basePath</basePath>
</pipesIterator>
</properties>
\ No newline at end of file
diff --git
a/tika-pipes/tika-pipes-core/src/test/resources/org/apache/tika/config/pipes-iterator-config.xml
b/tika-pipes/tika-pipes-core/src/test/resources/org/apache/tika/config/pipes-iterator-config.xml
index 31f7f0a1a..b613ca631 100644
---
a/tika-pipes/tika-pipes-core/src/test/resources/org/apache/tika/config/pipes-iterator-config.xml
+++
b/tika-pipes/tika-pipes-core/src/test/resources/org/apache/tika/config/pipes-iterator-config.xml
@@ -18,8 +18,8 @@
<properties>
<pipesIterator
class="org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator">
<params>
- <fetcherPluginId>fsf1</fetcherPluginId>
- <emitterPluginId>fse1</emitterPluginId>
+ <fetcherId>fsf1</fetcherId>
+ <emitterId>fse1</emitterId>
<basePath>/my/base/path1</basePath>
</params>
</pipesIterator>
diff --git
a/tika-pipes/tika-pipes-core/src/test/resources/org/apache/tika/config/pipes-iterator-multiple-config.xml
b/tika-pipes/tika-pipes-core/src/test/resources/org/apache/tika/config/pipes-iterator-multiple-config.xml
index f7bed7010..1fdc64ccd 100644
---
a/tika-pipes/tika-pipes-core/src/test/resources/org/apache/tika/config/pipes-iterator-multiple-config.xml
+++
b/tika-pipes/tika-pipes-core/src/test/resources/org/apache/tika/config/pipes-iterator-multiple-config.xml
@@ -18,15 +18,15 @@
<properties>
<pipesIterator
class="org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator">
<params>
- <fetcherPluginId>fsf1</fetcherPluginId>
- <emitterPluginId>fse1</emitterPluginId>
+ <fetcherId>fsf1</fetcherId>
+ <emitterId>fse1</emitterId>
<basePath>/my/base/path1</basePath>
</params>
</pipesIterator>
<pipesIterator
class="org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator">
<params>
- <fetcherPluginId>fsf2</fetcherPluginId>
- <emitterPluginId>fse2</emitterPluginId>
+ <fetcherId>fsf2</fetcherId>
+ <emitterId>fse2</emitterId>
<basePath>/my/base/path2</basePath>
</params>
</pipesIterator>
diff --git
a/tika-serialization/src/main/java/org/apache/tika/serialization/PluginsConfigDeserializer.java
b/tika-serialization/src/main/java/org/apache/tika/serialization/PluginsConfigDeserializer.java
index 72370e308..3b37c81a8 100644
---
a/tika-serialization/src/main/java/org/apache/tika/serialization/PluginsConfigDeserializer.java
+++
b/tika-serialization/src/main/java/org/apache/tika/serialization/PluginsConfigDeserializer.java
@@ -32,12 +32,12 @@ public class PluginsConfigDeserializer extends
JsonDeserializer<PluginConfig> {
public PluginConfig deserialize(JsonParser jsonParser,
DeserializationContext deserializationContext) throws IOException,
JacksonException {
JsonNode node = jsonParser.getCodec().readTree(jsonParser);
- String pluginId = node.get("pluginId").asText();
+ String factoryPluginId = node.get("factoryPluginId").asText();
- JsonNode jsonConfigNode = node.get("jsonConfig");
+ JsonNode jsonConfigNode = node.get("config");
String jsonConfigRaw = jsonConfigNode.toString();
- return new PluginConfig(pluginId, jsonConfigRaw);
+ return new PluginConfig("id", factoryPluginId, jsonConfigRaw);
}
}
diff --git
a/tika-serialization/src/main/java/org/apache/tika/serialization/PluginsConfigSerializer.java
b/tika-serialization/src/main/java/org/apache/tika/serialization/PluginsConfigSerializer.java
index baf45acb8..dee7edbb4 100644
---
a/tika-serialization/src/main/java/org/apache/tika/serialization/PluginsConfigSerializer.java
+++
b/tika-serialization/src/main/java/org/apache/tika/serialization/PluginsConfigSerializer.java
@@ -29,7 +29,7 @@ public class PluginsConfigSerializer extends
JsonSerializer<PluginConfig> {
@Override
public void serialize(PluginConfig pluginsConfig, JsonGenerator
jsonGenerator, SerializerProvider serializerProvider) throws IOException {
jsonGenerator.writeStartObject();
- jsonGenerator.writeStringField("pluginId", pluginsConfig.pluginId());
+ jsonGenerator.writeStringField("pluginId",
pluginsConfig.factoryPluginId());
jsonGenerator.writeFieldName("jsonConfig");
jsonGenerator.writeRawValue(pluginsConfig.jsonConfig());
jsonGenerator.writeEndObject();
diff --git
a/tika-serialization/src/test/java/org/apache/tika/serialization/PluginsConfigTest.java
b/tika-serialization/src/test/java/org/apache/tika/serialization/PluginsConfigTest.java
index 78bb28ab1..7e04f53f4 100644
---
a/tika-serialization/src/test/java/org/apache/tika/serialization/PluginsConfigTest.java
+++
b/tika-serialization/src/test/java/org/apache/tika/serialization/PluginsConfigTest.java
@@ -52,7 +52,7 @@ public class PluginsConfigTest {
String json = OBJECT_MAPPER.writeValueAsString(p1);
PluginConfig deserialized = OBJECT_MAPPER.readValue(json,
PluginConfig.class);
- assertEquals(p1.pluginId(), deserialized.pluginId());
+ assertEquals(p1.factoryPluginId(), deserialized.factoryPluginId());
assertEquals(flatten(p1.jsonConfig()),
flatten(deserialized.jsonConfig()));
}
@@ -67,16 +67,16 @@ public class PluginsConfigTest {
{"basePath":"/my/docs2","includeSystemInfo":false}
""");
Map<String, PluginConfig> map = new HashMap<>();
- map.put(p1.pluginId(), p1);
- map.put(p2.pluginId(), p2);
+ map.put(p1.factoryPluginId(), p1);
+ map.put(p2.factoryPluginId(), p2);
PluginConfigs pluginConfigManager = new PluginConfigs(map);
String json = OBJECT_MAPPER.writeValueAsString(pluginConfigManager);
PluginConfigs deserialized = OBJECT_MAPPER.readValue(json,
PluginConfigs.class);
- assertEquals(pluginConfigManager.get(p1.pluginId()).get().pluginId(),
deserialized.get(p1.pluginId()).get().pluginId());
-
assertEquals(flatten(pluginConfigManager.get(p1.pluginId()).get().jsonConfig()),
- flatten(deserialized.get(p1.pluginId()).get().jsonConfig()));
+
assertEquals(pluginConfigManager.get(p1.factoryPluginId()).get().factoryPluginId(),
deserialized.get(p1.factoryPluginId()).get().factoryPluginId());
+
assertEquals(flatten(pluginConfigManager.get(p1.factoryPluginId()).get().jsonConfig()),
+
flatten(deserialized.get(p1.factoryPluginId()).get().jsonConfig()));
}
private static String flatten(String s) {
