This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 54192b4e0 TIKA-3824 -- add distinct metadata values, add match
map/functionality
new 42da5357f Merge remote-tracking branch 'origin/main' into main
54192b4e0 is described below
commit 54192b4e0b8c07598171f58b402664ea45f41ecc
Author: tballison <[email protected]>
AuthorDate: Tue Jul 26 12:33:27 2022 -0400
TIKA-3824 -- add distinct metadata values, add match map/functionality
---
.../org/apache/tika/parser/RegexCaptureParser.java | 55 +++++++++++++++++++---
.../apache/tika/parser/RegexCaptureParserTest.java | 2 +-
.../tika/config/TIKA-3557-exiftool-example.xml | 2 +-
.../resources/org/apache/tika/config/TIKA-3557.xml | 2 +-
4 files changed, 51 insertions(+), 10 deletions(-)
diff --git
a/tika-core/src/main/java/org/apache/tika/parser/RegexCaptureParser.java
b/tika-core/src/main/java/org/apache/tika/parser/RegexCaptureParser.java
index 710d0395e..beb3d9d82 100644
--- a/tika-core/src/main/java/org/apache/tika/parser/RegexCaptureParser.java
+++ b/tika-core/src/main/java/org/apache/tika/parser/RegexCaptureParser.java
@@ -23,6 +23,7 @@ import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
@@ -45,7 +46,8 @@ public class RegexCaptureParser extends AbstractParser
implements Initializable
private static final Set<MediaType> SUPPORTED_TYPES =
Collections.singleton(MediaType.TEXT_PLAIN);
- private Map<String, Pattern> regexMap = new HashMap();
+ private Map<String, Pattern> captureMap = new HashMap<>();
+ private Map<String, Pattern> matchMap = new HashMap<>();
@Override
public void initialize(Map<String, Param> params) throws
TikaConfigException {
@@ -69,29 +71,68 @@ public class RegexCaptureParser extends AbstractParser
implements Initializable
try (BufferedReader reader = new BufferedReader(new
InputStreamReader(stream,
StandardCharsets.UTF_8))) {
String line = reader.readLine();
- Map<String, Matcher> matchers = new HashMap();
- for (Map.Entry<String, Pattern> e : regexMap.entrySet()) {
- matchers.put(e.getKey(), e.getValue().matcher(""));
+ Map<String, Matcher> localCaptureMap = new HashMap();
+ for (Map.Entry<String, Pattern> e : captureMap.entrySet()) {
+ localCaptureMap.put(e.getKey(), e.getValue().matcher(""));
}
+ Map<String, Matcher> localMatchMap = new HashMap<>();
+ for (Map.Entry<String, Pattern> e : matchMap.entrySet()) {
+ localMatchMap.put(e.getKey(), e.getValue().matcher(""));
+ }
+
+ Map<String, Set<String>> keyVals = new HashMap<>();
while (line != null) {
- for (Map.Entry<String, Matcher> e : matchers.entrySet()) {
+ for (Map.Entry<String, Matcher> e :
localCaptureMap.entrySet()) {
Matcher m = e.getValue();
if (m.reset(line).find()) {
String val = m.group(1);
- metadata.set(e.getKey(), val);
+ Set<String> vals = keyVals.get(e.getKey());
+ if (vals == null) {
+ vals = new LinkedHashSet<>();
+ keyVals.put(e.getKey(), vals);
+ }
+ vals.add(val);
+ }
+ }
+ for (Map.Entry<String, Matcher> e : localMatchMap.entrySet()) {
+ if (e.getValue().reset(line).find()) {
+ metadata.set(e.getKey(), "true");
}
}
line = reader.readLine();
}
+ for (Map.Entry<String, Set<String>> e : keyVals.entrySet()) {
+ for (String val : e.getValue()) {
+ metadata.add(e.getKey(), val);
+ }
+ }
}
}
@Field
+ @Deprecated
+ /**
+ * @deprecated use setCaptureMap
+ */
public void setRegexMap(Map<String, String> map) {
+ setCaptureMap(map);
+ }
+
+ @Field
+ public void setCaptureMap(Map<String, String> map) {
+ for (Map.Entry<String, String> e : map.entrySet()) {
+ String field = e.getKey();
+ Pattern pattern = Pattern.compile(e.getValue());
+ captureMap.put(field, pattern);
+ }
+ }
+
+ @Field
+ public void setMatchMap(Map<String, String> map) {
for (Map.Entry<String, String> e : map.entrySet()) {
String field = e.getKey();
Pattern pattern = Pattern.compile(e.getValue());
- regexMap.put(field, pattern);
+ matchMap.put(field, pattern);
}
}
}
diff --git
a/tika-core/src/test/java/org/apache/tika/parser/RegexCaptureParserTest.java
b/tika-core/src/test/java/org/apache/tika/parser/RegexCaptureParserTest.java
index a3f1e1a4e..13c5eada7 100644
--- a/tika-core/src/test/java/org/apache/tika/parser/RegexCaptureParserTest.java
+++ b/tika-core/src/test/java/org/apache/tika/parser/RegexCaptureParserTest.java
@@ -43,7 +43,7 @@ public class RegexCaptureParserTest {
RegexCaptureParser parser = new RegexCaptureParser();
Map<String, String> regexes = new HashMap<>();
regexes.put("title", "^Title: ([^\r\n]+)");
- parser.setRegexMap(regexes);
+ parser.setCaptureMap(regexes);
try (InputStream stream =
TikaInputStream.get(output.getBytes(StandardCharsets.UTF_8))) {
diff --git
a/tika-core/src/test/resources/org/apache/tika/config/TIKA-3557-exiftool-example.xml
b/tika-core/src/test/resources/org/apache/tika/config/TIKA-3557-exiftool-example.xml
index efe430fd4..c06d3592d 100644
---
a/tika-core/src/test/resources/org/apache/tika/config/TIKA-3557-exiftool-example.xml
+++
b/tika-core/src/test/resources/org/apache/tika/config/TIKA-3557-exiftool-example.xml
@@ -32,7 +32,7 @@
</param>
<param name="outputParser" type="class"
class="org.apache.tika.parser.RegexCaptureParser">
<params>
- <param name="regexMap" type="map">
+ <param name="captureMap" type="map">
<match key="mime" value="^MIME Type\s+: ([^\r\n]+)"/>
<match key="pages" value="^Page Count\s+: ([^\r\n]+)"/>
<match key="pdf:version" value="^PDF Version\s+: ([^\r\n]+)"/>
diff --git a/tika-core/src/test/resources/org/apache/tika/config/TIKA-3557.xml
b/tika-core/src/test/resources/org/apache/tika/config/TIKA-3557.xml
index 1df090bf7..93b13fa44 100644
--- a/tika-core/src/test/resources/org/apache/tika/config/TIKA-3557.xml
+++ b/tika-core/src/test/resources/org/apache/tika/config/TIKA-3557.xml
@@ -30,7 +30,7 @@
</param>
<param name="outputParser" type="class"
class="org.apache.tika.parser.RegexCaptureParser">
<params>
- <param name="regexMap" type="map">
+ <param name="captureMap" type="map">
<title>^Title: ([^\r\n]+)</title>
</param>
</params>
