This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new f58e44c022 TIKA-4971: make server and apps unpack output the same and
make frict… (#3347)
f58e44c022 is described below
commit f58e44c022a9952252f3e094dcc40abd931f7ecd
Author: Tim Allison <[email protected]>
AuthorDate: Mon Oct 19 17:40:08 2026 -0400
TIKA-4971: make server and apps unpack output the same and make frict…
(#3347)
---
.skills/users/file-forensics/SKILL.md | 22 +--
.skills/users/file-to-markdown/SKILL.md | 12 +-
CHANGES.txt | 23 +++
.../advanced/integration-testing/tika-app.adoc | 2 +-
.../advanced/integration-testing/tika-server.adoc | 2 +-
docs/modules/ROOT/pages/pipes/unpack-config.adoc | 94 +++++++--
docs/modules/ROOT/pages/using-tika/cli/index.adoc | 13 +-
.../ROOT/pages/using-tika/server/index.adoc | 4 +-
.../src/main/java/org/apache/tika/cli/TikaCLI.java | 4 +-
.../test/java/org/apache/tika/cli/TikaCLITest.java | 155 ++++++++++-----
.../org/apache/tika/async/cli/TikaAsyncCLI.java | 17 +-
.../apache/tika/async/cli/AsyncCliParserTest.java | 12 ++
.../core/extractor/FrictionlessUnpackHandler.java | 12 +-
.../tika/pipes/core/extractor/UnpackConfig.java | 69 ++++++-
.../core/extractor/frictionless/DataPackage.java | 20 +-
.../frictionless/FrictionlessResource.java | 58 +++---
.../tika/pipes/core/server/ConnectionHandler.java | 3 +-
.../apache/tika/pipes/core/server/EmitHandler.java | 48 +++--
.../core/server/MetadataListAndEmbeddedBytes.java | 13 ++
.../apache/tika/pipes/core/server/PipesServer.java | 5 +-
.../apache/tika/pipes/core/server/PipesWorker.java | 214 ++++++++++++++-------
.../apache/tika/pipes/core/presets/thumbnail.json | 2 +
.../frictionless/FrictionlessResourceTest.java | 40 ++++
.../tika/pipes/core/FrictionlessUnpackTest.java | 187 +++++++++++++++---
.../org/apache/tika/pipes/core/UnpackModeTest.java | 17 +-
.../server/core/resource/PipesParsingHelper.java | 3 +
.../tika/server/core/resource/TikaResource.java | 18 ++
.../server/core/resource/UnpackerResource.java | 12 +-
.../apache/tika/server/core/StackTraceTest.java | 10 +-
.../tika/server/standard/ThumbnailPresetTest.java | 2 +-
.../apache/tika/server/standard/TikaPipesTest.java | 2 +-
.../server/standard/UnpackFrictionlessTest.java | 12 +-
.../server/standard/UnpackRegularConfigTest.java | 2 +-
.../standard/UnpackerResourceHandlerTest.java | 14 +-
.../tika/server/standard/UnpackerResourceTest.java | 76 +++++---
35 files changed, 891 insertions(+), 308 deletions(-)
diff --git a/.skills/users/file-forensics/SKILL.md
b/.skills/users/file-forensics/SKILL.md
index 0675d28b5d..b774d28dee 100644
--- a/.skills/users/file-forensics/SKILL.md
+++ b/.skills/users/file-forensics/SKILL.md
@@ -106,7 +106,7 @@ java -jar tika-app.jar --config=file-forensics-config.json
-Z --extract-dir=evid
# mkdir -p evidence/suspect-embedded && unzip -q suspect-embedded.zip -d
evidence/suspect-embedded
jq 'map(del(."tk:content"))' suspect.rmeta.json > suspect.meta.json
-ls -lR evidence/suspect-embedded && sha256sum
evidence/suspect-embedded/*/unpacked/* 2>/dev/null || sha256sum
evidence/suspect-embedded/unpacked/*
+ls -lR evidence/suspect-embedded && sha256sum
evidence/suspect-embedded/unpacked/*
```
`suspect.rmeta.json` is the full evidence record — a JSON array where entry
@@ -339,20 +339,18 @@ files is incomplete.
back visually corrupted (`VB\_Name`). The `-z`/`-Z` extracted files are
pristine bytes. (Alternative: capture with `-J -t` for unescaped plain
text per entry.)
-- Extracted files are **renumbered** (`00000001.jpg`, ...); original names
- live in each rmeta entry's `tk:resource-name`, and `-z`/`-Z` writes a
- sidecar `<name>.json` metadata dump for mapping numbers back to names.
-- The output is a Frictionless data package (Tika 4.2+): the files sit under
- `<container>/unpacked/`, and `<container>/datapackage.json` already carries a
- SHA-256 per file. Digest what you extracted
- (`sha256sum evidence/suspect-embedded/*/unpacked/*`) and check it against the
+- Extracted files are **renumbered** (`1.jpg`, `2.pdf`, ...); each
+ `datapackage.json` resource carries the original name (`tk:resource-name`)
+ and `tk:embedded-id`, the key to the file's row in `rmeta.json`.
+- The output is a Frictionless data package (Tika 4.2+): for a single input
+ file the extract dir is the package — files under `unpacked/`, and
+ `datapackage.json` already carries a SHA-256 per file (and for
+ `rmeta.json`). Digest what you extracted
+ (`sha256sum evidence/suspect-embedded/unpacked/*`) and check it against the
manifest, so each artifact is pinned the same way the container file is.
- Mechanics and gotchas (Pipes-mode delay, chatty stderr) are in
`file-to-markdown`; the server-side equivalent is `/unpack`, which returns
- the same package as one zip over HTTP. **`/unpack` names differ from
- `-z`/`-Z`:** plain sequential names (`unpacked/1.jpg`, `unpacked/2.pdf`, ...)
- and the rmeta rows live in the package's `metadata.json` rather than a
- sidecar — map names back via each row's `tk:resource-name`. Against the
+ the same package as one zip over HTTP, laid out the same way. Against the
server started above (forensics config loaded):
```bash
diff --git a/.skills/users/file-to-markdown/SKILL.md
b/.skills/users/file-to-markdown/SKILL.md
index 51c410007b..0fb0d8b4fa 100644
--- a/.skills/users/file-to-markdown/SKILL.md
+++ b/.skills/users/file-to-markdown/SKILL.md
@@ -151,11 +151,13 @@ embedded content in: email attachments (`.eml`, `.msg`),
archive members
(`.zip`), PDF attachments, embedded objects/images in any office format
(an inline pasted picture counts), and so on.
-What lands on disk: a `<name>/` Frictionless package — `datapackage.json`
-(SHA-256 per file), `metadata.json`, and the embedded files **renumbered**
-under `unpacked/` (`00000001.jpg`, ...). Original names are not preserved on
-disk; they are in each rmeta entry's `tk:resource-name`, so keep the sibling
-`<name>.json` (an rmeta-shaped dump `-z` also writes) to map numbers back.
+What lands on disk: the extract dir (default `<name>-unpacked/`) is a
+Frictionless package — `datapackage.json` (SHA-256 per file),
+`rmeta.json` (the rmeta list), and the embedded files **renumbered**
+under `unpacked/` (`1.jpg`, ...). Original names are not preserved on disk;
+each `datapackage.json` resource has `tk:resource-name` and `tk:embedded-id`
+(its row in `rmeta.json`). An input
+directory gets one such package per file, under its relative path.
`-z`/`-Z` route through Tika Pipes mode rather than the fast synchronous
path the other flags use — expect several seconds and a burst of
diff --git a/CHANGES.txt b/CHANGES.txt
index 40f3a277e3..be595e6e8f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,28 @@
Release 4.2.0 - unreleased
+ * Frictionless unpack: the package's metadata.json is renamed rmeta.json,
is listed in
+ datapackage.json as a resource (size, sha256), and now gets the metadata
filters and
+ user metadata the emitted list gets; resources are in embedded-id order.
A document
+ with no attachments is a package of datapackage.json and rmeta.json:
tika-server
+ /unpack returns it with 200 instead of 204
(unpack-config.emptyPackage=SKIP, which the
+ thumbnail preset sets, restores the 204). The package name is the
caller's filename
+ on tika-server, never the server's spool file. When the package goes to
the container's
+ own emitter it replaces the metadata emit on every surface: no
<emitKey>.json next to
+ it, and with includeMetadata=false no metadata at all. tika-app -z/-Z on
a single file
+ makes the output dir the package, defaulting to <name>-unpacked/ in
either format, and
+ names files unpacked/1.<ext> as tika-server /unpack does. REGULAR
.metadata.json
+ sidecars are now the file's row from the metadata list: they get the
metadata filters
+ and text inference, and a row a filter removes gets no sidecar. New
+ unpack-config.packageRoot and packageName set where a DIRECTORY-mode
package goes
+ and its name. Packages pass frictionless validate: each resource's name
is derived
+ from its path, the original filename moves to tk:resource-name, and
tk:embedded-id
+ matches its rmeta.json row, if present (TIKA-4971).
+
+ * API: FrictionlessResource.name() is now the Frictionless resource name
derived from
+ the path; the original filename moved to resourceName(), and embeddedId()
is new.
+ FrictionlessResource.create(path, mediatype, bytes, hash, name) and
+ DataPackage.addResource(...) are removed (TIKA-4971).
+
* Per-request limits may tighten the server's but no longer loosen them
(TIKA-4978).
* Image checks (byte-size, min-image-size, blank-image-check; new
tika-image-processing
diff --git a/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc
b/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc
index 08b41395c1..3dcb754f42 100644
--- a/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc
+++ b/docs/modules/ROOT/pages/advanced/integration-testing/tika-app.adoc
@@ -338,7 +338,7 @@ ls -R /tmp/tika-app-test/frictionless-out
----
*Expected:* `test_recursive_embedded.docx.json` plus a
`test_recursive_embedded.docx/`
-directory holding `datapackage.json`, `metadata.json` and `unpacked/`.
+directory holding `datapackage.json`, `rmeta.json` and `unpacked/`.
=== Test 24: Unpack zipped, and the pre-4.2 flat layout
diff --git
a/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc
b/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc
index 414bb770f1..da378cfa6c 100644
--- a/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc
+++ b/docs/modules/ROOT/pages/advanced/integration-testing/tika-server.adoc
@@ -158,7 +158,7 @@ curl -s -X PUT -T test_recursive_embedded.docx
http://localhost:9998/unpack/all
unzip -l /tmp/unpack.zip
----
-*Expected:* A Frictionless package: `datapackage.json`, `metadata.json`, and
the embedded
+*Expected:* A Frictionless package: `datapackage.json`, `rmeta.json`, and the
embedded
files plus the original container (`0.docx`) under `unpacked/`.
=== Test 13: GET /parsers
diff --git a/docs/modules/ROOT/pages/pipes/unpack-config.adoc
b/docs/modules/ROOT/pages/pipes/unpack-config.adoc
index 9ac0881ca7..bdf33b24f5 100644
--- a/docs/modules/ROOT/pages/pipes/unpack-config.adoc
+++ b/docs/modules/ROOT/pages/pipes/unpack-config.adoc
@@ -103,7 +103,7 @@ immediately capped.
|`includeMetadata`
|boolean
|unset: on for `FRICTIONLESS`, off for `REGULAR`
-|Metadata for every extracted file, in either format: one `metadata.json`
under Frictionless (a data package without the parse is half a package, so it
is on unless you set `false`), a `.metadata.json` sidecar per file under
REGULAR zip output (opt-in; tika-server's `/unpack/all` sets it). REGULAR loose
output (no zip) writes none.
+|Metadata for every extracted file, in either format: one `rmeta.json` under
Frictionless (a data package without the parse is half a package, so it is on
unless you set `false`), a `.metadata.json` sidecar per file under REGULAR zip
output (opt-in; tika-server's `/unpack/all` sets it). REGULAR loose output (no
zip) writes none.
|`includeMetadataInZip`
|boolean
@@ -145,6 +145,21 @@ immediately capped.
|`ZIPPED`
|`ZIPPED` packages everything into one archive; `DIRECTORY` emits each
extracted file to the emitter as its own item.
+|`packageRoot`
+|String
+|_(container's emit key)_
+|`FRICTIONLESS` + `DIRECTORY` only: emit key prefix of the package. Unset,
each container gets `<emitKey>/datapackage.json`, `<emitKey>/unpacked/...`;
`""` writes the package at the emitter's root (tika-app does this for a single
input file).
+
+|`emptyPackage`
+|WRITE, SKIP
+|`WRITE`
+|`FRICTIONLESS` only: a document with nothing to unpack still gets a package
(`datapackage.json` and `rmeta.json`), or with `SKIP` none -- and tika-server's
`/unpack` answers `204`. The `thumbnail` preset sets `SKIP`.
+
+|`packageName`
+|String
+|_(fetch key's file name)_
+|`FRICTIONLESS` only: the `name` in `datapackage.json`; blank leaves it out.
tika-server sets the caller's filename, or blank when there is none.
+
|`includeFullMetadata`
|boolean
|--
@@ -272,14 +287,14 @@ A zipped package contains:
----
output.zip
├── datapackage.json # Manifest with file list, SHA256 hashes, mimetypes
-├── metadata.json # Full RMETA metadata (unless includeMetadata=false)
+├── rmeta.json # Full RMETA metadata (unless includeMetadata=false)
└── unpacked/
- ├── 00000001.pdf
- ├── 00000002.png
+ ├── 1.pdf
+ ├── 2.png
└── ...
----
-The `tk:content` field inside `metadata.json` (and inside the per-file
`.metadata.json`
+The `tk:content` field inside `rmeta.json` (and inside the per-file
`.metadata.json`
sidecars `includeMetadata` adds to REGULAR zip output) is Markdown by default.
On tika-server
change it with `/unpack/all/{handlerType}` or a `content-handler-factory` in
the `config`
part; in a config, the factory alone. Each entry records the handler it was
written with in
@@ -321,9 +336,18 @@ Naming the handler in both the path and a `config` part is
a `400`, as on `/tika
== What each surface produces
The five knobs interact differently per surface. Two defaults changed since
4.0.0: a
-Frictionless package carries `metadata.json` unless `includeMetadata` is
`false` (4.0.0 left
+Frictionless package carries `rmeta.json` unless `includeMetadata` is `false`
(4.0.0 left
it out unless `includeFullMetadata` was `true`), and since 4.2 Frictionless is
the format
-on every surface. REGULAR output is as it was.
+on every surface. REGULAR's layout is as it was; since 4.2 each
`.metadata.json` sidecar is
+the file's row from the metadata list, so it gets the metadata filters and
text inference as
+`rmeta.json` does, and a row a filter removes gets no sidecar. Before 4.2 the
package's `rmeta.json` was
+named `metadata.json`, was not listed in `datapackage.json`, and did not get
the metadata
+filters.
+
+A document with nothing to unpack is still a package: `datapackage.json` and
`rmeta.json`,
+with `rmeta.json` its only resource and no `unpacked/`. `/unpack` returns it
with `200`
+rather than the `204` it gave before 4.2 (REGULAR output still gives `204`).
To get the `204`
+back, send `"unpack-config": {"emptyPackage": "SKIP"}` in the request's
`config` part.
[cols="1,2,2,2"]
|===
@@ -361,7 +385,7 @@ on every surface. REGULAR output is as it was.
|===
`/unpack/all` means the source document's bytes plus metadata in both formats.
Under REGULAR
-that is where the per-file sidecars come from; under Frictionless
`metadata.json` is already
+that is where the per-file sidecars come from; under Frictionless `rmeta.json`
is already
there, so `/all` adds only the source document. `outputMode: DIRECTORY` is
refused on
`/unpack`: an HTTP response is one zip, so a request asking for it gets a
`400`, and a server
whose own `unpack-config` says so fails at startup if the `unpack` endpoint is
enabled -- use
@@ -371,11 +395,15 @@ Output trees for `doc.docx` with three attachments:
[source]
----
-tika-server (one zip) tika-app -Z (loose)
- datapackage.json out/doc.docx.json
- metadata.json out/doc.docx/datapackage.json
- unpacked/1.emf ... out/doc.docx/metadata.json
- + with /all: unpacked/0.docx out/doc.docx/unpacked/1.emf ...
+tika-server (one zip) tika-app -Z doc.docx out (loose)
+ datapackage.json out/datapackage.json
+ rmeta.json out/rmeta.json
+ unpacked/1.emf ... out/unpacked/1.emf ...
+ + with /all: unpacked/0.docx
+ tika-app -Z <dir> out (loose)
+ out/doc.docx/datapackage.json
+ out/doc.docx/rmeta.json
+ out/doc.docx/unpacked/1.emf ...
REGULAR, tika-server (one zip) REGULAR, tika-app -Z
--unpack-format=REGULAR
1.emf out/doc.docx.json
@@ -385,16 +413,48 @@ REGULAR, tika-server (one zip) REGULAR, tika-app
-Z --unpack-format=REGUL
<name>.metadata.json per file
----
-The top-level `doc.docx.json` on the command line is the normal pipes output
for the parse;
-the package's `metadata.json` carries the same rows again so it is
self-contained. Set
-`includeMetadata: false` in a `-c` file to drop it.
+For a single input file the output directory is the package; with no output
directory
+tika-app writes to `doc.docx-unpacked/` (REGULAR output too). For a directory
of inputs each
+gets its own package under its relative path.
+
+The package's `rmeta.json` is the parse's metadata list. On any surface, when
the package
+goes to the container's own emitter no separate `<emitKey>.json` is written;
with
+`includeMetadata: false` the package is the bytes and the manifest, and no
metadata is
+written at all. With `emptyPackage: SKIP`, a document with nothing to unpack
writes nothing.
+A package sent to a different emitter than the container's leaves the
container's metadata
+emit as it was.
The `datapackage.json` file contains:
-* List of all extracted files as "resources"
+* List of all extracted files and `rmeta.json` as "resources"
* SHA256 hash for each file
* MIME type for each file
* File size in bytes
+* `name`: the Frictionless resource name, derived from the path
(`unpacked/3.pdf` ->
+ `unpacked-3.pdf`)
+* `tk:embedded-id`: the extracted file's `tk:embedded-id` (`0` is the source
document, under
+ `includeOriginal`); it matches the file's row in `rmeta.json` if there is
one. The rows are
+ not in id order; match on the id.
+* `tk:resource-name`: the file's original name, when it had one
+
+The package passes `frictionless validate`:
+
+[source,json]
+----
+{
+ "name" : "doc.docx",
+ "resources" : [
+ { "name" : "rmeta.json", "path" : "rmeta.json", "mediatype" :
"application/json", ... },
+ { "name" : "unpacked-3.pdf", "path" : "unpacked/3.pdf", "mediatype" :
"application/pdf",
+ "bytes" : 109786, "hash" : "sha256:115d...", "tk:embedded-id" : 3,
+ "tk:resource-name" : "report.pdf" }
+ ]
+}
+----
+
+One thing the validator reports that is not a manifest problem: it
decompresses resources
+whose extension says `.zip` or `.gz`, so a corrupt or truncated embedded
archive shows up
+as a `compression-error`. The manifest is still valid; the file is what it is.
=== CLI Usage
diff --git a/docs/modules/ROOT/pages/using-tika/cli/index.adoc
b/docs/modules/ROOT/pages/using-tika/cli/index.adoc
index 1835b30749..3d6e3f1d76 100644
--- a/docs/modules/ROOT/pages/using-tika/cli/index.adoc
+++ b/docs/modules/ROOT/pages/using-tika/cli/index.adoc
@@ -380,7 +380,8 @@ activate it. `-a` and `--async` both work.
=== Unpacking attachments
`-z` and `-Z` differ only in depth: `-z` extracts direct attachments (depth
1), `-Z` recurses
-through every level. Both accept a single file or a directory as input.
+through every level. Both accept a single file or a directory as input. With a
single file
+and no output directory, output goes to `<input>-unpacked/`, in either format.
[cols="1,3"]
|===
@@ -399,19 +400,19 @@ through every level. Both accept a single file or a
directory as input.
|Output format: `FRICTIONLESS` (default) or `REGULAR`, the pre-4.2 flat layout.
|`--unpack-mode=<mode>`
-|Output mode: `DIRECTORY` (default) or `ZIPPED`. `DIRECTORY` lays the package
out under
-`<out>/<input>/` (`datapackage.json`, `metadata.json`, `unpacked/...`);
`ZIPPED` writes
-`<out>/<input>-frictionless.zip`.
+|Output mode: `DIRECTORY` (default) or `ZIPPED`. `DIRECTORY` with a single
input file makes
+`<out>` the package (`datapackage.json`, `rmeta.json`, `unpacked/...`); with
an input directory each file gets `<out>/<input>/`. `ZIPPED`
+writes `<out>/<input>-frictionless.zip`.
|`--unpack-include-metadata`
-|Metadata for every extracted file. A Frictionless package carries
`metadata.json` by default;
+|Metadata for every extracted file. A Frictionless package carries
`rmeta.json` by default;
this adds a `.metadata.json` sidecar per file to REGULAR zip output.
|===
A `-c`/`--config` file's `unpack-config` is honoured and the flags above are
applied on top
of it. That is the only way to get the source document itself into the output
(`"unpack-config": {"includeOriginal": true}`); no flag does it. File naming
stays the
-CLI's own (`<input>/unpacked/00000001.<ext>`, or
`<input>-embed/00000001.<ext>` under
+CLI's own (`unpacked/1.<ext>`, or `<input>-embed/1.<ext>` under
`REGULAR`, detected suffixes, loose files unless `--unpack-mode=ZIPPED`)
regardless of the
file's `suffixStrategy`, `zeroPadName`, `embeddedIdPrefix`, `keyBaseStrategy`
or `outputMode`. See
xref:pipes/unpack-config.adoc#_what_each_surface_produces[what each surface
produces].
diff --git a/docs/modules/ROOT/pages/using-tika/server/index.adoc
b/docs/modules/ROOT/pages/using-tika/server/index.adoc
index 220e61c97d..fd5c2169d5 100644
--- a/docs/modules/ROOT/pages/using-tika/server/index.adoc
+++ b/docs/modules/ROOT/pages/using-tika/server/index.adoc
@@ -212,7 +212,7 @@ config's, then the endpoint default (Markdown).
Output format: name the handler once. A format segment sets it --
`/tika/config/xml`,
`/rmeta/config/xml`, and `/unpack/all/{handlerType}` (or
`/unpack/all/config/{handlerType}`
with a config part), which selects how `tk:content` is rendered in the
metadata `/all` adds
-(`/unpack/all/ignore` for none). Plain `/unpack` takes no segment: the
`metadata.json` a
+(`/unpack/all/ignore` for none). Plain `/unpack` takes no segment: the
`rmeta.json` a
Frictionless package carries there uses the config's handler, or the default.
With no
segment, a `content-handler-factory` in the
`config` part does. Doing both is a `400`, whether or not the
@@ -603,7 +603,7 @@ before the exception are discarded. Known gap.
By default exception detail is returned in full. Stack traces and their
messages can contain the
spooled file's path, the source filename, and fragments of the document. That
detail appears in
`tk:exception:*` metadata on **successful** parses as well as in error bodies
— including inside
-the `metadata.json` of an `/unpack` zip.
+the `rmeta.json` of an `/unpack` zip.
One config setting governs every one of those channels — the metadata values,
every error body
built from an exception (`422`, `500`; a `400` echoes the caller's own request
problem, `415` and
diff --git a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
index f52cdfab27..5127226601 100644
--- a/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
+++ b/tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
@@ -914,7 +914,7 @@ public class TikaCLI {
out.println(" sha256, sha384, sha512)");
out.println(" --encoding=X Use output encoding X");
out.println(" --password=X Use document password X");
- out.println(" -z or --extract Extract all attachements into
current directory");
+ out.println(" -z or --extract Extract attachments (a single
file: into <input>-unpacked/)");
out.println(" --extract-dir=<dir> Specify target directory for
-z");
out.println(" --maxEmbeddedDepth=X Maximum depth for embedded
document extraction");
out.println(" --maxEmbeddedCount=X Maximum number of embedded
documents to extract");
@@ -998,7 +998,7 @@ public class TikaCLI {
out.println(" --unpack-format=<format> Output format:
FRICTIONLESS (default) or REGULAR");
out.println(" --unpack-mode=<mode> Output mode: DIRECTORY
(default: loose files) or ZIPPED");
out.println(" --unpack-include-metadata Metadata for every
extracted file. A Frictionless package");
- out.println(" carries metadata.json by
default; this adds per-file");
+ out.println(" carries rmeta.json by
default; this adds per-file");
out.println(" sidecars to REGULAR zip
output");
out.println();
out.println();
diff --git a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
index c105d56c82..8e3a5d801f 100644
--- a/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
+++ b/tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
@@ -36,10 +36,14 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
+import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.HexFormat;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.databind.JsonNode;
@@ -367,33 +371,34 @@ public class TikaCLITest {
public void testRUnpack() throws Exception {
//TODO -- rework this to use two separate emitters
//one for bytes and one for json
- // TODO: 00000001.bin extension may be wrong - see
~/Desktop/unpack-discussion/mime-todo.txt
+ // TODO: 1.bin extension may be wrong - see
~/Desktop/unpack-discussion/mime-todo.txt
+ // a single input file: the output dir is the package
String[] expectedChildren = new String[]{
- "testPDFPackage.pdf.json",
- //the first two test that the default single file config is
working
- "testPDFPackage.pdf/unpacked/00000001.bin",
- "testPDFPackage.pdf/unpacked/00000002.jpg",
- "testPDFPackage.pdf/unpacked/00000003.pdf",
- "testPDFPackage.pdf/unpacked/00000004.pdf"};
- testRecursiveUnpack("testPDFPackage.pdf", expectedChildren, 2);
+ "datapackage.json",
+ "rmeta.json",
+ "unpacked/1.bin",
+ "unpacked/2.jpg",
+ "unpacked/3.pdf",
+ "unpacked/4.pdf"};
+ testRecursiveUnpack("testPDFPackage.pdf", expectedChildren, 3);
}
@Test
public void testPSTRUnpack() throws Exception {
// TODO: The .bin extensions for embedded .msg files are wrong - they
should be .msg
// CONTENT_TYPE is not being set for embedded documents - see
~/Desktop/unpack-discussion/mime-todo.txt
- String[] expectedChildren = new String[]{"testPST.pst.json",
- "testPST.pst/unpacked/00000007.bin",
- "testPST.pst/unpacked/00000001.bin",
- "testPST.pst/unpacked/00000008.bin",
- "testPST.pst/unpacked/00000004.bin",
- "testPST.pst/unpacked/00000003.bin",
- "testPST.pst/unpacked/00000002.bin",
- "testPST.pst/unpacked/00000005.bin",
- "testPST.pst/unpacked/00000009.docx",
- "testPST.pst/unpacked/00000006.bin"};
- testRecursiveUnpack("testPST.pst", expectedChildren, 2);
- try (Reader reader =
Files.newBufferedReader(extractDir.resolve("testPST.pst.json"))) {
+ String[] expectedChildren = new String[]{"rmeta.json",
+ "unpacked/7.bin",
+ "unpacked/1.bin",
+ "unpacked/8.bin",
+ "unpacked/4.bin",
+ "unpacked/3.bin",
+ "unpacked/2.bin",
+ "unpacked/5.bin",
+ "unpacked/9.docx",
+ "unpacked/6.bin"};
+ testRecursiveUnpack("testPST.pst", expectedChildren, 3);
+ try (Reader reader =
Files.newBufferedReader(extractDir.resolve("rmeta.json"))) {
List<Metadata> metadataList = JsonMetadataList.fromJson(reader);
for (Metadata m : metadataList) {
String content = m.get(TikaCoreProperties.TIKA_CONTENT);
@@ -449,30 +454,30 @@ public class TikaCLITest {
@Test
public void testExtractSimple() throws Exception {
- assertShallow("coffee.xls", 11, "coffee.xls.json",
"coffee.xls/unpacked/00000001.emf",
- "coffee.xls/unpacked/00000006.cdx",
"coffee.xls/unpacked/00000005.png");
+ assertShallow("coffee.xls", 10, "coffee.xls/rmeta.json",
"coffee.xls/unpacked/1.emf",
+ "coffee.xls/unpacked/6.cdx", "coffee.xls/unpacked/5.png");
}
@Test
public void testExtractAbsolute() throws Exception {
- assertShallow("testZip_absolutePath.zip", 5,
"testZip_absolutePath.zip.json",
- "testZip_absolutePath.zip/unpacked/00000001.bin");
+ assertShallow("testZip_absolutePath.zip", 4,
"testZip_absolutePath.zip/rmeta.json",
+ "testZip_absolutePath.zip/unpacked/1.bin");
}
@Test
public void testExtractRelative() throws Exception {
- assertShallow("testZip_relative.zip", 4, "testZip_relative.zip.json");
+ assertShallow("testZip_relative.zip", 3,
"testZip_relative.zip/rmeta.json");
}
@Test
public void testExtractOverlapping() throws Exception {
// overlapping names are handled by numbering
- assertShallow("testZip_overlappingNames.zip", 5,
"testZip_overlappingNames.zip.json");
+ assertShallow("testZip_overlappingNames.zip", 4,
"testZip_overlappingNames.zip/rmeta.json");
}
@Test
public void testExtract0x00() throws Exception {
- assertShallow("testZip_zeroByte.zip", 4, "testZip_zeroByte.zip.json");
+ assertShallow("testZip_zeroByte.zip", 3,
"testZip_zeroByte.zip/rmeta.json");
}
private void testRecursiveUnpack(String targetFile, String[]
expectedChildrenFileNames) throws Exception {
@@ -624,23 +629,84 @@ public class TikaCLITest {
Set<String> fileNames = getFileNames(extractDir);
// Should have extracted files in the specified directory, not current
dir
- assertTrue(fileNames.stream().anyMatch(f -> f.endsWith(".json")),
- "Should have a .json metadata file in extractDir, got: " +
fileNames);
- assertTrue(fileNames.stream().anyMatch(f -> f.contains("/unpacked/")),
+ assertTrue(fileNames.contains("rmeta.json"),
+ "Should have rmeta.json in extractDir, got: " + fileNames);
+ assertTrue(fileNames.stream().anyMatch(f -> f.startsWith("unpacked/")),
"Should have extracted embedded files in extractDir, got: " +
fileNames);
}
- /** TIKA-4681: the default is a Frictionless package laid out as loose
files. */
+ /**
+ * TIKA-4681: the default is a Frictionless package laid out as loose
files; for a single
+ * input file the output dir is the package (TIKA-4971).
+ */
@Test
public void testDefaultIsALooseFrictionlessPackage() throws Exception {
Set<String> fileNames = unpack("-Z");
- assertTrue(fileNames.stream().anyMatch(f ->
f.endsWith("/datapackage.json")),
- "package should be laid out as a directory, got: " +
fileNames);
- assertTrue(fileNames.stream().anyMatch(f -> f.contains("/unpacked/")),
fileNames.toString());
- assertFalse(fileNames.stream().anyMatch(f -> f.endsWith(".zip") &&
!f.contains("/unpacked/")),
+ assertTrue(fileNames.contains("datapackage.json"),
+ "package should be laid out at the output dir's root, got: " +
fileNames);
+ assertTrue(fileNames.contains("rmeta.json"),
+ "a Frictionless package carries rmeta.json by default: " +
fileNames);
+ assertTrue(fileNames.stream().anyMatch(f ->
f.matches("unpacked/1\\.[^/]+")),
+ "embedded files are unpadded and under unpacked/: " +
fileNames);
+ assertFalse(fileNames.stream().anyMatch(f -> f.endsWith(".zip") &&
!f.startsWith("unpacked/")),
"no zip when no mode was asked for: " + fileNames);
- assertTrue(fileNames.stream().anyMatch(f ->
f.endsWith("/metadata.json")),
- "a Frictionless package carries metadata.json by default: " +
fileNames);
+ assertFalse(fileNames.contains("test_recursive_embedded.docx.json"),
+ "rmeta.json already carries the metadata list: " + fileNames);
+ }
+
+ /** TIKA-4971: rmeta.json is a resource of the package, with its size and
hash. */
+ @Test
+ public void testManifestListsRmetaJson() throws Exception {
+ unpack("-Z");
+ byte[] metadataJson =
Files.readAllBytes(extractDir.resolve("rmeta.json"));
+ JsonNode resources = new ObjectMapper().readTree(
+
extractDir.resolve("datapackage.json").toFile()).get("resources");
+ JsonNode resource = null;
+ for (JsonNode r : resources) {
+ if ("rmeta.json".equals(r.get("path").asText())) {
+ resource = r;
+ }
+ }
+ assertNotNull(resource, "rmeta.json not in " + resources);
+ assertEquals("application/json", resource.get("mediatype").asText());
+ assertEquals(metadataJson.length, resource.get("bytes").asLong());
+ assertEquals("sha256:" + HexFormat.of().formatHex(
+ MessageDigest.getInstance("SHA-256").digest(metadataJson)),
resource.get("hash").asText());
+ }
+
+ /**
+ * TIKA-4971: the rules frictionless validate enforces on resource names
-- present, unique,
+ * lowercase [a-z0-9._-] -- and tk:embedded-id resolves to the file's row
in rmeta.json.
+ */
+ @Test
+ public void testManifestIsValidFrictionless() throws Exception {
+ unpack("-Z");
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode resources =
mapper.readTree(extractDir.resolve("datapackage.json").toFile())
+ .get("resources");
+ Map<String, JsonNode> rowsById = new HashMap<>();
+ for (JsonNode row :
mapper.readTree(extractDir.resolve("rmeta.json").toFile())) {
+ if (row.has("tk:embedded-id")) {
+ rowsById.put(row.get("tk:embedded-id").asText(), row);
+ }
+ }
+ Set<String> names = new HashSet<>();
+ int unpacked = 0;
+ for (JsonNode r : resources) {
+ String name = r.path("name").asText(null);
+ assertNotNull(name, "every resource needs a name: " + r);
+ assertTrue(name.matches("[a-z0-9._-]+"), "invalid Frictionless
name: " + r);
+ assertTrue(names.add(name), "duplicate name: " + r);
+ if (!r.get("path").asText().startsWith("unpacked/")) {
+ continue;
+ }
+ unpacked++;
+ JsonNode row = rowsById.get(r.get("tk:embedded-id").asText());
+ assertNotNull(row, "tk:embedded-id has no rmeta.json row: " + r);
+ assertEquals(row.path("tk:resource-name").asText(null),
+ r.path("tk:resource-name").asText(null), r.toString());
+ }
+ assertTrue(unpacked > 0, resources.toString());
}
@Test
@@ -654,7 +720,7 @@ public class TikaCLITest {
@Test
public void testRegularFormatIsTheFlatLayout() throws Exception {
Set<String> fileNames = unpack("-Z", "--unpack-format=REGULAR");
- assertTrue(fileNames.stream().anyMatch(f ->
f.matches("test_recursive_embedded\\.docx-embed/0+1\\.[^/]+")),
+ assertTrue(fileNames.stream().anyMatch(f ->
f.matches("test_recursive_embedded\\.docx-embed/1\\.[^/]+")),
fileNames.toString());
assertFalse(fileNames.stream().anyMatch(f ->
f.endsWith("datapackage.json") || f.contains("/unpacked/")),
"REGULAR has no manifest and no unpacked/: " + fileNames);
@@ -672,11 +738,10 @@ public class TikaCLITest {
@Test
public void testFrictionlessIncludeMetadata() throws Exception {
Set<String> fileNames = unpack("-Z", "--unpack-include-metadata");
- String metadataJson = fileNames.stream().filter(f ->
f.endsWith("/metadata.json")).findFirst()
- .orElseThrow(() -> new AssertionError("no metadata.json in " +
fileNames));
- String content = Files.readString(extractDir.resolve(metadataJson));
+ assertTrue(fileNames.contains("rmeta.json"), "no rmeta.json in " +
fileNames);
+ String content = Files.readString(extractDir.resolve("rmeta.json"));
assertTrue(content.contains("\"tk:embedded-depth\" : \"0\""),
- "metadata.json should carry the container row: " + content);
+ "rmeta.json should carry the container row: " + content);
assertTrue(content.contains("\"tk:content\""), content);
}
@@ -684,9 +749,9 @@ public class TikaCLITest {
@Test
public void testConfigFileUnpackConfigIsHonoured() throws Exception {
Set<String> fileNames = unpack("-Z", "--config=" + CONFIGS_DIR +
"/tika-config-unpack-original.json");
- assertTrue(fileNames.stream().anyMatch(f ->
f.matches(".*/unpacked/0+\\.[^/]+")),
- "includeOriginal from -c should add the container as
00000000.<ext>: " + fileNames);
- assertTrue(fileNames.stream().anyMatch(f ->
f.matches(".*/unpacked/0+1\\.[^/]+")),
+ assertTrue(fileNames.stream().anyMatch(f ->
f.matches("unpacked/0\\.[^/]+")),
+ "includeOriginal from -c should add the container as 0.<ext>:
" + fileNames);
+ assertTrue(fileNames.stream().anyMatch(f ->
f.matches("unpacked/1\\.[^/]+")),
"-c must not change the CLI's file naming or packaging: " +
fileNames);
}
diff --git
a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
index 9311b1abc6..227136e7ce 100644
---
a/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
+++
b/tika-pipes/tika-async-cli/src/main/java/org/apache/tika/async/cli/TikaAsyncCLI.java
@@ -89,7 +89,7 @@ public class TikaAsyncCLI {
options.addOption(null, "unpack-mode", true,
"output mode for unpacking: DIRECTORY (default: loose files)
or ZIPPED");
options.addOption(null, "unpack-include-metadata", false,
- "metadata for every extracted file: a Frictionless package
carries metadata.json "
+ "metadata for every extracted file: a Frictionless package
carries rmeta.json "
+ "by default; this adds per-file sidecars to REGULAR
zip output");
options.addOption(null, "on-exists", true,
"behavior when an output file already exists: exception
(default), replace or skip");
@@ -290,7 +290,12 @@ public class TikaAsyncCLI {
inputDir = inString;
// Only set default outputDir if not already specified via -o
if (outputDir == null) {
- if (Files.isRegularFile(inputPath)) {
+ if (Files.isRegularFile(inputPath)
+ && extractBytesMode !=
SimpleAsyncConfig.ExtractBytesMode.NONE) {
+ // any format: a -c file may set it, and it isn't loaded
yet
+ outputDir = Paths.get(inputPath.getFileName() +
"-unpacked")
+ .toAbsolutePath().toString();
+ } else if (Files.isRegularFile(inputPath)) {
outputDir = Paths.get(".").toAbsolutePath().toString();
} else {
outputDir =
Paths.get("output").toAbsolutePath().toString();
@@ -402,7 +407,6 @@ public class TikaAsyncCLI {
config.setEmitter(TikaConfigAsyncWriter.EMITTER_NAME);
config.setSuffixStrategy(UnpackConfig.SUFFIX_STRATEGY.DETECTED);
config.setEmbeddedIdPrefix("-");
- config.setZeroPadName(8);
config.setKeyBaseStrategy(UnpackConfig.KEY_BASE_STRATEGY.DEFAULT);
// Packaging is the CLI's too: loose files unless --unpack-mode asks
for a zip
@@ -419,6 +423,13 @@ public class TikaAsyncCLI {
if (asyncConfig.isUnpackIncludeMetadata()) {
config.setIncludeMetadata(true);
}
+ // Single input file: the output dir is the package; one per input
file otherwise
+ if (config.getOutputFormat() == UnpackConfig.OUTPUT_FORMAT.FRICTIONLESS
+ && config.getOutputMode() == UnpackConfig.OUTPUT_MODE.DIRECTORY
+ && asyncConfig.getInputDir() != null
+ && Files.isRegularFile(Paths.get(asyncConfig.getInputDir()))) {
+ config.setPackageRoot("");
+ }
parseContext.set(UnpackConfig.class, config);
}
diff --git
a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java
b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java
index 003973e060..cd50ad3a15 100644
---
a/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java
+++
b/tika-pipes/tika-async-cli/src/test/java/org/apache/tika/async/cli/AsyncCliParserTest.java
@@ -119,6 +119,18 @@ public class AsyncCliParserTest {
new String[]{"-i", "input", "-o", "output", "--on-exists",
"bogus"}));
}
+ /** TIKA-4971: a single file's default unpack dir doesn't depend on the
format. */
+ @Test
+ public void testSingleFileUnpackDefaultsOutputDir(@TempDir Path tmp)
throws Exception {
+ Path input = Files.writeString(tmp.resolve("doc.pdf"), "x");
+ for (String format : new String[]{"FRICTIONLESS", "REGULAR"}) {
+ SimpleAsyncConfig config = TikaAsyncCLI.parseCommandLine(
+ new String[]{"-z", "--unpack-format=" + format,
input.toString()});
+
assertEquals(Path.of("doc.pdf-unpacked").toAbsolutePath().toString(),
+ config.getOutputDir(), format);
+ }
+ }
+
@Test
public void testFileListWithInputDir(@TempDir Path tmp) throws Exception {
Path fileList = tmp.resolve("files.txt");
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/FrictionlessUnpackHandler.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/FrictionlessUnpackHandler.java
index 0ceb8acc76..fc2899a1e0 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/FrictionlessUnpackHandler.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/FrictionlessUnpackHandler.java
@@ -26,6 +26,7 @@ import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import org.apache.commons.io.FileUtils;
@@ -160,7 +161,7 @@ public class FrictionlessUnpackHandler extends
AbstractUnpackHandler implements
DataPackage dataPackage = new DataPackage(containerName);
// Add all embedded files with unpacked/ prefix
- for (FrictionlessFileInfo fileInfo : embeddedFiles) {
+ for (FrictionlessFileInfo fileInfo : getEmbeddedFiles()) {
String path = UNPACKED_DIR + "/" + fileInfo.fileName();
String originalName =
fileInfo.metadata().get(TikaCoreProperties.RESOURCE_NAME_KEY);
dataPackage.addResource(FrictionlessResource.create(
@@ -168,6 +169,7 @@ public class FrictionlessUnpackHandler extends
AbstractUnpackHandler implements
fileInfo.mediatype(),
fileInfo.bytes(),
fileInfo.sha256Hash(),
+ fileInfo.id(),
originalName
));
}
@@ -189,11 +191,11 @@ public class FrictionlessUnpackHandler extends
AbstractUnpackHandler implements
return unpackedDirectory;
}
- /**
- * Returns information about all embedded files.
- */
+ /** All embedded files, in embedded-id order, not parse-completion order.
*/
public List<FrictionlessFileInfo> getEmbeddedFiles() {
- return embeddedFiles;
+ return embeddedFiles.stream()
+ .sorted(Comparator.comparingInt(FrictionlessFileInfo::id))
+ .toList();
}
/**
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java
index bc02d5ce6a..5919f05fe0 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/UnpackConfig.java
@@ -17,6 +17,7 @@
package org.apache.tika.pipes.core.extractor;
import java.io.Serializable;
+import java.util.Locale;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -124,6 +125,13 @@ public class UnpackConfig implements Serializable {
throw new IllegalArgumentException("can't parse OUTPUT_MODE: " +
s);
}
}
+ /** FRICTIONLESS: a document with nothing to unpack still gets a package,
or none. */
+ public enum EMPTY_PACKAGE {
+ WRITE,
+ /** No package; tika-server's /unpack answers 204 */
+ SKIP
+ }
+
private int zeroPadName = 0;
private SUFFIX_STRATEGY suffixStrategy = SUFFIX_STRATEGY.NONE;
@@ -151,6 +159,11 @@ public class UnpackConfig implements Serializable {
// Frictionless Data Package options
private OUTPUT_FORMAT outputFormat = OUTPUT_FORMAT.FRICTIONLESS;
private OUTPUT_MODE outputMode = OUTPUT_MODE.ZIPPED;
+ // DIRECTORY-mode package root; null: the container's emit key
+ private String packageRoot = null;
+ // datapackage.json name; null: the fetch key's file name
+ private String packageName = null;
+ private EMPTY_PACKAGE emptyPackage = EMPTY_PACKAGE.WRITE;
/**
* Create an UnpackConfig with default settings.
@@ -173,6 +186,9 @@ public class UnpackConfig implements Serializable {
c.maxUnpackBytes = maxUnpackBytes;
c.outputFormat = outputFormat;
c.outputMode = outputMode;
+ c.packageRoot = packageRoot;
+ c.packageName = packageName;
+ c.emptyPackage = emptyPackage;
return c;
}
@@ -264,7 +280,7 @@ public class UnpackConfig implements Serializable {
}
/**
- * Metadata for every extracted file: {@code metadata.json} under
FRICTIONLESS, a sidecar
+ * Metadata for every extracted file: {@code rmeta.json} under
FRICTIONLESS, a sidecar
* per file under REGULAR zip output, nothing under REGULAR loose output.
Unset means on
* for FRICTIONLESS, off for REGULAR. Not a bean getter: the class is
introspected.
*/
@@ -368,6 +384,46 @@ public class UnpackConfig implements Serializable {
setIncludeMetadata(includeFullMetadata);
}
+ /**
+ * Emit key prefix of a FRICTIONLESS package in DIRECTORY mode. Unset: the
container's
+ * emit key, one package per container. Empty: the emitter's root.
+ */
+ public String getPackageRoot() {
+ return packageRoot;
+ }
+
+ public void setPackageRoot(String packageRoot) {
+ this.packageRoot = packageRoot;
+ }
+
+ /**
+ * {@code name} of a FRICTIONLESS package. Unset: the fetch key's file
name. Blank: no
+ * {@code name} (tika-server sets the caller's filename, or blank when
there is none).
+ */
+ public String getPackageName() {
+ return packageName;
+ }
+
+ public void setPackageName(String packageName) {
+ this.packageName = packageName;
+ }
+
+ /**
+ * FRICTIONLESS: whether a document with nothing to unpack gets a package
(its
+ * datapackage.json and rmeta.json) or none, which tika-server's /unpack
answers with 204.
+ */
+ public EMPTY_PACKAGE getEmptyPackage() {
+ return emptyPackage;
+ }
+
+ public void setEmptyPackage(EMPTY_PACKAGE emptyPackage) {
+ this.emptyPackage = emptyPackage;
+ }
+
+ public void setEmptyPackage(String emptyPackage) {
+
setEmptyPackage(EMPTY_PACKAGE.valueOf(emptyPackage.toUpperCase(Locale.ROOT)));
+ }
+
@Override
public String toString() {
return "UnpackConfig{" + "zeroPadName=" + zeroPadName + ",
suffixStrategy=" +
@@ -376,7 +432,8 @@ public class UnpackConfig implements Serializable {
", keyBaseStrategy=" + keyBaseStrategy + ", emitKeyBase='" +
emitKeyBase + '\'' +
", zipEmbeddedFiles=" + zipEmbeddedFiles + ",
includeMetadata=" + includeMetadata +
", maxUnpackBytes=" + maxUnpackBytes + ", outputFormat=" +
outputFormat +
- ", outputMode=" + outputMode + '}';
+ ", outputMode=" + outputMode + ", packageRoot='" + packageRoot
+ '\'' +
+ ", packageName='" + packageName + '\'' + ", emptyPackage=" +
emptyPackage + '}';
}
@Override
@@ -395,7 +452,10 @@ public class UnpackConfig implements Serializable {
Objects.equals(includeMetadata, config.includeMetadata) &&
maxUnpackBytes == config.maxUnpackBytes &&
outputFormat == config.outputFormat &&
- outputMode == config.outputMode;
+ outputMode == config.outputMode &&
+ Objects.equals(packageRoot, config.packageRoot) &&
+ Objects.equals(packageName, config.packageName) &&
+ emptyPackage == config.emptyPackage;
}
@Override
@@ -412,6 +472,9 @@ public class UnpackConfig implements Serializable {
result = 31 * result + Long.hashCode(maxUnpackBytes);
result = 31 * result + Objects.hashCode(outputFormat);
result = 31 * result + Objects.hashCode(outputMode);
+ result = 31 * result + Objects.hashCode(packageRoot);
+ result = 31 * result + Objects.hashCode(packageName);
+ result = 31 * result + Objects.hashCode(emptyPackage);
return result;
}
}
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/DataPackage.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/DataPackage.java
index 5078eaddf8..5c10280fb0 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/DataPackage.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/DataPackage.java
@@ -31,6 +31,8 @@ import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
+import org.apache.tika.utils.StringUtils;
+
/**
* Represents a Frictionless Data Package manifest (datapackage.json).
* See: https://specs.frictionlessdata.io/data-package/
@@ -94,8 +96,9 @@ public class DataPackage {
* @return sanitized name
*/
private static String sanitizeName(String name) {
- if (name == null) {
- return "unknown";
+ // name is optional; an empty one is invalid
+ if (StringUtils.isBlank(name)) {
+ return null;
}
// Replace spaces, keep alphanumeric, dots, hyphens, underscores
return name.toLowerCase(java.util.Locale.ROOT)
@@ -152,19 +155,6 @@ public class DataPackage {
this.resources.add(resource);
}
- /**
- * Adds a resource to this data package with all parameters.
- *
- * @param path relative path within package
- * @param mediatype MIME type
- * @param bytes file size
- * @param hash SHA256 hash with "sha256:" prefix
- * @param name optional original filename
- */
- public void addResource(String path, String mediatype, long bytes, String
hash, String name) {
- this.resources.add(FrictionlessResource.create(path, mediatype, bytes,
hash, name));
- }
-
/**
* Serializes this DataPackage to JSON.
*
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResource.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResource.java
index 600aacd8d5..1994fdf434 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResource.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResource.java
@@ -16,55 +16,55 @@
*/
package org.apache.tika.pipes.core.extractor.frictionless;
+import java.util.Locale;
+
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
/**
* Represents a resource entry in a Frictionless Data Package.
* See: https://specs.frictionlessdata.io/data-resource/
*
- * @param path The relative path to the file within the package (e.g.,
"unpacked/00000001.pdf")
- * @param mediatype The MIME type of the file (e.g., "application/pdf")
- * @param bytes The file size in bytes
- * @param hash The SHA256 hash in format "sha256:hexstring"
- * @param name Optional original filename if available
+ * @param name Frictionless resource name, derived from {@code path}:
required,
+ * unique, lowercase {@code [a-z0-9._-]}
+ * @param path The relative path to the file within the package (e.g.,
"unpacked/1.pdf")
+ * @param mediatype The MIME type of the file (e.g., "application/pdf")
+ * @param bytes The file size in bytes
+ * @param hash The SHA256 hash in format "sha256:hexstring"
+ * @param embeddedId the file's {@code tk:embedded-id}, 0 for the container;
matches its
+ * row in rmeta.json, if present
+ * @param resourceName {@code tk:resource-name}, the original filename, if
there was one
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({"name", "path", "mediatype", "bytes", "hash",
"tk:embedded-id", "tk:resource-name"})
public record FrictionlessResource(
+ @JsonProperty("name") String name,
@JsonProperty("path") String path,
@JsonProperty("mediatype") String mediatype,
@JsonProperty("bytes") long bytes,
@JsonProperty("hash") String hash,
- @JsonProperty("name") String name
+ @JsonProperty("tk:embedded-id") Integer embeddedId,
+ @JsonProperty("tk:resource-name") String resourceName
) {
- /**
- * Creates a FrictionlessResource with all fields.
- *
- * @param path relative path within package
- * @param mediatype MIME type
- * @param bytes file size
- * @param hash SHA256 hash with "sha256:" prefix
- * @param name optional original filename
- * @return new FrictionlessResource
- */
+ /** A package file that is not an embedded document, e.g. rmeta.json. */
public static FrictionlessResource create(String path, String mediatype,
long bytes,
- String hash, String name) {
- return new FrictionlessResource(path, mediatype, bytes, hash, name);
+ String hash) {
+ return new FrictionlessResource(nameFromPath(path), path, mediatype,
bytes, hash,
+ null, null);
}
- /**
- * Creates a FrictionlessResource without the optional name field.
- *
- * @param path relative path within package
- * @param mediatype MIME type
- * @param bytes file size
- * @param hash SHA256 hash with "sha256:" prefix
- * @return new FrictionlessResource
- */
+ /** An embedded document under unpacked/. */
public static FrictionlessResource create(String path, String mediatype,
long bytes,
- String hash) {
- return new FrictionlessResource(path, mediatype, bytes, hash, null);
+ String hash, int embeddedId,
String resourceName) {
+ return new FrictionlessResource(nameFromPath(path), path, mediatype,
bytes, hash,
+ embeddedId, resourceName);
+ }
+
+ /** Paths are unique within a package, so the names derived from them are
too. */
+ static String nameFromPath(String path) {
+ return path.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9._-]", "-");
}
/**
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java
index e3fcf569bd..53524d02c6 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/ConnectionHandler.java
@@ -160,11 +160,12 @@ public class ConnectionHandler implements Runnable,
Closeable {
}
PipesWorker w = tLastWorker;
TIMING_LOG.trace("WORKER_TIMING handler={} id={} req_deser_us={}
ctx_merge_us={}"
- + " handoff_us={} fetch_us={} parse_us={} emit_us={}
worker_wall_us={}"
+ + " handoff_us={} fetch_us={} parse_us={}
prepare_us={} emit_us={} worker_wall_us={}"
+ " intermediate_us={} resp_ser_us={} resp_write_us={}
resp_ack_us={}"
+ " resp_bytes={}",
handlerId, id, us(tReqDeserNanos), us(tCtxMergeNanos),
us(tHandoffNanos),
us(w == null ? -1 : w.getFetchNanos()), us(w == null ? -1 :
w.getParseNanos()),
+ us(w == null ? -1 : w.getPrepareNanos()),
us(w == null ? -1 : w.getEmitNanos()), us(w == null ? -1 :
w.getWallNanos()),
us(protocolIO.getLastIntermediateNanos()),
us(protocolIO.getLastRespSerNanos()),
us(protocolIO.getLastRespWriteNanos()),
us(protocolIO.getLastRespAckNanos()),
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
index 535820b223..e31fe8cce4 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/EmitHandler.java
@@ -73,8 +73,32 @@ class EmitHandler {
this.directEmitThresholdBytes = directEmitThresholdBytes;
}
- public PipesResult emitParseData(FetchEmitTuple t,
MetadataListAndEmbeddedBytes parseData, ParseContext parseContext) {
- PipesResult result = emitParseDataInternal(t, parseData, parseContext);
+ /** Outcome of {@link #prepare}: the container stack trace and whether the
tuple emits. */
+ record Prepared(String stack, boolean emit) {
+ }
+
+ /**
+ * Text inference, then the metadata filters, then user metadata, in
place. Runs once per
+ * parse, before anything that writes the metadata list (the unpack
package, the emit).
+ */
+ Prepared prepare(FetchEmitTuple t, MetadataListAndEmbeddedBytes parseData,
ParseContext parseContext) {
+ String stack = getContainerStacktrace(t, parseData.getMetadataList());
+ boolean emit = StringUtils.isBlank(stack) ||
+ t.getOnParseException() ==
FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT;
+ if (emit) {
+ runTextInference(parseData, parseContext);
+ }
+ //we need to apply the metadata filter after we pull out the stacktrace
+ filterMetadata(parseData, parseContext);
+ if (emit) {
+ injectUserMetadata(t.getMetadata(), parseData.getMetadataList());
+ }
+ return new Prepared(stack, emit);
+ }
+
+ public PipesResult emitParseData(FetchEmitTuple t,
MetadataListAndEmbeddedBytes parseData,
+ Prepared prepared, ParseContext
parseContext) {
+ PipesResult result = emitParseDataInternal(t, parseData, prepared,
parseContext);
// Deadline exhaustion is a document-level fact, independent of which
SUCCESS
// variant (parsed-only, emitted, passed back...) the rest of this
class chose --
// relabel any of them uniformly rather than special-casing every
return site
@@ -112,19 +136,10 @@ class EmitHandler {
return parseRecord.isTaskDeadlineReached();
}
- private PipesResult emitParseDataInternal(FetchEmitTuple t,
MetadataListAndEmbeddedBytes parseData, ParseContext parseContext) {
- long start = System.currentTimeMillis();
- String stack = getContainerStacktrace(t, parseData.getMetadataList());
- FetchEmitTuple.ON_PARSE_EXCEPTION onParseException =
t.getOnParseException();
- boolean emit = StringUtils.isBlank(stack) ||
- onParseException == FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT;
- if (emit) {
- runTextInference(parseData, parseContext);
- }
- //we need to apply the metadata filter after we pull out the stacktrace
- filterMetadata(parseData, parseContext);
- if (emit) {
- injectUserMetadata(t.getMetadata(), parseData.getMetadataList());
+ private PipesResult emitParseDataInternal(FetchEmitTuple t,
MetadataListAndEmbeddedBytes parseData,
+ Prepared prepared, ParseContext
parseContext) {
+ String stack = prepared.stack();
+ if (prepared.emit()) {
EmitKey emitKey = t.getEmitKey();
if (StringUtils.isBlank(emitKey.getEmitKey())) {
emitKey = new EmitKey(emitKey.getEmitterId(),
t.getFetchKey().getFetchKey());
@@ -210,6 +225,9 @@ class EmitHandler {
ParseMode parseMode = parseContext.get(ParseMode.class);
if (parseMode == ParseMode.CONTENT_ONLY && emitter instanceof
StreamEmitter) {
emitContentOnly((StreamEmitter) emitter, emitKey, parseData,
parseContext);
+ } else if (parseData.isPackageReplacesEmit()) {
+ LOG.debug("Frictionless package replaces the metadata emit for
{}",
+ emitKey.getEmitKey());
} else if (isExtractEmbeddedBytes &&
parseData.toBePackagedForStreamEmitter()) {
emitContentsAndBytes(emitter, emitKey, parseData);
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/MetadataListAndEmbeddedBytes.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/MetadataListAndEmbeddedBytes.java
index 5f371696fc..0aa1cda455 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/MetadataListAndEmbeddedBytes.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/MetadataListAndEmbeddedBytes.java
@@ -31,6 +31,7 @@ class MetadataListAndEmbeddedBytes {
List<Metadata> metadataList;
final UnpackHandler unpackHandler;
+ private boolean packageReplacesEmit = false;
public MetadataListAndEmbeddedBytes(List<Metadata> metadataList,
UnpackHandler unpackHandler) {
@@ -60,6 +61,18 @@ class MetadataListAndEmbeddedBytes {
return unpackHandler != null;
}
+ /**
+ * True when the Frictionless output went to the container's own
emitter: the
+ * package's rmeta.json (or its absence) stands in for the metadata
emit.
+ */
+ public boolean isPackageReplacesEmit() {
+ return packageReplacesEmit;
+ }
+
+ public void setPackageReplacesEmit(boolean packageReplacesEmit) {
+ this.packageReplacesEmit = packageReplacesEmit;
+ }
+
/**
* If the intent is that the metadata and byte store be packaged in a
zip
* or similar and emitted via a single stream emitter.
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
index db1458095c..ce7269cf32 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java
@@ -567,12 +567,13 @@ public class PipesServer implements AutoCloseable {
}
PipesWorker w = tLastWorker;
TIMING_LOG.trace("WORKER_TIMING id={} req_deser_us={} ctx_merge_us={}
intermediate_wait_us={}"
- + " handoff_us={} fetch_us={} parse_us={} emit_us={}
worker_wall_us={}"
+ + " handoff_us={} fetch_us={} parse_us={}
prepare_us={} emit_us={} worker_wall_us={}"
+ " intermediate_us={} resp_ser_us={} resp_write_us={}
resp_ack_us={}"
+ " resp_bytes={}",
id, us(tReqDeserNanos), us(tCtxMergeNanos),
us(tIntermediateWaitNanos),
us(tHandoffNanos), us(w == null ? -1 : w.getFetchNanos()),
- us(w == null ? -1 : w.getParseNanos()), us(w == null ? -1 :
w.getEmitNanos()),
+ us(w == null ? -1 : w.getParseNanos()), us(w == null ? -1 :
w.getPrepareNanos()),
+ us(w == null ? -1 : w.getEmitNanos()),
us(w == null ? -1 : w.getWallNanos()),
us(protocolIO.getLastIntermediateNanos()),
us(protocolIO.getLastRespSerNanos()),
us(protocolIO.getLastRespWriteNanos()),
us(protocolIO.getLastRespAckNanos()),
protocolIO.getLastRespBytes());
diff --git
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
index da1f47600c..c7112c2fbd 100644
---
a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
+++
b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesWorker.java
@@ -22,7 +22,13 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;
@@ -60,6 +66,7 @@ import org.apache.tika.pipes.core.extractor.UnpackConfig;
import org.apache.tika.pipes.core.extractor.UnpackExtractor;
import org.apache.tika.pipes.core.extractor.UnpackedByteCount;
import org.apache.tika.pipes.core.extractor.frictionless.DataPackage;
+import org.apache.tika.pipes.core.extractor.frictionless.FrictionlessResource;
import org.apache.tika.utils.ExceptionUtils;
import org.apache.tika.utils.StringUtils;
@@ -67,6 +74,8 @@ class PipesWorker implements Callable<PipesResult> {
private static final Logger LOG =
LoggerFactory.getLogger(PipesWorker.class);
+ static final String METADATA_FILE_NAME = "rmeta.json";
+
private final FetchEmitTuple fetchEmitTuple;
private final ParseContext parseContext;
private final AutoDetectParser autoDetectParser;
@@ -96,6 +105,8 @@ class PipesWorker implements Callable<PipesResult> {
// worker completes. -1 means the stage never ran.
private volatile long fetchNanos = -1;
private volatile long parseNanos = -1;
+ // text inference, metadata filters, user metadata
+ private volatile long prepareNanos = -1;
private volatile long emitNanos = -1;
private volatile long wallNanos = -1;
@@ -107,6 +118,10 @@ class PipesWorker implements Callable<PipesResult> {
return parseNanos;
}
+ long getPrepareNanos() {
+ return prepareNanos;
+ }
+
long getEmitNanos() {
return emitNanos;
}
@@ -143,27 +158,34 @@ class PipesWorker implements Callable<PipesResult> {
return PipesResults.EMPTY_OUTPUT;
}
- // Check if we need to zip and emit embedded files
- UnpackHandler handler = parseContext.get(UnpackHandler.class);
- if (handler instanceof FrictionlessUnpackHandler) {
- frictionlessHandler = (FrictionlessUnpackHandler) handler;
- PipesResult frictionlessResult =
emitFrictionlessOutput(frictionlessHandler, parseData);
- if (frictionlessResult != null) {
- // Frictionless emit failed - return the error
- return frictionlessResult;
- }
- } else if (handler instanceof TempFileUnpackHandler) {
- tempHandler = (TempFileUnpackHandler) handler;
- PipesResult zipResult = zipAndEmitEmbeddedFiles(tempHandler);
- if (zipResult != null) {
- // Zipping/emitting failed - return the error
- return zipResult;
- }
- }
+ // before prepare: a filter may strip tk:embedded-id
+ Map<Integer, Metadata> rowsById =
rowsById(parseData.getMetadataList());
+ // before the unpack package: it carries the same metadata list as
the emit
+ long prepareStart = System.nanoTime();
+ EmitHandler.Prepared prepared =
emitHandler.prepare(fetchEmitTuple, parseData, parseContext);
+ prepareNanos = System.nanoTime() - prepareStart;
+ // the unpack output is part of the emit
long emitStart = System.nanoTime();
try {
- return emitHandler.emitParseData(fetchEmitTuple, parseData,
parseContext);
+ // Check if we need to zip and emit embedded files
+ UnpackHandler handler = parseContext.get(UnpackHandler.class);
+ if (handler instanceof FrictionlessUnpackHandler) {
+ frictionlessHandler = (FrictionlessUnpackHandler) handler;
+ PipesResult frictionlessResult =
emitFrictionlessOutput(frictionlessHandler, parseData);
+ if (frictionlessResult != null) {
+ // Frictionless emit failed - return the error
+ return frictionlessResult;
+ }
+ } else if (handler instanceof TempFileUnpackHandler) {
+ tempHandler = (TempFileUnpackHandler) handler;
+ PipesResult zipResult =
zipAndEmitEmbeddedFiles(tempHandler, parseData, rowsById);
+ if (zipResult != null) {
+ // Zipping/emitting failed - return the error
+ return zipResult;
+ }
+ }
+ return emitHandler.emitParseData(fetchEmitTuple, parseData,
prepared, parseContext);
} finally {
emitNanos = System.nanoTime() - emitStart;
}
@@ -203,7 +225,9 @@ class PipesWorker implements Callable<PipesResult> {
* @param tempHandler the handler containing embedded files in temp
directory
* @return PipesResult if there was an error, null if successful
*/
- private PipesResult zipAndEmitEmbeddedFiles(TempFileUnpackHandler
tempHandler) {
+ private PipesResult zipAndEmitEmbeddedFiles(TempFileUnpackHandler
tempHandler,
+ MetadataListAndEmbeddedBytes
parseData,
+ Map<Integer, Metadata>
rowsById) {
// Skip if no embedded files
if (!tempHandler.hasEmbeddedFiles()) {
LOG.debug("No embedded files to zip");
@@ -234,7 +258,7 @@ class PipesWorker implements Callable<PipesResult> {
// Create zip file in temp directory
Path zipFile =
tempHandler.getTempDirectory().resolve("embedded-files.zip");
try {
- createZipFile(zipFile, tempHandler, unpackConfig);
+ createZipFile(zipFile, tempHandler, unpackConfig,
parseData.getMetadataList(), rowsById);
} catch (IOException e) {
LOG.warn("Failed to create zip file", e);
return new PipesResult(PipesResult.RESULT_STATUS.EMIT_EXCEPTION,
@@ -266,14 +290,17 @@ class PipesWorker implements Callable<PipesResult> {
private PipesResult emitFrictionlessOutput(FrictionlessUnpackHandler
frictionlessHandler,
MetadataListAndEmbeddedBytes
parseData) {
UnpackConfig unpackConfig = frictionlessHandler.getUnpackConfig();
-
- // Skip if no embedded files (unless includeOriginal is set)
- if (!frictionlessHandler.hasEmbeddedFiles() &&
!unpackConfig.isIncludeOriginal()) {
- LOG.debug("No embedded files for Frictionless output");
+ String emitterName = unpackConfig.getEmitter();
+ // Same emitter: no separate <emitKey>.json, the package carries the
metadata list
+ boolean replacesEmit = emitterName != null &&
+ emitterName.equals(fetchEmitTuple.getEmitKey().getEmitterId());
+ // Written even with no embedded files, same shape on every surface,
unless asked not to
+ if (!frictionlessHandler.hasEmbeddedFiles() &&
+ unpackConfig.getEmptyPackage() ==
UnpackConfig.EMPTY_PACKAGE.SKIP) {
+ parseData.setPackageReplacesEmit(replacesEmit);
return null;
}
- String emitterName = unpackConfig.getEmitter();
Emitter emitter;
try {
emitter = emitterManager.getEmitter(emitterName);
@@ -293,20 +320,60 @@ class PipesWorker implements Callable<PipesResult> {
StreamEmitter streamEmitter = (StreamEmitter) emitter;
EmitKey containerEmitKey = fetchEmitTuple.getEmitKey();
- // Get container name from fetch key
- String fetchKey = fetchEmitTuple.getFetchKey().getFetchKey();
- String containerName = fetchKey;
- int lastSlash = Math.max(fetchKey.lastIndexOf('/'),
fetchKey.lastIndexOf('\\'));
- if (lastSlash >= 0 && lastSlash < fetchKey.length() - 1) {
- containerName = fetchKey.substring(lastSlash + 1);
+ // Package name: the host's, else the fetch key's file name
+ String containerName = unpackConfig.getPackageName();
+ if (containerName == null) {
+ String fetchKey = fetchEmitTuple.getFetchKey().getFetchKey();
+ containerName = fetchKey;
+ int lastSlash = Math.max(fetchKey.lastIndexOf('/'),
fetchKey.lastIndexOf('\\'));
+ if (lastSlash >= 0 && lastSlash < fetchKey.length() - 1) {
+ containerName = fetchKey.substring(lastSlash + 1);
+ }
+ }
+
+ DataPackage dataPackage =
frictionlessHandler.buildDataPackage(containerName);
+ byte[] metadataJson = null;
+ if (unpackConfig.writesMetadata() && parseData != null &&
+ parseData.getMetadataList() != null) {
+ try {
+ metadataJson =
writeMetadataListAsJson(parseData.getMetadataList());
+ } catch (IOException e) {
+ LOG.warn("Failed to serialize metadata for Frictionless
output", e);
+ return new
PipesResult(PipesResult.RESULT_STATUS.EMIT_EXCEPTION,
+ "Failed to serialize metadata for Frictionless output:
" +
+ ExceptionUtils.format(e, parseContext));
+ }
+ dataPackage.getResources().add(0,
FrictionlessResource.create(METADATA_FILE_NAME,
+ "application/json", metadataJson.length,
+ FrictionlessResource.formatHash(sha256(metadataJson))));
}
+ PipesResult result;
if (unpackConfig.getOutputMode() == UnpackConfig.OUTPUT_MODE.ZIPPED) {
- return emitFrictionlessZipped(frictionlessHandler, streamEmitter,
containerEmitKey,
- containerName, parseData, unpackConfig);
+ result = emitFrictionlessZipped(frictionlessHandler,
streamEmitter, containerEmitKey,
+ dataPackage, metadataJson);
} else {
- return emitFrictionlessDirectory(frictionlessHandler,
streamEmitter, containerEmitKey,
- containerName, parseData, unpackConfig);
+ result = emitFrictionlessDirectory(frictionlessHandler,
streamEmitter,
+ packageRoot(containerEmitKey, unpackConfig), dataPackage,
metadataJson);
+ }
+ if (result == null) {
+ parseData.setPackageReplacesEmit(replacesEmit);
+ }
+ return result;
+ }
+
+ /** Emit key prefix of a directory-mode package; see {@link
UnpackConfig#getPackageRoot()}. */
+ private static String packageRoot(EmitKey containerEmitKey, UnpackConfig
unpackConfig) {
+ String root = unpackConfig.getPackageRoot() != null ?
+ unpackConfig.getPackageRoot() : containerEmitKey.getEmitKey();
+ return root.isEmpty() ? "" : root + "/";
+ }
+
+ private static byte[] sha256(byte[] bytes) {
+ try {
+ return MessageDigest.getInstance("SHA-256").digest(bytes);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("SHA-256 algorithm not available",
e);
}
}
@@ -316,16 +383,12 @@ class PipesWorker implements Callable<PipesResult> {
private PipesResult emitFrictionlessZipped(FrictionlessUnpackHandler
frictionlessHandler,
StreamEmitter streamEmitter,
EmitKey containerEmitKey,
- String containerName,
- MetadataListAndEmbeddedBytes
parseData,
- UnpackConfig unpackConfig) {
- // Build the data package manifest
- DataPackage dataPackage =
frictionlessHandler.buildDataPackage(containerName);
-
+ DataPackage dataPackage,
+ byte[] metadataJson) {
// Create zip file in temp directory
Path zipFile =
frictionlessHandler.getTempDirectory().resolve("frictionless-package.zip");
try {
- createFrictionlessZipFile(zipFile, frictionlessHandler,
dataPackage, parseData, unpackConfig);
+ createFrictionlessZipFile(zipFile, frictionlessHandler,
dataPackage, metadataJson);
} catch (IOException e) {
LOG.warn("Failed to create Frictionless zip file", e);
return new PipesResult(PipesResult.RESULT_STATUS.EMIT_EXCEPTION,
@@ -352,40 +415,31 @@ class PipesWorker implements Callable<PipesResult> {
*/
private PipesResult emitFrictionlessDirectory(FrictionlessUnpackHandler
frictionlessHandler,
StreamEmitter streamEmitter,
- EmitKey containerEmitKey,
- String containerName,
- MetadataListAndEmbeddedBytes
parseData,
- UnpackConfig unpackConfig) {
- String baseEmitKey = containerEmitKey.getEmitKey();
-
- // Build the data package manifest
- DataPackage dataPackage =
frictionlessHandler.buildDataPackage(containerName);
-
+ String packageRoot,
+ DataPackage dataPackage,
+ byte[] metadataJson) {
try {
// Emit each embedded file under unpacked/.
// When includeOriginal=true the container itself is added as id 0
by
// ParseHandler._preParse, so it appears here as one of the
embedded entries.
for (FrictionlessUnpackHandler.FrictionlessFileInfo fileInfo :
frictionlessHandler.getEmbeddedFiles()) {
- String fileEmitKey = baseEmitKey + "/unpacked/" +
fileInfo.fileName();
+ String fileEmitKey = packageRoot + "unpacked/" +
fileInfo.fileName();
try (InputStream is =
Files.newInputStream(fileInfo.filePath())) {
streamEmitter.emit(fileEmitKey, is, fileInfo.metadata(),
parseContext);
}
}
// Emit datapackage.json
- String dpEmitKey = baseEmitKey + "/datapackage.json";
+ String dpEmitKey = packageRoot + "datapackage.json";
byte[] dpBytes =
dataPackage.toJson().getBytes(java.nio.charset.StandardCharsets.UTF_8);
try (InputStream dpStream = new
java.io.ByteArrayInputStream(dpBytes)) {
streamEmitter.emit(dpEmitKey, dpStream, new Metadata(),
parseContext);
}
- // Emit metadata.json if requested
- if (unpackConfig.writesMetadata() && parseData != null &&
- parseData.getMetadataList() != null) {
- String metadataEmitKey = baseEmitKey + "/metadata.json";
- byte[] metadataBytes =
writeMetadataListAsJson(parseData.getMetadataList());
- try (InputStream metadataStream = new
java.io.ByteArrayInputStream(metadataBytes)) {
- streamEmitter.emit(metadataEmitKey, metadataStream, new
Metadata(), parseContext);
+ if (metadataJson != null) {
+ try (InputStream metadataStream = new
java.io.ByteArrayInputStream(metadataJson)) {
+ streamEmitter.emit(packageRoot + METADATA_FILE_NAME,
metadataStream,
+ new Metadata(), parseContext);
}
}
} catch (IOException e) {
@@ -395,7 +449,7 @@ class PipesWorker implements Callable<PipesResult> {
}
LOG.debug("Successfully emitted Frictionless package with {} resources
(directory mode) to {}",
- dataPackage.resourceCount(), baseEmitKey);
+ dataPackage.resourceCount(), packageRoot);
return null;
}
@@ -403,8 +457,7 @@ class PipesWorker implements Callable<PipesResult> {
* Creates a Frictionless Data Package zip file.
*/
private void createFrictionlessZipFile(Path zipFile,
FrictionlessUnpackHandler frictionlessHandler,
- DataPackage dataPackage,
MetadataListAndEmbeddedBytes parseData,
- UnpackConfig unpackConfig) throws
IOException {
+ DataPackage dataPackage, byte[]
metadataJson) throws IOException {
try (ZipOutputStream zos = new
ZipOutputStream(Files.newOutputStream(zipFile))) {
// Add datapackage.json at root
ZipEntry dpEntry = new ZipEntry("datapackage.json");
@@ -412,12 +465,9 @@ class PipesWorker implements Callable<PipesResult> {
dataPackage.writeTo(zos);
zos.closeEntry();
- // Add metadata.json if requested
- if (unpackConfig.writesMetadata() && parseData != null &&
- parseData.getMetadataList() != null) {
- ZipEntry metadataEntry = new ZipEntry("metadata.json");
- zos.putNextEntry(metadataEntry);
- writeMetadataListAsJson(zos, parseData.getMetadataList());
+ if (metadataJson != null) {
+ zos.putNextEntry(new ZipEntry(METADATA_FILE_NAME));
+ zos.write(metadataJson);
zos.closeEntry();
}
@@ -461,7 +511,11 @@ class PipesWorker implements Callable<PipesResult> {
* Creates a zip file containing all embedded files.
*/
private void createZipFile(Path zipFile, TempFileUnpackHandler tempHandler,
- UnpackConfig unpackConfig) throws IOException {
+ UnpackConfig unpackConfig, List<Metadata>
metadataList,
+ Map<Integer, Metadata> rowsById) throws
IOException {
+ // Sidecars are the list's rows, so they get what rmeta.json gets:
inference, filters
+ Set<Metadata> kept = Collections.newSetFromMap(new
IdentityHashMap<>());
+ kept.addAll(metadataList);
try (ZipOutputStream zos = new
ZipOutputStream(Files.newOutputStream(zipFile))) {
// When includeOriginal=true the container itself is added as id 0
by
// ParseHandler._preParse, so it appears here as one of the
embedded entries.
@@ -472,18 +526,32 @@ class PipesWorker implements Callable<PipesResult> {
Files.copy(fileInfo.filePath(), zos);
zos.closeEntry();
- // Add metadata JSON if requested
- if (unpackConfig.writesMetadata()) {
+ // Add metadata JSON if requested; a row a filter removed gets
no sidecar
+ Metadata row = rowsById.get(fileInfo.id());
+ if (unpackConfig.writesMetadata() && row != null &&
kept.contains(row)) {
String metadataFileName = fileInfo.fileName() +
".metadata.json";
ZipEntry metadataEntry = new ZipEntry(metadataFileName);
zos.putNextEntry(metadataEntry);
- writeMetadataAsJson(zos, fileInfo.metadata());
+ writeMetadataAsJson(zos, row);
zos.closeEntry();
}
}
}
}
+ /** Rows by {@code tk:embedded-id}; the container, which has none, is 0. */
+ private static Map<Integer, Metadata> rowsById(List<Metadata>
metadataList) {
+ Map<Integer, Metadata> rows = new HashMap<>();
+ for (int i = 0; i < metadataList.size(); i++) {
+ Metadata row = metadataList.get(i);
+ Integer id = i == 0 ? Integer.valueOf(0) :
row.getInt(TikaCoreProperties.EMBEDDED_ID);
+ if (id != null) {
+ rows.put(id, row);
+ }
+ }
+ return rows;
+ }
+
/**
* Writes metadata as JSON to the output stream.
* Note: Does not close the output stream.
diff --git
a/tika-pipes/tika-pipes-core/src/main/resources/org/apache/tika/pipes/core/presets/thumbnail.json
b/tika-pipes/tika-pipes-core/src/main/resources/org/apache/tika/pipes/core/presets/thumbnail.json
index acb46b7333..e6e8c38d02 100644
---
a/tika-pipes/tika-pipes-core/src/main/resources/org/apache/tika/pipes/core/presets/thumbnail.json
+++
b/tika-pipes/tika-pipes-core/src/main/resources/org/apache/tika/pipes/core/presets/thumbnail.json
@@ -1,6 +1,8 @@
{
// at most one raster thumbnail per parse, nothing else; the rule is in
ThumbnailUnpackSelector
"thumbnail-unpack-selector": {},
+ // no thumbnail is no package: /unpack answers 204
+ "unpack-config": {"emptyPackage": "SKIP"},
// every rendering parser: the top-level document's first page only, fitted
into 256 x 256 in
// colour, and text comes from the document itself, never OCR
"pages": {
diff --git
a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResourceTest.java
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResourceTest.java
new file mode 100644
index 0000000000..8e9c38d998
--- /dev/null
+++
b/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/extractor/frictionless/FrictionlessResourceTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.core.extractor.frictionless;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+public class FrictionlessResourceTest {
+
+ /** Frictionless resource names are lowercase [a-z0-9._-]; extensions in
path are not. */
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', value = {
+ "rmeta.json|rmeta.json",
+ "unpacked/1.png|unpacked-1.png",
+ "unpacked/4.PDF|unpacked-4.pdf",
+ "unpacked/5.tar.gz|unpacked-5.tar.gz",
+ "unpacked/6.c++|unpacked-6.c--",
+ "unpacked/7.p df|unpacked-7.p-df",
+ "unpacked/8.\u00fcn|unpacked-8.-n"
+ })
+ public void testNameFromPath(String path, String name) {
+ assertEquals(name, FrictionlessResource.nameFromPath(path));
+ }
+}
diff --git
a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/FrictionlessUnpackTest.java
b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/FrictionlessUnpackTest.java
index 19e6d7cc0a..24b0ba1a30 100644
---
a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/FrictionlessUnpackTest.java
+++
b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/FrictionlessUnpackTest.java
@@ -36,6 +36,7 @@ import java.util.zip.ZipFile;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -63,7 +64,7 @@ import org.apache.tika.pipes.core.extractor.UnpackConfig;
* <pre>
* output/
* ├── datapackage.json # Frictionless manifest with file list, hashes,
mimetypes
- * ├── metadata.json # Full RMETA-style metadata (optional)
+ * ├── rmeta.json # Full RMETA-style metadata (optional)
* └── unpacked/
* ├── 00000001.xml
* ├── 00000002.xml
@@ -149,7 +150,7 @@ public class FrictionlessUnpackTest {
assertFalse(names.contains(TEST_DOC_WITH_EMBEDDED + "-embedded.zip"),
names.toString());
Set<String> entries = zipEntries(onlyFrictionlessZip(outputDir));
assertTrue(entries.contains("datapackage.json"), entries.toString());
- assertTrue(entries.contains("metadata.json"), "a package carries
metadata.json by default: " + entries);
+ assertTrue(entries.contains("rmeta.json"), "a package carries
rmeta.json by default: " + entries);
}
@Test
@@ -203,8 +204,8 @@ public class FrictionlessUnpackTest {
"Each resource must have 'hash' (SHA256): " +
resource);
String path = resource.get("path").asText();
- assertTrue(path.startsWith("unpacked/"),
- "Resource path should start with 'unpacked/': " +
path);
+ assertTrue(path.startsWith("unpacked/") ||
path.equals("rmeta.json"),
+ "Resource path should be rmeta.json or under
'unpacked/': " + path);
String hash = resource.get("hash").asText();
assertTrue(hash.startsWith("sha256:"),
@@ -256,7 +257,7 @@ public class FrictionlessUnpackTest {
@Test
public void testIncludeMetadataJson() throws Exception {
- // Test that includeFullMetadata=true creates metadata.json with
RMETA-style output
+ // Test that includeFullMetadata=true creates rmeta.json with
RMETA-style output
ParseContext parseContext =
frictionless(UnpackConfig.OUTPUT_MODE.ZIPPED, true);
parseContext.get(UnpackConfig.class).setIncludeFullMetadata(true);
PipesResult pipesResult = process("fullMetadata",
TEST_DOC_WITH_EMBEDDED, parseContext);
@@ -266,20 +267,20 @@ public class FrictionlessUnpackTest {
Path zipFile = onlyFrictionlessZip(outputDir("fullMetadata"));
Set<String> zipEntries = zipEntries(zipFile);
- assertTrue(zipEntries.contains("metadata.json"),
- "Zip should contain metadata.json when
includeFullMetadata=true. Found: " + zipEntries);
+ assertTrue(zipEntries.contains("rmeta.json"),
+ "Zip should contain rmeta.json when includeFullMetadata=true.
Found: " + zipEntries);
JsonNode metadataJson;
try (ZipFile zip = new ZipFile(zipFile.toFile());
- InputStream is =
zip.getInputStream(zip.getEntry("metadata.json"))) {
+ InputStream is =
zip.getInputStream(zip.getEntry("rmeta.json"))) {
metadataJson = OBJECT_MAPPER.readTree(is);
}
- assertNotNull(metadataJson, "metadata.json should be parseable JSON");
+ assertNotNull(metadataJson, "rmeta.json should be parseable JSON");
assertTrue(metadataJson.isArray(),
- "metadata.json should be an array of metadata objects");
+ "rmeta.json should be an array of metadata objects");
// container + 4 embedded
assertTrue(metadataJson.size() >= 5,
- "metadata.json should have at least 5 entries (container + 4
embedded). Found: " +
+ "rmeta.json should have at least 5 entries (container + 4
embedded). Found: " +
metadataJson.size());
JsonNode containerMeta = metadataJson.get(0);
@@ -299,7 +300,7 @@ public class FrictionlessUnpackTest {
assertTrue(pipesResult.isSuccess(),
"Processing with UnpackSelector should succeed");
- assertNothingPackaged(outputDir("selector"), TEST_DOC_WITH_EMBEDDED);
+ assertMetadataOnlyPackage(outputDir("selector"),
TEST_DOC_WITH_EMBEDDED);
}
@Test
@@ -334,7 +335,7 @@ public class FrictionlessUnpackTest {
assertTrue(pipesResult.isSuccess(),
"Frictionless should succeed with no embedded files");
- assertNothingPackaged(outputDir("noEmbedded"), SIMPLE_DOC);
+ assertMetadataOnlyPackage(outputDir("noEmbedded"), SIMPLE_DOC);
}
@Test
@@ -368,8 +369,8 @@ public class FrictionlessUnpackTest {
boolean manifestListsContainer = false;
for (JsonNode resource : dataPackage.get("resources")) {
String path = resource.get("path").asText();
- assertTrue(path.startsWith("unpacked/"),
- "Manifest resources should only list unpacked/ paths;
got " + path);
+ assertTrue(path.startsWith("unpacked/") ||
path.equals("rmeta.json"),
+ "Manifest resources should only list rmeta.json and
unpacked/ paths; got " + path);
if (path.equals("unpacked/0") ||
path.startsWith("unpacked/0.")) {
manifestListsContainer = true;
}
@@ -379,6 +380,143 @@ public class FrictionlessUnpackTest {
}
}
+ /**
+ * TIKA-4971: rmeta.json gets what the emitted list gets -- the request's
metadata filters,
+ * then user metadata, which the filters must not strip.
+ */
+ @Test
+ public void testRmetaJsonIsFilteredThenGetsUserMetadata() throws Exception
{
+ ParseContext parseContext =
frictionless(UnpackConfig.OUTPUT_MODE.ZIPPED, false);
+ parseContext.setJsonConfig("metadata-filters", """
+ [{"include-field-metadata-filter": {"include": ["tk:content",
"Content-Type"]}}]
+ """);
+ Metadata userMetadata = new Metadata();
+ userMetadata.set("my:key", "my-value");
+ PipesResult pipesResult = pipesClient.process(
+ new FetchEmitTuple(TEST_DOC_WITH_EMBEDDED,
+ new FetchKey(FETCHER_NAME, TEST_DOC_WITH_EMBEDDED),
+ new EmitKey(EMITTER_NAME, "filtered/" +
TEST_DOC_WITH_EMBEDDED),
+ userMetadata, parseContext,
+ FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT));
+ assertTrue(pipesResult.isSuccess(), "Status: " + pipesResult.status());
+
+ JsonNode rows;
+ try (ZipFile zip = new
ZipFile(onlyFrictionlessZip(outputDir("filtered")).toFile());
+ InputStream is =
zip.getInputStream(zip.getEntry("rmeta.json"))) {
+ rows = OBJECT_MAPPER.readTree(is);
+ }
+ assertTrue(rows.size() > 1, rows.toString());
+ Set<String> allowed = Set.of("tk:content", "Content-Type", "my:key");
+ for (JsonNode row : rows) {
+ row.fieldNames().forEachRemaining(k ->
+ assertTrue(allowed.contains(k), "filter should have
removed " + k + ": " + row));
+ }
+ assertEquals("my-value", rows.get(0).get("my:key").asText(),
rows.get(0).toString());
+ }
+
+ /** TIKA-4971: includeMetadata=false is the bytes and the manifest, and no
metadata anywhere. */
+ @Test
+ public void testNoMetadataMeansNoMetadataAnywhere() throws Exception {
+ ParseContext parseContext =
frictionless(UnpackConfig.OUTPUT_MODE.ZIPPED, false);
+ parseContext.get(UnpackConfig.class).setIncludeMetadata(false);
+ PipesResult pipesResult = process("noMetadata",
TEST_DOC_WITH_EMBEDDED, parseContext);
+ assertTrue(pipesResult.isSuccess(), "Status: " + pipesResult.status());
+
+ Set<String> entries =
zipEntries(onlyFrictionlessZip(outputDir("noMetadata")));
+ assertTrue(entries.contains("datapackage.json"), entries.toString());
+ assertFalse(entries.contains("rmeta.json"), entries.toString());
+
assertFalse(Files.exists(outputDir("noMetadata").resolve(TEST_DOC_WITH_EMBEDDED
+ ".json")),
+ "the package replaces the metadata emit");
+ }
+
+ /** TIKA-4971: emptyPackage=SKIP with nothing to unpack writes nothing at
all. */
+ @Test
+ public void testSkippedEmptyPackageWritesNothing() throws Exception {
+ ParseContext parseContext =
frictionless(UnpackConfig.OUTPUT_MODE.ZIPPED, false);
+
parseContext.get(UnpackConfig.class).setEmptyPackage(UnpackConfig.EMPTY_PACKAGE.SKIP);
+ PipesResult pipesResult = process("skipEmpty", SIMPLE_DOC,
parseContext);
+ assertTrue(pipesResult.isSuccess(), "Status: " + pipesResult.status());
+
+ Path outputDir = outputDir("skipEmpty");
+ if (Files.isDirectory(outputDir)) {
+ try (Stream<Path> files = Files.list(outputDir)) {
+ assertEquals(List.of(), files.toList());
+ }
+ }
+ }
+
+ /** TIKA-4971: a package sent to another emitter leaves the container's
metadata emit alone. */
+ @Test
+ public void testPackageToOtherEmitterKeepsMetadataEmit() throws Exception {
+ Path configPath = tmp.resolve("pipes-config-two-emitters.json");
+ Path otherOutput = tmp.resolve("output-other");
+ ObjectNode config = (ObjectNode)
OBJECT_MAPPER.readTree(tmp.resolve("pipes-config.json").toFile());
+ ObjectNode other = ((ObjectNode)
config.get("emitters").get(EMITTER_NAME)).deepCopy();
+ ((ObjectNode) other.get("file-system-emitter")).put("basePath",
otherOutput.toString());
+ ((ObjectNode) config.get("emitters")).set("fse-other", other);
+ OBJECT_MAPPER.writeValue(configPath.toFile(), config);
+
+ ParseContext parseContext =
frictionless(UnpackConfig.OUTPUT_MODE.ZIPPED, false);
+ parseContext.get(UnpackConfig.class).setEmitter("fse-other");
+ try (PipesClient client = new PipesClient(
+ PipesConfig.load(TikaJsonConfig.load(configPath)),
configPath)) {
+ PipesResult pipesResult = client.process(
+ new FetchEmitTuple(TEST_DOC_WITH_EMBEDDED,
+ new FetchKey(FETCHER_NAME, TEST_DOC_WITH_EMBEDDED),
+ new EmitKey(EMITTER_NAME, "otherEmitter/" +
TEST_DOC_WITH_EMBEDDED),
+ new Metadata(), parseContext,
+ FetchEmitTuple.ON_PARSE_EXCEPTION.EMIT));
+ assertTrue(pipesResult.isSuccess(), "Status: " +
pipesResult.status());
+ }
+
assertTrue(zipEntries(onlyFrictionlessZip(otherOutput.resolve("otherEmitter")))
+ .contains("rmeta.json"));
+ assertTrue(Files.isRegularFile(
+ outputDir("otherEmitter").resolve(TEST_DOC_WITH_EMBEDDED +
".json")));
+ }
+
+ /** TIKA-4971: REGULAR sidecars are the list's rows, so they get the
metadata filters. */
+ @Test
+ public void testRegularSidecarsAreFiltered() throws Exception {
+ ParseContext parseContext = new ParseContext();
+ parseContext.set(ParseMode.class, ParseMode.UNPACK);
+ UnpackConfig unpackConfig = new UnpackConfig();
+ unpackConfig.setEmitter(EMITTER_NAME);
+ unpackConfig.setOutputFormat(UnpackConfig.OUTPUT_FORMAT.REGULAR);
+ unpackConfig.setZipEmbeddedFiles(true);
+ unpackConfig.setIncludeMetadata(true);
+ parseContext.set(UnpackConfig.class, unpackConfig);
+ parseContext.setJsonConfig("metadata-filters", """
+ [{"include-field-metadata-filter": {"include": ["tk:content",
"Content-Type"]}}]
+ """);
+ PipesResult pipesResult = process("regularFiltered",
TEST_DOC_WITH_EMBEDDED, parseContext);
+ assertTrue(pipesResult.isSuccess(), "Status: " + pipesResult.status());
+
+ Path zipFile =
outputDir("regularFiltered").resolve(TEST_DOC_WITH_EMBEDDED + "-embedded.zip");
+ Set<String> allowed = Set.of("tk:content", "Content-Type");
+ int files = 0;
+ int sidecars = 0;
+ try (ZipFile zip = new ZipFile(zipFile.toFile())) {
+ Enumeration<? extends ZipEntry> entries = zip.entries();
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ if (!entry.getName().endsWith(".metadata.json")) {
+ files++;
+ continue;
+ }
+ sidecars++;
+ JsonNode sidecar;
+ try (InputStream is = zip.getInputStream(entry)) {
+ sidecar = OBJECT_MAPPER.readTree(is);
+ }
+ assertTrue(sidecar.has("Content-Type"), entry.getName() + ": "
+ sidecar);
+ sidecar.fieldNames().forEachRemaining(k ->
assertTrue(allowed.contains(k),
+ "filter should have removed " + k + " from " +
entry.getName()));
+ }
+ }
+ assertTrue(files > 0, "no files in " + zipFile);
+ assertEquals(files, sidecars, "one sidecar per embedded file in " +
zipFile);
+ }
+
private static Path onlyFrictionlessZip(Path outputDir) throws IOException
{
List<Path> zipFiles;
try (Stream<Path> files = Files.list(outputDir)) {
@@ -410,13 +548,18 @@ public class FrictionlessUnpackTest {
return sb.toString();
}
- /** Nothing to package writes no zip; the metadata JSON shows the parse
still ran. */
- private static void assertNothingPackaged(Path outputDir, String emitKey)
throws IOException {
- try (Stream<Path> files = Files.list(outputDir)) {
- List<String> zips = files.map(p -> p.getFileName().toString())
- .filter(n -> n.endsWith("-frictionless.zip")).toList();
- assertTrue(zips.isEmpty(), "unexpected zips: " + zips);
+ /** TIKA-4971: nothing to unpack is still a package: the manifest and
rmeta.json, no sidecar. */
+ private static void assertMetadataOnlyPackage(Path outputDir, String
emitKey) throws IOException {
+ Path zipFile = outputDir.resolve(emitKey + "-frictionless.zip");
+ assertEquals(Set.of("datapackage.json", "rmeta.json"),
zipEntries(zipFile));
+ JsonNode resources;
+ try (ZipFile zip = new ZipFile(zipFile.toFile());
+ InputStream is =
zip.getInputStream(zip.getEntry("datapackage.json"))) {
+ resources = OBJECT_MAPPER.readTree(is).get("resources");
}
- assertTrue(Files.isRegularFile(outputDir.resolve(emitKey + ".json")),
"no metadata JSON for " + emitKey);
+ assertEquals(1, resources.size(), resources.toString());
+ assertEquals("rmeta.json", resources.get(0).get("path").asText());
+ assertFalse(Files.exists(outputDir.resolve(emitKey + ".json")),
+ "rmeta.json already carries the metadata list");
}
}
diff --git
a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/UnpackModeTest.java
b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/UnpackModeTest.java
index 755d10fe44..9c2d6d8d2e 100644
---
a/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/UnpackModeTest.java
+++
b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/UnpackModeTest.java
@@ -17,6 +17,7 @@
package org.apache.tika.pipes.core;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -352,7 +353,7 @@ public class UnpackModeTest {
Path outputDir = outputDir("zipNoEmbedded");
assertTrue(zips(outputDir).isEmpty(), "no embedded files, no zip: " +
zips(outputDir));
- assertEquals(1, readMetadataList(outputDir, simpleDoc).size());
+ assertEquals(1, readSidecar(outputDir, simpleDoc).size());
}
@Test
@@ -462,13 +463,25 @@ public class UnpackModeTest {
return names;
}
- private static List<Metadata> readMetadataList(Path outputDir, String
emitKey) throws IOException {
+ /** REGULAR: the metadata list is the {@code <emitKey>.json} beside the
bytes. */
+ private static List<Metadata> readSidecar(Path outputDir, String emitKey)
throws IOException {
try (Reader reader = Files.newBufferedReader(outputDir.resolve(emitKey
+ ".json"),
StandardCharsets.UTF_8)) {
return JsonMetadataList.fromJson(reader);
}
}
+ /** The package's rmeta.json: same emitter, so no separate {@code
<emitKey>.json} (TIKA-4971). */
+ private static List<Metadata> readMetadataList(Path outputDir, String
emitKey) throws IOException {
+ assertFalse(Files.exists(outputDir.resolve(emitKey + ".json")),
+ "the package already carries the metadata list");
+ try (ZipFile zip = new ZipFile(outputDir.resolve(emitKey +
"-frictionless.zip").toFile());
+ Reader reader = new java.io.InputStreamReader(
+ zip.getInputStream(zip.getEntry("rmeta.json")),
StandardCharsets.UTF_8)) {
+ return JsonMetadataList.fromJson(reader);
+ }
+ }
+
/** The default package is a ZIPPED Frictionless one: entry name under
unpacked/ -> size. */
private static Map<String, Long> embeddedFiles(Path outputDir, String
emitKey) throws IOException {
Path zipFile = outputDir.resolve(emitKey + "-frictionless.zip");
diff --git
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
index be592413bf..e24c5ba4cd 100644
---
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
+++
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/PipesParsingHelper.java
@@ -666,6 +666,9 @@ public class PipesParsingHelper {
// Set emitter to our file-system emitter
unpackConfig.setEmitter(UNPACK_EMITTER_ID);
+ // The fetch key may be the spool file; name the package for the
caller's file
+ unpackConfig.setPackageName(callerSuppliedName == null ? "" :
callerSuppliedName);
+
// /all = the original document plus metadata, in either output
format.
if (saveAll) {
unpackConfig.setIncludeOriginal(true);
diff --git
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
index 403f7977d1..0dbeb0bee8 100644
---
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
+++
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/TikaResource.java
@@ -55,6 +55,7 @@ import org.apache.tika.config.JsonConfig;
import org.apache.tika.config.OutputLimits;
import org.apache.tika.config.loader.PresetRegistry;
import org.apache.tika.config.loader.TikaLoader;
+import org.apache.tika.config.loader.TikaObjectMapperFactory;
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
@@ -215,6 +216,23 @@ public class TikaResource {
return loadConfigDefaults().get(UnpackConfig.class);
}
+ /**
+ * The config's unpack-config with the preset's unpack-config fields over
it, field by
+ * field: the worker's preset merge replaces whole components, and the
request's
+ * UnpackConfig (which the server always sends) would replace the
preset's. Null when
+ * neither has one.
+ */
+ public UnpackConfig newPresetUnpackConfig(String presetName) throws
IOException {
+ UnpackConfig base = newConfigUnpackConfig();
+ String presetJson = presetRegistry.parseContextJson(presetName);
+ ObjectMapper mapper = TikaObjectMapperFactory.getMapper();
+ JsonNode presetUnpack = presetJson == null ? null :
mapper.readTree(presetJson).get("unpack-config");
+ if (presetUnpack == null) {
+ return base;
+ }
+ return mapper.readerForUpdating(base != null ? base : new
UnpackConfig()).readValue(presetUnpack);
+ }
+
public TikaLoader getTikaLoader() {
return tikaLoader;
diff --git
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
index cdf96de978..990456ca85 100644
---
a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
+++
b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/resource/UnpackerResource.java
@@ -18,6 +18,7 @@ package org.apache.tika.server.core.resource;
import static org.apache.tika.server.core.resource.TikaResource.fillMetadata;
+import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -96,7 +97,7 @@ import org.apache.tika.pipes.core.extractor.UnpackConfig;
* <pre>
* output.zip
* ├── datapackage.json # Manifest with file list, SHA256 hashes,
mimetypes
- * ├── metadata.json # Full RMETA metadata (unless
includeMetadata=false)
+ * ├── rmeta.json # Full RMETA metadata (unless
includeMetadata=false)
* └── unpacked/
* ├── 00000001.pdf
* ├── 00000002.png
@@ -291,6 +292,7 @@ public class UnpackerResource {
@jakarta.ws.rs.PathParam("presetName")
String presetName)
throws Exception {
ParseContext pc = tikaResource.createPresetContext(presetName);
+ seedPresetUnpackConfig(pc, presetName);
rejectDirectoryMode(pc);
Metadata metadata = tikaResource.newRequestMetadata();
try (TikaInputStream tis = TikaInputStream.get(is)) {
@@ -310,6 +312,7 @@ public class UnpackerResource {
@jakarta.ws.rs.PathParam("presetName")
String presetName)
throws Exception {
ParseContext pc = tikaResource.createPresetContext(presetName);
+ seedPresetUnpackConfig(pc, presetName);
rejectDirectoryMode(pc);
Metadata metadata = tikaResource.newRequestMetadata();
try (TikaInputStream tis = TikaInputStream.get(is)) {
@@ -319,6 +322,13 @@ public class UnpackerResource {
}
}
+ private void seedPresetUnpackConfig(ParseContext pc, String presetName)
throws IOException {
+ UnpackConfig unpackConfig =
tikaResource.newPresetUnpackConfig(presetName);
+ if (unpackConfig != null) {
+ pc.set(UnpackConfig.class, unpackConfig);
+ }
+ }
+
/** A request asking for DIRECTORY contradicts the one-zip transport: 400,
not a silent pin. */
static void rejectDirectoryMode(ParseContext pc) {
UnpackConfig requested = pc.get(UnpackConfig.class);
diff --git
a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java
b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java
index 80081c850b..06737f3adb 100644
---
a/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java
+++
b/tika-server/tika-server-core/src/test/java/org/apache/tika/server/core/StackTraceTest.java
@@ -183,13 +183,9 @@ public class StackTraceTest extends CXFTestBase {
Response response = WebClient
.create(endPoint + actualPath)
.put(ClassLoader.getSystemResourceAsStream("test-documents/testDigilite.fdf"));
- if (path.equals("/unpack")) {
- //"NO CONTENT"
- assertEquals(204, response.getStatus(), "bad type: " +
actualPath);
- } else {
- assertEquals(200, response.getStatus(), "bad type: " +
actualPath);
- assertNotNull(response, "null response: " + actualPath);
- }
+ // /unpack too: nothing to unpack is still a package (TIKA-4971)
+ assertEquals(200, response.getStatus(), "bad type: " + actualPath);
+ assertNotNull(response, "null response: " + actualPath);
}
}
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/ThumbnailPresetTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/ThumbnailPresetTest.java
index 4ac707aa42..fdc72a5f3e 100644
---
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/ThumbnailPresetTest.java
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/ThumbnailPresetTest.java
@@ -166,7 +166,7 @@ public class ThumbnailPresetTest extends CXFTestBase {
/**
* @return the embedded documents of the Frictionless package, without its
- * datapackage.json and metadata.json
+ * datapackage.json and rmeta.json
*/
private Map<String, byte[]> unpack(String path, String file) throws
IOException {
Response response = put(path, fixture(file));
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java
index f544e50c54..8a28798698 100644
---
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java
@@ -334,7 +334,7 @@ public class TikaPipesTest extends CXFTestBase {
try (ZipFile zip = new ZipFile(
outputBytesDir.resolve(TEST_RECURSIVE_DOC +
"-frictionless.zip").toFile())) {
assertNotNull(zip.getEntry("datapackage.json"));
- assertNotNull(zip.getEntry("metadata.json"));
+ assertNotNull(zip.getEntry("rmeta.json"));
for (Enumeration<? extends ZipEntry> e = zip.entries();
e.hasMoreElements(); ) {
ZipEntry entry = e.nextElement();
if (entry.getName().startsWith("unpacked/")) {
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java
index 7c671b29e2..2d69ba48c3 100644
---
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackFrictionlessTest.java
@@ -116,15 +116,15 @@ public class UnpackFrictionlessTest extends CXFTestBase {
assertTrue(entries.keySet().stream().anyMatch(k ->
k.startsWith("unpacked/")),
entries.keySet().toString());
assertTrue(entries.keySet().stream().allMatch(k ->
k.startsWith("unpacked/")
- || k.equals("datapackage.json") ||
k.equals("metadata.json")),
+ || k.equals("datapackage.json") ||
k.equals("rmeta.json")),
"nothing but the package at the root: " + entries.keySet());
}
- /** A Frictionless package carries metadata.json by default; /all adds
only the original. */
+ /** A Frictionless package carries rmeta.json by default; /all adds only
the original. */
@Test
public void testPlainUnpackHasMetadataJsonByDefault() throws Exception {
Map<String, byte[]> entries = unpack("/unpack");
- assertTrue(entries.containsKey("metadata.json"),
entries.keySet().toString());
+ assertTrue(entries.containsKey("rmeta.json"),
entries.keySet().toString());
assertFalse(entries.keySet().stream().anyMatch(k ->
k.startsWith("unpacked/0.")),
"plain /unpack must not pack the container's bytes: " +
entries.keySet());
}
@@ -147,7 +147,7 @@ public class UnpackFrictionlessTest extends CXFTestBase {
@Test
public void testUnpackAllCarriesMetadataAndTheOriginal() throws Exception {
Map<String, byte[]> entries = unpack("/unpack/all");
- assertTrue(entries.containsKey("metadata.json"),
entries.keySet().toString());
+ assertTrue(entries.containsKey("rmeta.json"),
entries.keySet().toString());
assertTrue(entries.keySet().stream().anyMatch(k ->
k.startsWith("unpacked/0.")),
"includeOriginal should add the container: " +
entries.keySet());
}
@@ -156,7 +156,7 @@ public class UnpackFrictionlessTest extends CXFTestBase {
@Test
public void testMetadataJsonCarriesTheContainerRow() throws Exception {
Map<String, byte[]> entries = unpack("/unpack");
- JsonNode rows = MAPPER.readTree(entries.get("metadata.json"));
+ JsonNode rows = MAPPER.readTree(entries.get("rmeta.json"));
assertTrue(rows.isArray() && rows.size() > 1, "expected an RMETA list,
got: " + rows);
JsonNode container = rows.get(0);
@@ -172,7 +172,7 @@ public class UnpackFrictionlessTest extends CXFTestBase {
@Test
public void testRowsCarryEmbeddedPathData() throws Exception {
Map<String, byte[]> entries = unpack("/unpack/all");
- JsonNode rows = MAPPER.readTree(entries.get("metadata.json"));
+ JsonNode rows = MAPPER.readTree(entries.get("rmeta.json"));
JsonNode embedded = rows.get(1);
for (String key : new String[]{"tk:embedded-id", "tk:embedded-id-path",
"tk:embedded-resource-path", "tk:resource-name"}) {
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackRegularConfigTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackRegularConfigTest.java
index 07bf63c0aa..290182a8f5 100644
---
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackRegularConfigTest.java
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackRegularConfigTest.java
@@ -113,7 +113,7 @@ public class UnpackRegularConfigTest extends CXFTestBase {
public void testServerConfigRestoresTheFlatLayout() throws Exception {
Map<String, byte[]> entries = unpack("/unpack");
assertFalse(entries.containsKey("datapackage.json"),
entries.keySet().toString());
- assertFalse(entries.containsKey("metadata.json"),
entries.keySet().toString());
+ assertFalse(entries.containsKey("rmeta.json"),
entries.keySet().toString());
assertTrue(entries.keySet().stream().noneMatch(k -> k.contains("/")),
"REGULAR puts every file at the zip root: " +
entries.keySet());
assertFalse(entries.isEmpty());
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java
index c1169f14ee..34be6f5275 100644
---
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceHandlerTest.java
@@ -53,9 +53,9 @@ import org.apache.tika.server.core.writer.ZipWriter;
/**
* /unpack has no format segment, so the config part is the only place a
handler is named --
- * and it reaches the RMETA list written to metadata.json.
+ * and it reaches the RMETA list written to rmeta.json.
* Per-file sidecars exist only in the REGULAR layout, so the server config
pins it; the
- * metadata.json probes ask for FRICTIONLESS explicitly.
+ * rmeta.json probes ask for FRICTIONLESS explicitly.
*/
public class UnpackerResourceHandlerTest extends CXFTestBase {
@@ -131,7 +131,7 @@ public class UnpackerResourceHandlerTest extends
CXFTestBase {
String json = metadataJson("{" + FRICTIONLESS
+ ", \"basic-content-handler-factory\": {\"type\": \"XML\"}}");
assertTrue(json.contains("<html xmlns="),
- "config part should select the handler for metadata.json");
+ "config part should select the handler for rmeta.json");
}
/** /unpack/all/<handler> used to 200 and silently ignore the segment. */
@@ -155,11 +155,11 @@ public class UnpackerResourceHandlerTest extends
CXFTestBase {
public void testAllConfigSpellingIsNotAHandlerNamedConfig() throws
Exception {
Response response = post("/unpack/all/config", "{" + FRICTIONLESS +
"}");
assertEquals(200, response.getStatus());
- assertNotNull(readZipArchiveBytes((InputStream)
response.getEntity()).get("metadata.json"));
+ assertNotNull(readZipArchiveBytes((InputStream)
response.getEntity()).get("rmeta.json"));
response = post("/unpack/all/config/ignore", "{" + FRICTIONLESS + "}");
assertEquals(200, response.getStatus());
String json = new String(readZipArchiveBytes((InputStream)
response.getEntity())
- .get("metadata.json"), StandardCharsets.UTF_8);
+ .get("rmeta.json"), StandardCharsets.UTF_8);
assertFalse(json.contains("\"tk:content\""), "ignore after /config
must suppress tk:content");
}
@@ -256,8 +256,8 @@ public class UnpackerResourceHandlerTest extends
CXFTestBase {
Response response = post("/unpack", configJson);
assertEquals(200, response.getStatus());
Map<String, byte[]> entries = readZipArchiveBytes((InputStream)
response.getEntity());
- byte[] md = entries.get("metadata.json");
- assertNotNull(md, "includeFullMetadata should write metadata.json");
+ byte[] md = entries.get("rmeta.json");
+ assertNotNull(md, "includeFullMetadata should write rmeta.json");
return new String(md, StandardCharsets.UTF_8);
}
diff --git
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java
index 26ffa7680f..0c5e37a3ec 100644
---
a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java
+++
b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/UnpackerResourceTest.java
@@ -172,8 +172,8 @@ public class UnpackerResourceTest extends CXFTestBase {
.toList();
assertEquals(2, wavFiles.size(), "Should have 2 WAV files");
- assertTrue(data.containsKey("metadata.json"),
- "/all carries the parse as metadata.json. Entries: " +
data.keySet());
+ assertTrue(data.containsKey("rmeta.json"),
+ "/all carries the parse as rmeta.json. Entries: " +
data.keySet());
}
@Test
@@ -221,10 +221,41 @@ public class UnpackerResourceTest extends CXFTestBase {
assertTrue(data.containsValue(DOCX_IMAGE2_MD5), "Should contain second
DOCX image");
}
+ /** TIKA-4971: a request can ask for no package when there is nothing to
unpack -- a 204. */
@Test
- @org.junit.jupiter.api.Disabled("Behavior changed in Tika 4.0 pipes-based
implementation")
- public void test204() throws Exception {
- // This test verified 204 behavior which changed in 4.0
+ public void testEmptyPackageSkipIs204() throws Exception {
+ String configJson = """
+ {"parse-context": {"unpack-config": {"emptyPackage": "SKIP"}}}
+ """;
+ ContentDisposition fileCd = new ContentDisposition("form-data;
name=\"file\"; filename=\"foo.csv\"");
+ Attachment fileAtt = new Attachment("file",
+
ClassLoader.getSystemResourceAsStream("test-documents/foo.csv"), fileCd);
+ Attachment configAtt = new Attachment("config", "application/json",
+ new
ByteArrayInputStream(configJson.getBytes(StandardCharsets.UTF_8)));
+ Response response = WebClient
+ .create(endPoint + UNPACKER_PATH)
+ .type("multipart/form-data")
+ .accept("application/zip")
+ .post(new MultipartBody(Arrays.asList(fileAtt, configAtt)));
+ assertEquals(204, response.getStatus());
+ }
+
+ /** TIKA-4971: no attachments is still a package -- the manifest and
rmeta.json, not a 204. */
+ @Test
+ public void testNoAttachmentsIsAMetadataOnlyPackage() throws Exception {
+ Response response = WebClient
+ .create(endPoint + UNPACKER_PATH)
+ .header("Content-Disposition", "attachment;
filename=\"foo.csv\"")
+ .accept("application/zip")
+
.put(ClassLoader.getSystemResourceAsStream("test-documents/foo.csv"));
+
+ assertEquals(200, response.getStatus());
+ Map<String, byte[]> data = readZipArchiveBytes((InputStream)
response.getEntity());
+ assertEquals(Set.of("datapackage.json", "rmeta.json"), data.keySet());
+ JsonNode dataPackage = MAPPER.readTree(data.get("datapackage.json"));
+ assertEquals("foo.csv", dataPackage.get("name").asText());
+ assertEquals(1, dataPackage.get("resources").size(),
dataPackage.toString());
+ assertEquals("rmeta.json",
dataPackage.get("resources").get(0).get("path").asText());
}
@Test
@@ -276,8 +307,8 @@ public class UnpackerResourceTest extends CXFTestBase {
Map<String, byte[]> data = readZipArchiveBytes((InputStream)
response.getEntity());
- byte[] metadata = data.get("metadata.json");
- assertNotNull(metadata, "Should have metadata.json. Entries: " +
data.keySet());
+ byte[] metadata = data.get("rmeta.json");
+ assertNotNull(metadata, "Should have rmeta.json. Entries: " +
data.keySet());
String metadataJson = new String(metadata, StandardCharsets.UTF_8);
assertTrue(metadataJson.contains("Content-Type"), "Metadata JSON
should contain Content-Type");
}
@@ -458,9 +489,8 @@ public class UnpackerResourceTest extends CXFTestBase {
/**
* The datapackage.json "resources" array is the manifest for the
Frictionless
* package. Verifies it lists exactly the data files present in the zip,
with
- * no missing or extraneous entries. The package envelope (datapackage.json
- * and the optional metadata.json) is not itself a "resource" and is
excluded
- * from the comparison.
+ * no missing or extraneous entries. Only datapackage.json, the manifest
itself,
+ * is not a resource; rmeta.json is (TIKA-4971).
*/
@Test
public void testFrictionlessDataPackageMatchesArchiveContents() throws
Exception {
@@ -507,7 +537,6 @@ public class UnpackerResourceTest extends CXFTestBase {
Set<String> archiveDataFiles = new HashSet<>(data.keySet());
archiveDataFiles.remove("datapackage.json");
- archiveDataFiles.remove("metadata.json");
assertEquals(archiveDataFiles, manifestPaths,
"datapackage.json 'resources' must list exactly the data files
in the zip. " +
@@ -516,9 +545,9 @@ public class UnpackerResourceTest extends CXFTestBase {
}
/**
- * The Frictionless manifest's "name" field is supposed to carry the
- * original filename of each resource. For the container
(unpacked/0.<ext>),
- * that name should be the filename the user supplied on the multipart
+ * The manifest's "tk:resource-name" carries the original filename of each
resource
+ * ("name" is the Frictionless identifier, derived from the path). For the
container
+ * (unpacked/0.<ext>), it should be the filename the user supplied on the
multipart
* upload -- not the server's internal spool filename.
*/
@Test
@@ -566,9 +595,10 @@ public class UnpackerResourceTest extends CXFTestBase {
"Manifest should list the container at unpacked/0. Resources:
" +
dataPackage.get("resources"));
- JsonNode nameNode = containerResource.get("name");
+ assertEquals(0, containerResource.get("tk:embedded-id").asInt(),
containerResource.toString());
+ JsonNode nameNode = containerResource.get("tk:resource-name");
assertNotNull(nameNode,
- "Container resource should carry a 'name' field. Resource: " +
containerResource);
+ "Container resource should carry 'tk:resource-name'. Resource:
" + containerResource);
assertEquals(uploadFilename, nameNode.asText(),
"Container's manifest name should be the user-supplied upload
filename, " +
"not the server's internal spool filename. Resource: "
+ containerResource);
@@ -616,7 +646,7 @@ public class UnpackerResourceTest extends CXFTestBase {
/**
* Documents the shape of /unpack output in FRICTIONLESS mode (no /all,
- * default config): a datapackage.json manifest, the metadata.json envelope
+ * default config): a datapackage.json manifest, the rmeta.json envelope
* a Frictionless package carries by default, and the unpacked/<id>
children.
*/
@Test
@@ -648,8 +678,8 @@ public class UnpackerResourceTest extends CXFTestBase {
assertTrue(data.containsKey("datapackage.json"),
"Should contain datapackage.json manifest. Entries: " +
data.keySet());
- assertTrue(data.containsKey("metadata.json"),
- "A Frictionless package carries metadata.json by default.
Entries: " + data.keySet());
+ assertTrue(data.containsKey("rmeta.json"),
+ "A Frictionless package carries rmeta.json by default.
Entries: " + data.keySet());
boolean hasUnpacked = data.keySet().stream().anyMatch(k ->
k.startsWith("unpacked/"));
assertTrue(hasUnpacked, "Should contain unpacked/ entries. Entries: "
+ data.keySet());
}
@@ -716,7 +746,7 @@ public class UnpackerResourceTest extends CXFTestBase {
/**
* Documents that includeFullMetadata=true in FRICTIONLESS adds a
- * metadata.json envelope whose entries carry tk:content (the extracted
+ * rmeta.json envelope whose entries carry tk:content (the extracted
* text) and Content-Type alongside the per-document metadata.
*/
@Test
@@ -746,13 +776,13 @@ public class UnpackerResourceTest extends CXFTestBase {
assertEquals(200, response.getStatus());
Map<String, byte[]> data = readZipArchiveBytes((InputStream)
response.getEntity());
- byte[] metadataBytes = data.get("metadata.json");
- assertNotNull(metadataBytes, "metadata.json should be present when
includeFullMetadata=true. " +
+ byte[] metadataBytes = data.get("rmeta.json");
+ assertNotNull(metadataBytes, "rmeta.json should be present when
includeFullMetadata=true. " +
"Entries: " + data.keySet());
JsonNode metadata = MAPPER.readTree(metadataBytes);
assertTrue(metadata.isArray() && metadata.size() > 0,
- "metadata.json should be a non-empty array");
+ "rmeta.json should be a non-empty array");
JsonNode container = metadata.get(0);
assertTrue(container.has("Content-Type"),
