This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 0b2b09c823 TIKA-5042: Parse Canon CR3 raws, with their Exif data and
previews (#3535)
0b2b09c823 is described below
commit 0b2b09c8236613a02f9ab32a52270fc0b45fb455
Author: Dominik Schmidt <[email protected]>
AuthorDate: Fri Nov 6 17:34:08 2026 +0100
TIKA-5042: Parse Canon CR3 raws, with their Exif data and previews (#3535)
* TIKA-5042: emit the Canon CR3 previews
---
CHANGES.txt | 6 +
docs/modules/ROOT/pages/formats.adoc | 3 +
docs/modules/ROOT/partials/supported-formats.adoc | 2 +
.../tika/parser/image/Cr3AutoDetectTest.java | 88 +++++
.../org/apache/tika/parser/image/Cr3Parser.java | 377 +++++++++++++++++++++
.../apache/tika/parser/image/Cr3ParserTest.java | 359 ++++++++++++++++++++
.../src/test/resources/test-documents/testCR3.cr3 | Bin 0 -> 5896 bytes
7 files changed, 835 insertions(+)
diff --git a/CHANGES.txt b/CHANGES.txt
index 406a6a186d..5c6977995f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -81,6 +81,12 @@ Release 4.3.0 - ???
EXTRACTION AND FORMATS
+ * Canon CR3 raws have a parser of their own, Cr3Parser in the image module,
instead
+ of going through the MP4 parser: their Exif data (IFD0, Exif IFD, Canon
maker note,
+ GPS) is extracted, the JPEG preview (PRVW, 1620x1080 on current bodies)
is the
+ THUMBNAIL and the 160x120 THMB thumbnail is INLINE, so the thumbnail
preset serves
+ a preview (TIKA-5042).
+
* Blender files (.blend, .blender, application/x-blender, new) yield the
preview Blender stores
as the THUMBNAIL embedded document, for every file format version
including the one
of Blender 5.0. A gzip- or zstd-compressed file is detected by its
content and read
diff --git a/docs/modules/ROOT/pages/formats.adoc
b/docs/modules/ROOT/pages/formats.adoc
index f04e24cb70..0c0197b6c6 100644
--- a/docs/modules/ROOT/pages/formats.adoc
+++ b/docs/modules/ROOT/pages/formats.adoc
@@ -234,6 +234,9 @@
link:{api}/org/apache/tika/parser/image/TiffParser.html[TiffParser], which use
the metadata-extractor library for Exif, IPTC and XMP; and from
link:{api}/org/apache/tika/parser/image/RawTiffParser.html[RawTiffParser] for
camera RAW files built on TIFF.
+link:{api}/org/apache/tika/parser/image/Cr3Parser.html[Cr3Parser] reads Canon
CR3
+raw files, which are built on the ISO base media file format: their Exif data
and
+their stored JPEG previews.
link:{api}/org/apache/tika/parser/image/HeifParser.html[HeifParser] (HEIF/HEIC,
AVIF),
link:{api}/org/apache/tika/parser/image/JXLParser.html[JXLParser] (JPEG XL),
diff --git a/docs/modules/ROOT/partials/supported-formats.adoc
b/docs/modules/ROOT/partials/supported-formats.adoc
index 5202459962..d7a8fec5eb 100644
--- a/docs/modules/ROOT/partials/supported-formats.adoc
+++ b/docs/modules/ROOT/partials/supported-formats.adoc
@@ -107,6 +107,8 @@
* `org.apache.tika.parser.image.`
link:{tika-javadoc-url}/org/apache/tika/parser/image/BPGParser.html[BPGParser]
** `image/bpg`
** `image/x-bpg`
+* `org.apache.tika.parser.image.`
link:{tika-javadoc-url}/org/apache/tika/parser/image/Cr3Parser.html[Cr3Parser]
+** `image/x-canon-cr3`
* `org.apache.tika.parser.image.`
link:{tika-javadoc-url}/org/apache/tika/parser/image/HeifParser.html[HeifParser]
** `image/avif`
** `image/heic`
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/image/Cr3AutoDetectTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/image/Cr3AutoDetectTest.java
new file mode 100644
index 0000000000..f4b0447936
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/image/Cr3AutoDetectTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.image;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HexFormat;
+import java.util.List;
+import javax.imageio.ImageIO;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+
+/** With every standard parser, a CR3 goes to Cr3Parser, not to the MP4 parser
of its family. */
+public class Cr3AutoDetectTest extends TikaTest {
+
+ @Test
+ public void testAutoDetectParserPicksCr3Parser() throws Exception {
+ List<Metadata> list;
+ try (TikaInputStream tis = TikaInputStream.get(cr3())) {
+ list = getRecursiveMetadata(tis, true);
+ }
+ Metadata cr3 = list.get(0);
+ assertEquals("image/x-canon-cr3", cr3.get(HttpHeaders.CONTENT_TYPE));
+ assertContains(Cr3Parser.class.getName(),
+
Arrays.asList(cr3.getValues(TikaCoreProperties.TIKA_PARSED_BY)));
+ assertEquals(2, list.size());
+
assertEquals(TikaCoreProperties.EmbeddedResourceType.THUMBNAIL.toString(),
+ list.get(1).get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+ }
+
+ /** ftyp crx, an empty moov and the Canon preview box with a PRVW record.
*/
+ private static byte[] cr3() throws IOException {
+ byte[] jpeg = jpeg();
+ ByteArrayOutputStream prvw = new ByteArrayOutputStream();
+ prvw.writeBytes(new byte[4]);
+ prvw.writeBytes(ByteBuffer.allocate(8).putShort((short)
1).putShort((short) 48)
+ .putShort((short) 32).putShort((short) 1).array());
+ prvw.writeBytes(ByteBuffer.allocate(4).putInt(jpeg.length).array());
+ prvw.writeBytes(jpeg);
+ ByteArrayOutputStream preview = new ByteArrayOutputStream();
+
preview.writeBytes(HexFormat.of().parseHex("eaf42b5e1c984b88b9fbb7dc406e4d16"));
+ preview.writeBytes(new byte[]{0, 0, 0, 0, 0, 0, 0, 1});
+ preview.writeBytes(box("PRVW", prvw.toByteArray()));
+ ByteArrayOutputStream file = new ByteArrayOutputStream();
+ file.writeBytes(box("ftyp", "crx \0\0\0\1crx
isom".getBytes(StandardCharsets.US_ASCII)));
+ file.writeBytes(box("moov", box("mvhd", new byte[100])));
+ file.writeBytes(box("uuid", preview.toByteArray()));
+ file.writeBytes(box("mdat", new byte[16]));
+ return file.toByteArray();
+ }
+
+ private static byte[] box(String type, byte[] payload) {
+ return ByteBuffer.allocate(8 + payload.length).putInt(8 +
payload.length)
+
.put(type.getBytes(StandardCharsets.US_ASCII)).put(payload).array();
+ }
+
+ private static byte[] jpeg() throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ImageIO.write(new BufferedImage(48, 32, BufferedImage.TYPE_INT_RGB),
"jpg", out);
+ return out.toByteArray();
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/Cr3Parser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/Cr3Parser.java
new file mode 100644
index 0000000000..bb711441e1
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/main/java/org/apache/tika/parser/image/Cr3Parser.java
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.image;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+
+import com.drew.imaging.tiff.TiffProcessingException;
+import com.drew.imaging.tiff.TiffReader;
+import com.drew.imaging.tiff.TiffStandard;
+import com.drew.lang.ByteArrayReader;
+import com.drew.metadata.Directory;
+import com.drew.metadata.MetadataException;
+import com.drew.metadata.exif.ExifIFD0Directory;
+import com.drew.metadata.exif.ExifSubIFDDirectory;
+import com.drew.metadata.exif.ExifTiffHandler;
+import com.drew.metadata.exif.GpsDirectory;
+import com.drew.metadata.exif.makernotes.CanonMakernoteDirectory;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+import org.apache.tika.annotation.TikaComponent;
+import org.apache.tika.config.ConfigDeserializer;
+import org.apache.tika.config.JsonConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.extractor.EmbeddedDocumentUtil;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TIFF;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.xmp.XmpExtractor;
+import org.apache.tika.sax.XHTMLContentHandler;
+
+/**
+ * Parser for Canon CR3 raw images. CR3 is built on the ISO base media file
format,
+ * like HEIF, and keeps what a still image needs in Canon {@code uuid} boxes:
+ * <ul>
+ * <li>{@code 85c0b687-820f-11e0-8111-f4ce462b6a48} in {@code moov} holds
the Exif
+ * data as four TIFF blocks, {@code CMT1} (IFD0), {@code CMT2} (Exif IFD),
+ * {@code CMT3} (Canon maker note) and {@code CMT4} (GPS), and a {@code THMB}
+ * record with a 160x120 JPEG, emitted as
+ * {@link TikaCoreProperties.EmbeddedResourceType#INLINE}</li>
+ * <li>{@code eaf42b5e-1c98-4b88-b9fb-b7dc406e4d16} at the top level holds,
after
+ * eight bytes, a {@code PRVW} record with the large JPEG preview (1620
pixels wide
+ * on current bodies), emitted as the
+ * {@link TikaCoreProperties.EmbeddedResourceType#THUMBNAIL}</li>
+ * <li>{@code be7acfcb-97a9-42e8-9c71-999491e3afac} at the top level holds
an XMP packet,
+ * with the rating given in the camera or in Canon's software</li>
+ * </ul>
+ * The {@code THMB} and {@code PRVW} records carry width and height and the
JPEG
+ * length ahead of the JPEG, which starts 24 bytes into the record.
+ */
+@TikaComponent(name = "cr3-parser")
+public class Cr3Parser implements Parser {
+
+ private static final long serialVersionUID = 7130874255196421580L;
+
+ private static final Set<MediaType> SUPPORTED_TYPES =
+ Collections.singleton(MediaType.image("x-canon-cr3"));
+
+ private static final byte[] METADATA_UUID =
uuid("85c0b687820f11e08111f4ce462b6a48");
+ private static final byte[] PREVIEW_UUID =
uuid("eaf42b5e1c984b88b9fbb7dc406e4d16");
+ private static final byte[] XMP_UUID =
uuid("be7acfcb97a942e89c71999491e3afac");
+ private static final int UUID_LENGTH = 16;
+ /** The preview box's children start after eight bytes of its own. */
+ private static final int PREVIEW_CHILDREN_OFFSET = 8;
+ /** From the start of a PRVW or THMB box to its JPEG: size, type and
sixteen header bytes. */
+ private static final int JPEG_OFFSET = 24;
+ /** An Exif block above this size is not read; real ones are a few KB to
some tens of KB. */
+ private static final int MAX_TIFF_BLOCK = 4 * 1024 * 1024;
+ /** A preview above this size is not read; real ones are some hundred KB.
*/
+ private static final int MAX_PREVIEW = 64 * 1024 * 1024;
+ /** An XMP packet above this size is not read; Canon writes one of 64 KB,
padding included. */
+ private static final int MAX_XMP = 1024 * 1024;
+
+ private final ImageMetadataConfig defaultConfig;
+
+ public Cr3Parser() {
+ this(new ImageMetadataConfig());
+ }
+
+ public Cr3Parser(ImageMetadataConfig config) {
+ this.defaultConfig = config;
+ }
+
+ public Cr3Parser(JsonConfig jsonConfig) {
+ this(ConfigDeserializer.buildConfig(jsonConfig,
ImageMetadataConfig.class));
+ }
+
+ @Override
+ public Set<MediaType> getSupportedTypes(ParseContext context) {
+ return SUPPORTED_TYPES;
+ }
+
+ /** A JPEG record: where its JPEG starts, how long it is, and the size it
declares. */
+ private record Jpeg(long offset, int length, int width, int height) {
+ }
+
+ @Override
+ public void parse(TikaInputStream tis, ContentHandler handler, Metadata
metadata,
+ ParseContext context) throws IOException, SAXException,
TikaException {
+ Path path = tis.getPath();
+ com.drew.metadata.Metadata exif = new com.drew.metadata.Metadata();
+ Jpeg thumbnail = null;
+ Jpeg preview = null;
+ byte[] xmp = null;
+ try (FileChannel channel = FileChannel.open(path,
StandardOpenOption.READ)) {
+ long length = channel.size();
+ long offset = 0;
+ while (offset + 8 <= length) {
+ Box box = Box.read(channel, offset, length);
+ if (box == null) {
+ break;
+ }
+ if (box.is("moov")) {
+ thumbnail = readMoov(channel, box, exif);
+ } else if (box.is("uuid") && box.hasUuid(channel,
PREVIEW_UUID)) {
+ preview = readRecord(channel, box.payload + UUID_LENGTH +
PREVIEW_CHILDREN_OFFSET,
+ box.end, "PRVW", 10, 16);
+ } else if (box.is("uuid") && box.hasUuid(channel, XMP_UUID)) {
+ long xmpLength = box.end - box.payload - UUID_LENGTH;
+ if (xmpLength > 0 && xmpLength <= MAX_XMP) {
+ xmp = readBytes(channel, box.payload + UUID_LENGTH,
(int) xmpLength);
+ }
+ }
+ offset = box.end;
+ }
+
+ if (xmp != null) {
+ try {
+ new XmpExtractor().extract(xmp, metadata, context);
+ } catch (SecurityException e) {
+ throw e;
+ } catch (IOException | SAXException | TikaException |
RuntimeException e) {
+ EmbeddedDocumentUtil.recordException(e, metadata, context);
+ }
+ }
+ try {
+ new ImageMetadataExtractor(metadata,
+ ImageMetadataConfig.resolve(context, "cr3-parser",
defaultConfig))
+ .handle(exif);
+ } catch (MetadataException e) {
+ throw new TikaException("Can't read the CR3 Exif data", e);
+ }
+
+ XHTMLContentHandler xhtml = new XHTMLContentHandler(handler,
metadata, context);
+ xhtml.startDocument();
+ if (thumbnail != null) {
+ emit(channel, thumbnail,
TikaCoreProperties.EmbeddedResourceType.INLINE, xhtml,
+ context);
+ }
+ if (preview != null) {
+ emit(channel, preview,
TikaCoreProperties.EmbeddedResourceType.THUMBNAIL, xhtml,
+ context);
+ }
+ xhtml.endDocument();
+ }
+ }
+
+ /** Reads the Exif blocks of the Canon box in {@code moov} and returns its
THMB record. */
+ private static Jpeg readMoov(FileChannel channel, Box moov,
com.drew.metadata.Metadata exif)
+ throws IOException {
+ long offset = moov.payload;
+ while (offset + 8 <= moov.end) {
+ Box box = Box.read(channel, offset, moov.end);
+ if (box == null) {
+ return null;
+ }
+ if (box.is("uuid") && box.hasUuid(channel, METADATA_UUID)) {
+ Jpeg thumbnail = null;
+ long child = box.payload + UUID_LENGTH;
+ while (child + 8 <= box.end) {
+ Box record = Box.read(channel, child, box.end);
+ if (record == null) {
+ break;
+ }
+ switch (record.type) {
+ case "CMT1" -> readExif(channel, record, exif,
ExifIFD0Directory.class);
+ case "CMT2" -> readExif(channel, record, exif,
ExifSubIFDDirectory.class);
+ case "CMT3" -> readExif(channel, record, exif,
CanonMakernoteDirectory.class);
+ case "CMT4" -> readExif(channel, record, exif,
GpsDirectory.class);
+ case "THMB" -> thumbnail = readRecord(channel,
record.offset, box.end,
+ "THMB", 8, 12);
+ default -> {
+ }
+ }
+ child = record.end;
+ }
+ return thumbnail;
+ }
+ offset = box.end;
+ }
+ return null;
+ }
+
+ private static void readExif(FileChannel channel, Box record,
com.drew.metadata.Metadata exif,
+ Class<? extends Directory> firstDirectory)
throws IOException {
+ long size = record.end - record.payload;
+ if (size <= 8 || size > MAX_TIFF_BLOCK) {
+ return;
+ }
+ byte[] tiff = readBytes(channel, record.payload, (int) size);
+ try {
+ new TiffReader().processTiff(new ByteArrayReader(tiff),
+ new StartingDirectoryHandler(exif, firstDirectory), 0);
+ } catch (TiffProcessingException | IOException e) {
+ //a block that cannot be read leaves the others;
metadata-extractor records
+ //what it found up to there
+ }
+ }
+
+ /**
+ * Finds the record of the given type from {@code start} on and returns
where its
+ * JPEG is; {@code sizeOffset} and {@code lengthOffset} count from the
record's type.
+ */
+ private static Jpeg readRecord(FileChannel channel, long start, long end,
String type,
+ int sizeOffset, int lengthOffset) throws
IOException {
+ long offset = start;
+ while (offset + 8 <= end) {
+ Box box = Box.read(channel, offset, end);
+ if (box == null) {
+ return null;
+ }
+ if (box.is(type)) {
+ if (box.end - box.offset < JPEG_OFFSET) {
+ return null;
+ }
+ ByteBuffer header = ByteBuffer.wrap(readBytes(channel,
box.offset + 4, JPEG_OFFSET - 4));
+ long jpegLength = header.getInt(lengthOffset) & 0xFFFFFFFFL;
+ long jpeg = box.offset + JPEG_OFFSET;
+ if (jpegLength < 2 || jpegLength > box.end - jpeg ||
jpegLength > MAX_PREVIEW) {
+ return null;
+ }
+ return new Jpeg(jpeg, (int) jpegLength,
header.getShort(sizeOffset) & 0xFFFF,
+ header.getShort(sizeOffset + 2) & 0xFFFF);
+ }
+ offset = box.end;
+ }
+ return null;
+ }
+
+ private static void emit(FileChannel channel, Jpeg jpeg,
+ TikaCoreProperties.EmbeddedResourceType type,
+ XHTMLContentHandler xhtml, ParseContext context)
+ throws IOException, SAXException {
+ byte[] bytes = readBytes(channel, jpeg.offset(), jpeg.length());
+ if ((bytes[0] & 0xFF) != 0xFF || (bytes[1] & 0xFF) != 0xD8) {
+ return;
+ }
+ Metadata metadata = Metadata.newInstance(context);
+ metadata.set(HttpHeaders.CONTENT_TYPE, "image/jpeg");
+ metadata.set(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE, type.name());
+ EmbeddedDocumentUtil.setGeneratedResourceName(metadata,
+ type == TikaCoreProperties.EmbeddedResourceType.THUMBNAIL
+ ? EmbeddedDocumentUtil.EmbeddedResourcePrefix.THUMBNAIL
+ : EmbeddedDocumentUtil.EmbeddedResourcePrefix.IMAGE,
0, "image/jpeg");
+ metadata.set(TIFF.IMAGE_WIDTH, jpeg.width());
+ metadata.set(TIFF.IMAGE_LENGTH, jpeg.height());
+ EmbeddedDocumentExtractor extractor =
+ EmbeddedDocumentUtil.getEmbeddedDocumentExtractor(context);
+ if (!extractor.shouldParseEmbedded(metadata, context)) {
+ return;
+ }
+ try (TikaInputStream image = TikaInputStream.get(bytes)) {
+ extractor.parseEmbedded(image, xhtml, metadata, context, true);
+ }
+ }
+
+ private static byte[] readBytes(FileChannel channel, long offset, int
length) throws IOException {
+ ByteBuffer buffer = ByteBuffer.allocate(length);
+ while (buffer.hasRemaining()) {
+ if (channel.read(buffer, offset + buffer.position()) < 0) {
+ throw new IOException("unexpected end of the CR3 file");
+ }
+ }
+ return buffer.array();
+ }
+
+ private static byte[] uuid(String hex) {
+ byte[] bytes = new byte[UUID_LENGTH];
+ for (int i = 0; i < UUID_LENGTH; i++) {
+ bytes[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i +
2), 16);
+ }
+ return bytes;
+ }
+
+ /** An ISO base media box: where it starts, where its payload starts and
where it ends. */
+ private static final class Box {
+ final long offset;
+ final long payload;
+ final long end;
+ final String type;
+
+ private Box(long offset, long payload, long end, String type) {
+ this.offset = offset;
+ this.payload = payload;
+ this.end = end;
+ this.type = type;
+ }
+
+ /** The box at {@code offset}, or null when its header does not fit
before {@code limit}. */
+ static Box read(FileChannel channel, long offset, long limit) throws
IOException {
+ if (offset + 8 > limit) {
+ return null;
+ }
+ ByteBuffer header = ByteBuffer.wrap(readBytes(channel, offset, 8));
+ long size = header.getInt(0) & 0xFFFFFFFFL;
+ String type = new String(header.array(), 4, 4,
StandardCharsets.ISO_8859_1);
+ long payload = offset + 8;
+ if (size == 1) {
+ if (offset + 16 > limit) {
+ return null;
+ }
+ size = ByteBuffer.wrap(readBytes(channel, offset + 8,
8)).getLong();
+ payload = offset + 16;
+ } else if (size == 0) {
+ size = limit - offset;
+ }
+ if (size < payload - offset || size > limit - offset) {
+ return null;
+ }
+ return new Box(offset, payload, offset + size, type);
+ }
+
+ boolean is(String boxType) {
+ return type.equals(boxType);
+ }
+
+ boolean hasUuid(FileChannel channel, byte[] uuid) throws IOException {
+ return end - payload >= UUID_LENGTH
+ && Arrays.equals(readBytes(channel, payload, UUID_LENGTH),
uuid);
+ }
+ }
+
+ /** Starts a TIFF block in the given directory instead of IFD0. */
+ private static final class StartingDirectoryHandler extends
ExifTiffHandler {
+ private final Class<? extends Directory> firstDirectory;
+
+ StartingDirectoryHandler(com.drew.metadata.Metadata metadata,
+ Class<? extends Directory> firstDirectory) {
+ super(metadata, null, 0);
+ this.firstDirectory = firstDirectory;
+ }
+
+ @Override
+ public TiffStandard processTiffMarker(int marker) {
+ pushDirectory(firstDirectory);
+ return TiffStandard.TIFF;
+ }
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/Cr3ParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/Cr3ParserTest.java
new file mode 100644
index 0000000000..fca9882cca
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/java/org/apache/tika/parser/image/Cr3ParserTest.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.image;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import javax.imageio.ImageIO;
+
+import org.junit.jupiter.api.Test;
+import org.xml.sax.ContentHandler;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.extractor.EmbeddedDocumentExtractor;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Geographic;
+import org.apache.tika.metadata.HttpHeaders;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TIFF;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.XMP;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.sax.BodyContentHandler;
+
+/**
+ * Canon CR3, on a file built here with the box layout of a real CR3: the Exif
blocks
+ * and a {@code THMB} record in the Canon {@code uuid} box inside {@code
moov}, a
+ * {@code PRVW} record in the Canon {@code uuid} box at the top level.
+ */
+public class Cr3ParserTest extends TikaTest {
+
+ private static final String METADATA_UUID =
"85c0b687820f11e08111f4ce462b6a48";
+ private static final String PREVIEW_UUID =
"eaf42b5e1c984b88b9fbb7dc406e4d16";
+ private static final String XMP_UUID = "be7acfcb97a942e89c71999491e3afac";
+
+ private static final short ASCII = 2;
+ private static final short SHORT = 3;
+ private static final short RATIONAL = 5;
+
+ @Test
+ public void testExif() throws Exception {
+ Metadata metadata = new Metadata();
+ parse(cr3(jpeg(16, 12), jpeg(48, 32)), metadata, new ArrayList<>(),
new ArrayList<>());
+ assertEquals("Canon", metadata.get(TIFF.EQUIPMENT_MAKE));
+ assertEquals("Canon EOS R7", metadata.get(TIFF.EQUIPMENT_MODEL));
+ assertEquals("0.004", metadata.get(TIFF.EXPOSURE_TIME));
+ assertEquals("100", metadata.get(TIFF.ISO_SPEED_RATINGS));
+ assertEquals("48.5", metadata.get(Geographic.LATITUDE));
+ assertEquals("11.25", metadata.get(Geographic.LONGITUDE));
+ assertEquals("Firmware Version 1.0.1", metadata.get("img:Firmware
Version"),
+ "the maker note block is read as Canon's maker note");
+ }
+
+ @Test
+ public void testPreviewIsThumbnailAndThmbIsInline() throws Exception {
+ List<Metadata> embedded = new ArrayList<>();
+ parse(cr3(jpeg(16, 12), jpeg(48, 32)), new Metadata(), embedded, new
ArrayList<>());
+ assertEquals(2, embedded.size());
+ assertEquals("INLINE",
embedded.get(0).get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+ assertEquals("16", embedded.get(0).get(TIFF.IMAGE_WIDTH));
+ assertEquals("THUMBNAIL",
embedded.get(1).get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+ assertEquals("48", embedded.get(1).get(TIFF.IMAGE_WIDTH));
+ assertEquals("32", embedded.get(1).get(TIFF.IMAGE_LENGTH));
+ assertEquals("image/jpeg",
embedded.get(1).get(HttpHeaders.CONTENT_TYPE));
+ assertEquals("image-0.jpg",
embedded.get(0).get(TikaCoreProperties.RESOURCE_NAME_KEY));
+ assertEquals("thumbnail-0.jpg",
embedded.get(1).get(TikaCoreProperties.RESOURCE_NAME_KEY));
+ }
+
+ @Test
+ public void testJpegBytesAreExact() throws Exception {
+ byte[] prvw = jpeg(48, 32);
+ List<byte[]> bytes = new ArrayList<>();
+ parse(cr3(jpeg(16, 12), prvw), new Metadata(), new ArrayList<>(),
bytes);
+ assertArrayEquals(prvw, bytes.get(1));
+ }
+
+ @Test
+ public void testLengthBeyondTheBoxIsIgnored() throws Exception {
+ byte[] file = cr3(jpeg(16, 12), jpeg(48, 32));
+ int prvw = indexOf(file, "PRVW".getBytes(StandardCharsets.US_ASCII));
+ // within the 64 MB cap, but running past the box into mdat, where the
bytes still
+ // start like the JPEG
+ ByteBuffer.wrap(file).putInt(prvw + 16, jpeg(48, 32).length + 10);
+ List<Metadata> embedded = new ArrayList<>();
+ Metadata metadata = new Metadata();
+ parse(file, metadata, embedded, new ArrayList<>());
+ assertEquals(1, embedded.size());
+ assertEquals("INLINE",
embedded.get(0).get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+ assertEquals("Canon EOS R7", metadata.get(TIFF.EQUIPMENT_MODEL));
+ }
+
+ /** The XMP box as the camera writes it: a packet with the rating, padded
to 64 KB. */
+ @Test
+ public void testXmpRating() throws Exception {
+ ByteArrayOutputStream file = new ByteArrayOutputStream();
+ file.write(cr3(jpeg(16, 12), jpeg(48, 32)));
+ file.write(xmpBox(4, 65536));
+ Metadata metadata = new Metadata();
+ List<Metadata> embedded = new ArrayList<>();
+ parse(file.toByteArray(), metadata, embedded, new ArrayList<>());
+ assertEquals("4", metadata.get(XMP.RATING));
+ assertEquals("Canon EOS R7", metadata.get(TIFF.EQUIPMENT_MODEL));
+ assertEquals(2, embedded.size());
+ }
+
+ @Test
+ public void testBadXmpKeepsTheRest() throws Exception {
+ ByteArrayOutputStream xmpUuid = new ByteArrayOutputStream();
+ xmpUuid.write(hex(XMP_UUID));
+ xmpUuid.write(ascii("<x:xmpmeta xmlns:x=\"adobe:ns:meta/\"><rdf:RDF"));
+ ByteArrayOutputStream file = new ByteArrayOutputStream();
+ file.write(cr3(jpeg(16, 12), jpeg(48, 32)));
+ file.write(box("uuid", xmpUuid.toByteArray()));
+ Metadata metadata = new Metadata();
+ List<Metadata> embedded = new ArrayList<>();
+ parse(file.toByteArray(), metadata, embedded, new ArrayList<>());
+ assertNull(metadata.get(XMP.RATING));
+ assertEquals("Canon EOS R7", metadata.get(TIFF.EQUIPMENT_MODEL));
+ assertEquals(2, embedded.size());
+ }
+
+ /**
+ * testCR3.cr3 is the file these tests build, as cr3(), with colored
previews and an XMP
+ * box rated 3, saved for trying the parser by hand.
+ */
+ @Test
+ public void testFixture() throws Exception {
+ Metadata metadata = new Metadata();
+ List<Metadata> embedded = new ArrayList<>();
+ try (TikaInputStream tis =
getResourceAsStream("/test-documents/testCR3.cr3")) {
+ parse(tis.readAllBytes(), metadata, embedded, new ArrayList<>());
+ }
+ assertEquals("Canon", metadata.get(TIFF.EQUIPMENT_MAKE));
+ assertEquals("Canon EOS R7", metadata.get(TIFF.EQUIPMENT_MODEL));
+ assertEquals("0.004", metadata.get(TIFF.EXPOSURE_TIME));
+ assertEquals("48.5", metadata.get(Geographic.LATITUDE));
+ assertEquals("Firmware Version 1.0.1", metadata.get("img:Firmware
Version"));
+ assertEquals("3", metadata.get(XMP.RATING));
+ assertEquals(2, embedded.size());
+ assertEquals("INLINE",
embedded.get(0).get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+ assertEquals("THUMBNAIL",
embedded.get(1).get(TikaCoreProperties.EMBEDDED_RESOURCE_TYPE));
+ assertEquals("48", embedded.get(1).get(TIFF.IMAGE_WIDTH));
+ }
+
+ @Test
+ public void testNoCanonBoxes() throws Exception {
+ ByteArrayOutputStream file = new ByteArrayOutputStream();
+ file.write(box("ftyp", ascii("crx \0\0\0\1crx isom")));
+ file.write(box("moov", box("mvhd", new byte[100])));
+ List<Metadata> embedded = new ArrayList<>();
+ Metadata metadata = new Metadata();
+ parse(file.toByteArray(), metadata, embedded, new ArrayList<>());
+ assertEquals(0, embedded.size());
+ assertNull(metadata.get(TIFF.EQUIPMENT_MAKE));
+ }
+
+ private void parse(byte[] file, Metadata metadata, List<Metadata>
embedded, List<byte[]> bytes)
+ throws Exception {
+ ParseContext context = new ParseContext();
+ context.set(EmbeddedDocumentExtractor.class, new
EmbeddedDocumentExtractor() {
+ @Override
+ public boolean shouldParseEmbedded(Metadata m, ParseContext ctx) {
+ return true;
+ }
+
+ @Override
+ public void parseEmbedded(TikaInputStream stream, ContentHandler
handler, Metadata m,
+ ParseContext ctx, boolean outputHtml)
throws IOException {
+ embedded.add(m);
+ bytes.add(stream.readAllBytes());
+ }
+ });
+ try (TikaInputStream tis = TikaInputStream.get(file)) {
+ new Cr3Parser().parse(tis, new BodyContentHandler(), metadata,
context);
+ }
+ }
+
+ private static byte[] cr3(byte[] thmb, byte[] prvw) throws IOException {
+ ByteArrayOutputStream thmbBox = new ByteArrayOutputStream();
+ thmbBox.write(new byte[4]);
+ thmbBox.write(shorts(16, 12));
+ thmbBox.write(ints(thmb.length));
+ thmbBox.write(new byte[]{0, 1, 0, 0});
+ thmbBox.write(thmb);
+ ByteArrayOutputStream metadataUuid = new ByteArrayOutputStream();
+ metadataUuid.write(hex(METADATA_UUID));
+ metadataUuid.write(box("CNCV",
ascii("CanonCR3_001/01.09.00/00.00.00")));
+ metadataUuid.write(box("CMT1", tiff(new Entry(0x010F, ASCII,
ascii("Canon\0")),
+ new Entry(0x0110, ASCII, ascii("Canon EOS R7\0")))));
+ metadataUuid.write(box("CMT2", tiff(new Entry(0x829A, RATIONAL,
rationals(1, 250)),
+ new Entry(0x8827, SHORT,
le(ByteBuffer.allocate(2).putShort((short) 100))))));
+ metadataUuid.write(box("CMT3", tiff(new Entry(0x0007, ASCII,
+ ascii("Firmware Version 1.0.1\0")))));
+ metadataUuid.write(box("CMT4", tiff(new Entry(0x0001, ASCII,
ascii("N\0")),
+ new Entry(0x0002, RATIONAL, rationals(48, 1, 30, 1, 0, 1)),
+ new Entry(0x0003, ASCII, ascii("E\0")),
+ new Entry(0x0004, RATIONAL, rationals(11, 1, 15, 1, 0, 1)))));
+ metadataUuid.write(box("THMB", thmbBox.toByteArray()));
+
+ ByteArrayOutputStream prvwBox = new ByteArrayOutputStream();
+ prvwBox.write(new byte[4]);
+ prvwBox.write(shorts(1, 48, 32, 1));
+ prvwBox.write(ints(prvw.length));
+ prvwBox.write(prvw);
+ ByteArrayOutputStream previewUuid = new ByteArrayOutputStream();
+ previewUuid.write(hex(PREVIEW_UUID));
+ previewUuid.write(new byte[]{0, 0, 0, 0, 0, 0, 0, 1});
+ previewUuid.write(box("PRVW", prvwBox.toByteArray()));
+
+ ByteArrayOutputStream file = new ByteArrayOutputStream();
+ file.write(box("ftyp", ascii("crx \0\0\0\1crx isom")));
+ file.write(box("moov", box("uuid", metadataUuid.toByteArray())));
+ file.write(box("uuid", previewUuid.toByteArray()));
+ file.write(box("mdat", new byte[16]));
+ return file.toByteArray();
+ }
+
+ private record Entry(int tag, short type, byte[] value) {
+ int count() {
+ return type == ASCII ? value.length : type == SHORT ? value.length
/ 2 : value.length / 8;
+ }
+ }
+
+ /** A little-endian TIFF block with one IFD; values that do not fit four
bytes follow it. */
+ private static byte[] tiff(Entry... entries) {
+ int ifdSize = 2 + 12 * entries.length + 4;
+ int dataOffset = 8 + ifdSize;
+ ByteBuffer out = ByteBuffer.allocate(dataOffset +
256).order(ByteOrder.LITTLE_ENDIAN);
+ out.put(new byte[]{'I', 'I', 42, 0}).putInt(8).putShort((short)
entries.length);
+ ByteArrayOutputStream data = new ByteArrayOutputStream();
+ for (Entry e : entries) {
+ out.putShort((short) e.tag()).putShort(e.type()).putInt(e.count());
+ if (e.value().length <= 4) {
+ out.put(Arrays.copyOf(e.value(), 4));
+ } else {
+ out.putInt(dataOffset + data.size());
+ data.writeBytes(e.value());
+ }
+ }
+ out.putInt(0).put(data.toByteArray());
+ return Arrays.copyOf(out.array(), out.position());
+ }
+
+ private static byte[] rationals(int... values) {
+ ByteBuffer b = ByteBuffer.allocate(4 *
values.length).order(ByteOrder.LITTLE_ENDIAN);
+ for (int v : values) {
+ b.putInt(v);
+ }
+ return b.array();
+ }
+
+ private static byte[] le(ByteBuffer b) {
+ byte[] a = b.array();
+ return new byte[]{a[1], a[0]};
+ }
+
+ private static byte[] box(String type, byte[] payload) {
+ return ByteBuffer.allocate(8 + payload.length).putInt(8 +
payload.length)
+ .put(ascii(type)).put(payload).array();
+ }
+
+ private static byte[] jpeg(int width, int height) throws IOException {
+ return jpeg(width, height, 0x000000);
+ }
+
+ private static byte[] jpeg(int width, int height, int rgb) throws
IOException {
+ BufferedImage image = new BufferedImage(width, height,
BufferedImage.TYPE_INT_RGB);
+ for (int y = 0; y < height; y++) {
+ for (int x = 0; x < width; x++) {
+ image.setRGB(x, y, rgb);
+ }
+ }
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ ImageIO.write(image, "jpeg", out);
+ return out.toByteArray();
+ }
+
+ /**
+ * The XMP box as the camera writes it: a packet with the rating, padded
with spaces so
+ * the box holds the given number of bytes after its UUID.
+ */
+ private static byte[] xmpBox(int rating, int length) throws IOException {
+ String packet = "<?xpacket begin='\uFEFF'
id='W5M0MpCehiHzreSzNTczkc9d'?>"
+ + "<x:xmpmeta xmlns:x=\"adobe:ns:meta/\"><rdf:RDF "
+ + "xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\";>"
+ + "<rdf:Description rdf:about=\"\"
xmlns:xmp=\"http://ns.adobe.com/xap/1.0/\";>"
+ + "<xmp:Rating>" + rating +
"</xmp:Rating></rdf:Description></rdf:RDF></x:xmpmeta>";
+ byte[] end = ascii("<?xpacket end='w'?>");
+ byte[] bytes = packet.getBytes(StandardCharsets.UTF_8);
+ byte[] padded = Arrays.copyOf(bytes, length - end.length);
+ Arrays.fill(padded, bytes.length, padded.length, (byte) ' ');
+ ByteArrayOutputStream uuid = new ByteArrayOutputStream();
+ uuid.write(hex(XMP_UUID));
+ uuid.write(padded);
+ uuid.write(end);
+ return box("uuid", uuid.toByteArray());
+ }
+
+ private static byte[] shorts(int... values) {
+ ByteBuffer b = ByteBuffer.allocate(2 * values.length);
+ for (int v : values) {
+ b.putShort((short) v);
+ }
+ return b.array();
+ }
+
+ private static byte[] ints(int value) {
+ return ByteBuffer.allocate(4).putInt(value).array();
+ }
+
+ private static byte[] ascii(String s) {
+ return s.getBytes(StandardCharsets.US_ASCII);
+ }
+
+ private static byte[] hex(String hex) {
+ byte[] bytes = new byte[hex.length() / 2];
+ for (int i = 0; i < bytes.length; i++) {
+ bytes[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i +
2), 16);
+ }
+ return bytes;
+ }
+
+ private static int indexOf(byte[] data, byte[] needle) {
+ outer:
+ for (int i = 0; i <= data.length - needle.length; i++) {
+ for (int j = 0; j < needle.length; j++) {
+ if (data[i + j] != needle[j]) {
+ continue outer;
+ }
+ }
+ return i;
+ }
+ return -1;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/resources/test-documents/testCR3.cr3
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/resources/test-documents/testCR3.cr3
new file mode 100644
index 0000000000..68ecb32f9f
Binary files /dev/null and
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-image-module/src/test/resources/test-documents/testCR3.cr3
differ
