qingwei727 commented on code in PR #10905:
URL: https://github.com/apache/paimon/pull/10905#discussion_r4480278663
##########
paimon-python/pypaimon/table/row/video_keyframe_index.py:
##########
@@ -280,25 +480,76 @@ def read(offset, size):
raise ValueError("Invalid video keyframe index: truncated
read.")
return data
- version = read(0, 1)[0]
+ version, magic = struct.unpack_from('<BQ', prefix)
+ if magic != cls.MAGIC:
+ raise ValueError("Invalid video keyframe index magic.")
+ if version == 1:
+ return cls._open_v1(read, length, payload_length, prefix)
if version != cls.VERSION:
- raise ValueError("Invalid video keyframe index version.")
- header = cls.HEADER
- _, magic, metadata_count, count = header.unpack(read(0, header.size))
- if magic != cls.MAGIC or count == 0:
- raise ValueError("Invalid video keyframe index header.")
+ raise UnsupportedVideoIndex("Unsupported video keyframe index
version: %s." % version)
+ if length < cls.HEADER.size:
+ raise ValueError("Invalid video keyframe index length.")
+ _, _, metadata_count, count, codec = cls.HEADER.unpack_from(prefix)
+ if codec not in (0, 1):
+ raise UnsupportedVideoIndex("Unsupported video keyframe index
codec: %s." % codec)
+ if count == 0:
+ raise ValueError("Invalid video keyframe index count.")
if metadata_count > cls.MAX_METADATA_RANGE_COUNT:
raise ValueError("Video keyframe index exceeds the metadata-range
limit.")
- offset = header.size + metadata_count * cls.METADATA_RANGE.size
- ranges = cls._metadata(cls.METADATA_RANGE.iter_unpack(
- read(header.size, metadata_count * 16)), payload_length)
- pages = (count + _PagedKeyframes.PAGE_ENTRIES - 1) //
_PagedKeyframes.PAGE_ENTRIES
- if offset + count * 24 + pages * 4 != length:
+ offset = cls.HEADER.size + metadata_count * cls.METADATA_RANGE.size
+ pages = (count - 1) // _PagedKeyframes.PAGE_ENTRIES + 1
+ if offset + pages * (_IndexedKeyframes.DIRECTORY.size + 1) > length:
raise ValueError("Invalid video keyframe index entries length.")
+ if max(length, count * cls.ENTRY.size) <=
_IndexedKeyframes.EAGER_BYTES:
+ prefix += read(len(prefix), length - len(prefix))
+ ranges = cls._metadata(cls.METADATA_RANGE.iter_unpack(
+ read(cls.HEADER.size, metadata_count * 16)), payload_length)
result = cls.__new__(cls)
result.metadata_ranges = ranges
- result._keyframes = _PagedKeyframes(lambda start, size: read(offset +
start, size), count, payload_length)
+ result._keyframes = _IndexedKeyframes(read, count, offset, length,
codec, payload_length)
+ result.keyframes._descriptor(pages - 1)
result.keyframes[0]
+ result.keyframes.load_small()
+ return result
+
+ @classmethod
+ def _open_v1(cls, read, length, payload_length, prefix):
+ if length < cls.V1_HEADER.size:
+ raise ValueError("Invalid video keyframe index length.")
+ _, _, metadata_count, count = cls.V1_HEADER.unpack_from(prefix)
+ if metadata_count > cls.MAX_METADATA_RANGE_COUNT:
+ raise ValueError("Video keyframe index exceeds the metadata-range
limit.")
+ offset = cls.V1_HEADER.size + metadata_count * cls.METADATA_RANGE.size
+ if not count or count > 64 * 1024 or offset >= length:
+ raise ValueError("Invalid video keyframe index entries length.")
+ expected = count * cls.ENTRY.size
+ if length - offset > expected + 64 * 1024:
+ raise ValueError("Invalid video keyframe index compressed length.")
+ ranges = cls._metadata(cls.METADATA_RANGE.iter_unpack(
+ read(cls.V1_HEADER.size, metadata_count * 16)), payload_length)
+ inflater = zlib.decompressobj()
+ chunks = []
+ decoded = 0
+ try:
+ while offset < length:
+ size = min(64 * 1024, length - offset)
+ block = inflater.decompress(read(offset, size), expected -
decoded + 1)
+ chunks.append(block)
+ decoded += len(block)
+ offset += size
+ if decoded > expected or inflater.unused_data or
inflater.unconsumed_tail:
+ raise ValueError("Invalid video keyframe index compressed
length.")
+ if inflater.eof:
+ break
+ except zlib.error as error:
+ raise ValueError("Invalid video keyframe index compressed page.")
from error
+ if decoded != expected or not inflater.eof or offset != length:
+ raise ValueError("Invalid video keyframe index compressed length.")
+ raw = b''.join(chunks)
+ _validate_entries(raw, payload_length, initial=True)
+ result = cls.__new__(cls)
+ result.metadata_ranges = ranges
+ result.keyframes = cls.ENTRY.iter_unpack(raw)
Review Comment:
This routes a released v1 index through the `keyframes` setter. So every
open spools the entries to a temp file and serves them through
`_PagedKeyframes` plus a Python `bisect` over `_Ordinals`. The page cache holds
10 pages, while a full v1 index has 16.
Measured with 65,536 keyframes:
| | 2.1.0 | this PR |
| --- | ---: | ---: |
| open | 37 ms | 88 ms + 1 temp file |
| `preceding()` | 0.5 µs | 207 µs |
The same keyframes stored as eager v2 take 2.5 µs per lookup. A decoded v1
index is at most 1.5 MiB, so `raw` could stay in memory and reuse the eager
path with `searchsorted`, with no temp file.
##########
paimon-python/pypaimon/table/row/video_keyframe_index.py:
##########
@@ -280,25 +480,76 @@ def read(offset, size):
raise ValueError("Invalid video keyframe index: truncated
read.")
return data
- version = read(0, 1)[0]
+ version, magic = struct.unpack_from('<BQ', prefix)
+ if magic != cls.MAGIC:
+ raise ValueError("Invalid video keyframe index magic.")
+ if version == 1:
+ return cls._open_v1(read, length, payload_length, prefix)
Review Comment:
Indexes written by master since #10891 (Oct 31) also carry version byte 1,
so they take this branch and are parsed as the 2.1.0 layout, with every field
after the header shifted by 4 bytes. Opening one raises a plain `ValueError:
Video metadata range is outside the video payload.` rather than
`UnsupportedVideoIndex`. As a result:
- `VideoFrameCollator` doesn't fall back to scanning.
- The copy path in both writers rejects it, so compaction fails. That path
is `video_format_writer.py:216`, plus `VideoFormatWriter.java:343`, which goes
through the same `version == 1` branch at `VideoKeyframeIndex.java:61`.
The layout was never released, but anything built from master since then
writes it. Could we recognize it and raise `UnsupportedVideoIndex` (or
`UnsupportedIndexException` in Java)? Its length is exactly `21 + 16 *
metadata_count + 24 * count + 4 * pages`, and the CRC32 of the first page can
confirm it. Then it gets scanned and survives rewrite like any other unknown
index.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]