PR #25299 opened by Marti2203
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25299
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25299.patch
`shufflepixels` mishandles frame sizes that aren't a multiple of the block
size, in three ways.
1. Block mode shuffles only the full blocks, and the map entries for the
remaining right and bottom margin stay zero from `av_calloc()`. As a result
every margin pixel becomes a copy of the top-left pixel. For a 67x53 gray frame
with `m=block:w=10:h=10` that is all 551 margin pixels. A block larger than the
frame turns the whole output into that one pixel value.
2. In horizontal and vertical mode, the inverse direction sizes each block by
the space left at the output position, `FFMIN(block_w, w - x)`. The forward
direction uses the size of the source block, `FFMIN(block_w, w - rand *
block_w)`. So with the same seed, `d=inverse` doesn't undo `d=forward`.
3. Because of the mismatch in 2, if the shorter last block is drawn before the
end of the row, the inverse writes `block_w` map entries starting at its index,
past the end of the row. On a frame of height 1 (width 1 in vertical mode) that
goes past the end of the `w * h` map allocation:
```
ffmpeg -f lavfi -i nullsrc=s=13x1:d=0.04,format=gray16le -vf
shufflepixels=d=inverse:w=10:s=2 -f null -
==ERROR: AddressSanitizer: heap-buffer-overflow ... WRITE of size 4
#0 in make_horizontal_map libavfilter/vf_shufflepixels.c:95
```
The same thing shows up with 1001x1 `w=1000`, and with 1x1001
`m=vertical:h=1000` in `make_vertical_map` (line 133).
The patch initializes the block map to the identity, so pixels outside the full
blocks stay in place. It also makes the inverse use the same block size as the
forward direction. Both directions then cut the frame into the same pieces, so
the inverse map really is the inverse of the forward map, and every index it
writes is below `w` (or `h`).
Reproducer (67x53 gray, seed 7):
```
SRC='testsrc2=s=67x53:d=0.04,format=gray'
ffmpeg -f lavfi -i "$SRC" -vf
"shufflepixels=m=horizontal:w=10:s=7,shufflepixels=m=horizontal:w=10:s=7:d=inverse"
-f md5 -
```
Before: the input is 2d75e2c11fa3c1876ec6d009ecff452a, but the forward+inverse
output is 62c6f27864c3a23ee2686dfa2f287074 for horizontal,
9b9b4c789f66c01146eb33a471eedfd9 for vertical (`h=10`) and
6e453f99bab17c0f430ab5937205d588 for block (`w=16:h=8`). In block mode with
`w=h=10`, 551/551 margin pixels change. After: all three round trips return the
input, no margin pixel changes, and the full blocks are still a permutation of
the input blocks. Forward horizontal/vertical output is unchanged, and so is
anything whose size is a multiple of the block size. The ASan build reports
nothing for the three cases above, or for a sweep of 384 combinations of sizes
(1x1 to 67x53), modes, directions, block sizes and seeds.
The second commit adds `fate-filter-shufflepixels-block` (block mode on 67x53)
and `fate-filter-shufflepixels-inverse` (forward followed by inverse in all
three modes, so its output equals the input). Both fail without the fix and
pass with it (THREADS=1 and 4). shufflepixels had no FATE coverage before.
This issue was found by an LLM-Enhanced Static Analysis Engine and verified
manually.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
From 1716a4723371534d3977f1028947e31b85e49be1 Mon Sep 17 00:00:00 2001
From: Martin Mirchev <[email protected]>
Date: Wed, 21 Oct 2026 18:22:16 +0800
Subject: [PATCH 1/2] avfilter/vf_shufflepixels: fix sizes not a multiple of
the block size
In block mode only the full blocks are shuffled, but the map of the
remaining right and bottom margin was left zeroed, so those pixels were
all replaced by the top-left pixel. Initialize the map to the identity
so the margin stays in place.
In horizontal and vertical mode the inverse direction sized each block
by the remaining output size instead of the size of the source block,
like the forward direction does. When the frame size is not a multiple
of the block size, the inverse therefore did not undo the forward
shuffle with the same seed, and the map entries written for the last,
shorter block could go past the end of the row; for a frame of height
1 (or width 1 in vertical mode) this wrote past the end of the map
allocation.
Signed-off-by: Martin Mirchev <[email protected]>
---
libavfilter/vf_shufflepixels.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/libavfilter/vf_shufflepixels.c b/libavfilter/vf_shufflepixels.c
index ebe0b81155e..15941de3ddd 100644
--- a/libavfilter/vf_shufflepixels.c
+++ b/libavfilter/vf_shufflepixels.c
@@ -79,13 +79,11 @@ static void make_horizontal_map(AVFilterContext *ctx)
int rand = av_lfg_get(c) % nb_blocks;
if (used[rand] == 0) {
- int width;
+ const int width = FFMIN(s->block_w, s->planewidth[0] - rand *
s->block_w);
if (s->direction) {
- width = FFMIN(s->block_w, s->planewidth[0] - x);
map[rand * s->block_w] = x;
} else {
- width = FFMIN(s->block_w, s->planewidth[0] - rand *
s->block_w);
map[x] = rand * s->block_w;
}
used[rand] = 1;
@@ -117,13 +115,11 @@ static void make_vertical_map(AVFilterContext *ctx)
int rand = av_lfg_get(c) % nb_blocks;
if (used[rand] == 0) {
- int height;
+ const int height = FFMIN(s->block_h, s->planeheight[0] - rand *
s->block_h);
if (s->direction) {
- height = FFMIN(s->block_h, s->planeheight[0] - y);
map[rand * s->block_h] = y;
} else {
- height = FFMIN(s->block_h, s->planeheight[0] - rand *
s->block_h);
map[y] = rand * s->block_h;
}
used[rand] = 1;
@@ -152,6 +148,10 @@ static void make_block_map(AVFilterContext *ctx)
uint8_t *used = s->used;
int32_t *map = s->map;
+ /* pixels not covered by a full block stay in place */
+ for (int i = 0; i < s->planewidth[0] * s->planeheight[0]; i++)
+ map[i] = i;
+
for (int i = 0; i < nb_blocks;) {
int rand = av_lfg_get(c) % nb_blocks;
--
2.54.0
From cbc3423169e522fbf941a8f628694356730dc153 Mon Sep 17 00:00:00 2001
From: Martin Mirchev <[email protected]>
Date: Wed, 21 Oct 2026 18:24:23 +0800
Subject: [PATCH 2/2] fate/filter-video: add shufflepixels tests
Test block mode on a frame size that is not a multiple of the block
size, and that the inverse direction undoes the forward shuffle in all
three modes; the output of the second test must match the input.
Signed-off-by: Martin Mirchev <[email protected]>
---
tests/fate/filter-video.mak | 4 ++++
tests/ref/fate/filter-shufflepixels-block | 10 ++++++++++
tests/ref/fate/filter-shufflepixels-inverse | 10 ++++++++++
3 files changed, 24 insertions(+)
create mode 100644 tests/ref/fate/filter-shufflepixels-block
create mode 100644 tests/ref/fate/filter-shufflepixels-inverse
diff --git a/tests/fate/filter-video.mak b/tests/fate/filter-video.mak
index 4a065ec0320..f3659a5111f 100644
--- a/tests/fate/filter-video.mak
+++ b/tests/fate/filter-video.mak
@@ -388,6 +388,10 @@ fate-filter-shuffleplanes-swapuv: CMD = framecrc -c:v
pgmyuv -i $(SRC) -vf shuff
FATE_FILTER_VSYNTH_PGMYUV-yes += $(FATE_SHUFFLEPLANES-yes)
+FATE_FILTER-$(call FILTERFRAMECRC, TESTSRC2 FORMAT SHUFFLEPIXELS) +=
fate-filter-shufflepixels-block fate-filter-shufflepixels-inverse
+fate-filter-shufflepixels-block: CMD = framecrc -lavfi
testsrc2=s=67x53:r=5:d=1,format=yuv444p,shufflepixels=m=block:w=10:h=10:s=1
-pix_fmt yuv444p
+fate-filter-shufflepixels-inverse: CMD = framecrc -lavfi
testsrc2=s=67x53:r=5:d=1,format=yuv444p,shufflepixels=m=horizontal:w=10:s=1,shufflepixels=m=horizontal:w=10:s=1:d=inverse,shufflepixels=m=vertical:h=10:s=2,shufflepixels=m=vertical:h=10:s=2:d=inverse,shufflepixels=m=block:w=16:h=8:s=3,shufflepixels=m=block:w=16:h=8:s=3:d=inverse
-pix_fmt yuv444p
+
FATE_SWAPRECT += fate-filter-swaprect
fate-filter-swaprect: CMD = framecrc -c:v pgmyuv -i $(SRC) -vf swaprect
diff --git a/tests/ref/fate/filter-shufflepixels-block
b/tests/ref/fate/filter-shufflepixels-block
new file mode 100644
index 00000000000..de70754e536
--- /dev/null
+++ b/tests/ref/fate/filter-shufflepixels-block
@@ -0,0 +1,10 @@
+#tb 0: 1/5
+#media_type 0: video
+#codec_id 0: rawvideo
+#dimensions 0: 67x53
+#sar 0: 1/1
+0, 0, 0, 1, 10653, 0x4a722d6b
+0, 1, 1, 1, 10653, 0xa8622882
+0, 2, 2, 1, 10653, 0x94492877
+0, 3, 3, 1, 10653, 0x89af2be7
+0, 4, 4, 1, 10653, 0x1de0329b
diff --git a/tests/ref/fate/filter-shufflepixels-inverse
b/tests/ref/fate/filter-shufflepixels-inverse
new file mode 100644
index 00000000000..879fbe92730
--- /dev/null
+++ b/tests/ref/fate/filter-shufflepixels-inverse
@@ -0,0 +1,10 @@
+#tb 0: 1/5
+#media_type 0: video
+#codec_id 0: rawvideo
+#dimensions 0: 67x53
+#sar 0: 1/1
+0, 0, 0, 1, 10653, 0x00f72d6b
+0, 1, 1, 1, 10653, 0xd2bc2882
+0, 2, 2, 1, 10653, 0x2f1f2877
+0, 3, 3, 1, 10653, 0x2d2b2be7
+0, 4, 4, 1, 10653, 0xac49329b
--
2.54.0
_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]
