# HG changeset patch
# User Steve Borho <[email protected]>
# Date 1424891691 21600
# Wed Feb 25 13:14:51 2015 -0600
# Node ID 603de2f3d4380127f77462bb5497fd8cf232c332
# Parent 515a229245f687815b0f1c9d6973daa56219ba6b
quant: avoid right-shift overflow and undefined behavior trap
diff -r 515a229245f6 -r 603de2f3d438 source/common/quant.cpp
--- a/source/common/quant.cpp Wed Feb 25 12:08:52 2015 -0600
+++ b/source/common/quant.cpp Wed Feb 25 13:14:51 2015 -0600
@@ -1105,7 +1105,8 @@
const uint32_t trSizeCG = 1 << log2TrSizeCG;
X265_CHECK(trSizeCG <= 8, "transform CG is too large\n");
- const uint32_t sigPos = (uint32_t)(sigCoeffGroupFlag64 >> (1 + (cgPosY <<
log2TrSizeCG) + cgPosX));
+ const uint32_t shift = (cgPosY << log2TrSizeCG) + cgPosX + 1;
+ const uint32_t sigPos = (uint32_t)(shift >= 64 ? 0 : sigCoeffGroupFlag64
>> shift);
const uint32_t sigRight = ((int32_t)(cgPosX - (trSizeCG - 1)) >> 31) &
(sigPos & 1);
const uint32_t sigLower = ((int32_t)(cgPosY - (trSizeCG - 1)) >> 31) &
(sigPos >> (trSizeCG - 2)) & 2;
_______________________________________________
x265-devel mailing list
[email protected]
https://mailman.videolan.org/listinfo/x265-devel
