poppler/SplashOutputDev.cc | 68 +++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 30 deletions(-)
New commits: commit 75663433262c087d3729f16a528e37a345e0f6b4 Author: Adam Reichold <[email protected]> Date: Sat Sep 22 10:15:14 2018 +0200 Make look-up size computation for drawSoftMaskedImage unsigned to avoid overflowing into negative number and only check multiplied allocations. oss-fuzz/10158 diff --git a/poppler/SplashOutputDev.cc b/poppler/SplashOutputDev.cc index 39dd0858..edf63a96 100644 --- a/poppler/SplashOutputDev.cc +++ b/poppler/SplashOutputDev.cc @@ -3955,7 +3955,6 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object * /* ref */, GfxColor deviceN; #endif Guchar pix; - int n, i; #ifdef SPLASH_CMYK colorMap->getColorSpace()->createMapping(bitmap->getSeparationList(), SPOT_NCOMPS); @@ -3964,7 +3963,7 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object * /* ref */, state->getOverprintMode(), nullptr); ctm = state->getCTM(); - for (i = 0; i < 6; ++i) { + for (int i = 0; i < 6; ++i) { if (!std::isfinite(ctm[i])) return; } mat[0] = ctm[0]; @@ -3977,11 +3976,11 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object * /* ref */, //----- set up the soft mask if (maskColorMap->getMatteColor() != nullptr) { - const int maskChars = maskWidth * maskHeight; - Guchar *data = (Guchar *) gmalloc_checkoverflow(maskChars); - if (unlikely(data == nullptr)) { + int maskChars; + if (checkedMultiply(maskWidth, maskHeight, &maskChars)) { return; } + Guchar *data = (Guchar *) gmalloc(maskChars); maskStr->reset(); const int readChars = maskStr->doGetChars(maskChars, data); if (unlikely(readChars < maskChars)) { @@ -4002,13 +4001,9 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object * /* ref */, imgMaskData.y = 0; imgMaskData.maskStr = nullptr; imgMaskData.maskColorMap = nullptr; - n = 1 << maskColorMap->getBits(); - imgMaskData.lookup = (SplashColorPtr)gmalloc_checkoverflow(n); - if (unlikely(imgMaskData.lookup == nullptr)) { - delete imgMaskData.imgStr; - return; - } - for (i = 0; i < n; ++i) { + const unsigned n = 1 << maskColorMap->getBits(); + imgMaskData.lookup = (SplashColorPtr)gmalloc(n); + for (unsigned i = 0; i < n; ++i) { pix = (Guchar)i; maskColorMap->getGray(&pix, &gray); imgMaskData.lookup[i] = colToByte(gray); @@ -4055,24 +4050,22 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object * /* ref */, // build a lookup table here imgData.lookup = nullptr; if (colorMap->getNumPixelComps() == 1) { - n = 1 << colorMap->getBits(); + const unsigned n = 1 << colorMap->getBits(); switch (colorMode) { case splashModeMono1: case splashModeMono8: - imgData.lookup = (SplashColorPtr)gmalloc_checkoverflow(n); - if (likely(imgData.lookup != nullptr)) { - for (i = 0; i < n; ++i) { - pix = (Guchar)i; - colorMap->getGray(&pix, &gray); - imgData.lookup[i] = colToByte(gray); - } + imgData.lookup = (SplashColorPtr)gmalloc(n); + for (unsigned i = 0; i < n; ++i) { + pix = (Guchar)i; + colorMap->getGray(&pix, &gray); + imgData.lookup[i] = colToByte(gray); } break; case splashModeRGB8: case splashModeBGR8: - imgData.lookup = (SplashColorPtr)gmallocn(n, 3); + imgData.lookup = (SplashColorPtr)gmallocn_checkoverflow(n, 3); if (likely(imgData.lookup != nullptr)) { - for (i = 0; i < n; ++i) { + for (unsigned i = 0; i < n; ++i) { pix = (Guchar)i; colorMap->getRGB(&pix, &rgb); imgData.lookup[3*i] = colToByte(rgb.r); @@ -4084,7 +4077,7 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object * /* ref */, case splashModeXBGR8: imgData.lookup = (SplashColorPtr)gmallocn_checkoverflow(n, 4); if (likely(imgData.lookup != nullptr)) { - for (i = 0; i < n; ++i) { + for (unsigned i = 0; i < n; ++i) { pix = (Guchar)i; colorMap->getRGB(&pix, &rgb); imgData.lookup[4*i] = colToByte(rgb.r); @@ -4098,7 +4091,7 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object * /* ref */, case splashModeCMYK8: imgData.lookup = (SplashColorPtr)gmallocn_checkoverflow(n, 4); if (likely(imgData.lookup != nullptr)) { - for (i = 0; i < n; ++i) { + for (unsigned i = 0; i < n; ++i) { pix = (Guchar)i; colorMap->getCMYK(&pix, &cmyk); imgData.lookup[4*i] = colToByte(cmyk.c); @@ -4111,7 +4104,7 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object * /* ref */, case splashModeDeviceN8: imgData.lookup = (SplashColorPtr)gmallocn_checkoverflow(n, SPOT_NCOMPS+4); if (likely(imgData.lookup != nullptr)) { - for (i = 0; i < n; ++i) { + for (unsigned i = 0; i < n; ++i) { pix = (Guchar)i; colorMap->getDeviceN(&pix, &deviceN); for (int cp = 0; cp < SPOT_NCOMPS+4; cp++) commit 769308e8f46bd6b9eaaed9159071fd78943e74e9 Author: Adam Reichold <[email protected]> Date: Fri Sep 21 10:52:28 2018 +0200 Do some more checks for allocation failure in SplashOutputDev::drawSoftMaskedImage. oss-fuzz/10582 diff --git a/poppler/SplashOutputDev.cc b/poppler/SplashOutputDev.cc index 6faa7477..39dd0858 100644 --- a/poppler/SplashOutputDev.cc +++ b/poppler/SplashOutputDev.cc @@ -3932,7 +3932,7 @@ void SplashOutputDev::drawMaskedImage(GfxState *state, Object *ref, } } -void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object *ref, +void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object * /* ref */, Stream *str, int width, int height, GfxImageColorMap *colorMap, GBool interpolate, @@ -3978,7 +3978,10 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object *ref, if (maskColorMap->getMatteColor() != nullptr) { const int maskChars = maskWidth * maskHeight; - Guchar *data = (Guchar *) gmalloc(maskChars); + Guchar *data = (Guchar *) gmalloc_checkoverflow(maskChars); + if (unlikely(data == nullptr)) { + return; + } maskStr->reset(); const int readChars = maskStr->doGetChars(maskChars, data); if (unlikely(readChars < maskChars)) { @@ -4000,7 +4003,11 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object *ref, imgMaskData.maskStr = nullptr; imgMaskData.maskColorMap = nullptr; n = 1 << maskColorMap->getBits(); - imgMaskData.lookup = (SplashColorPtr)gmalloc(n); + imgMaskData.lookup = (SplashColorPtr)gmalloc_checkoverflow(n); + if (unlikely(imgMaskData.lookup == nullptr)) { + delete imgMaskData.imgStr; + return; + } for (i = 0; i < n; ++i) { pix = (Guchar)i; maskColorMap->getGray(&pix, &gray); @@ -4052,22 +4059,26 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object *ref, switch (colorMode) { case splashModeMono1: case splashModeMono8: - imgData.lookup = (SplashColorPtr)gmalloc(n); - for (i = 0; i < n; ++i) { - pix = (Guchar)i; - colorMap->getGray(&pix, &gray); - imgData.lookup[i] = colToByte(gray); + imgData.lookup = (SplashColorPtr)gmalloc_checkoverflow(n); + if (likely(imgData.lookup != nullptr)) { + for (i = 0; i < n; ++i) { + pix = (Guchar)i; + colorMap->getGray(&pix, &gray); + imgData.lookup[i] = colToByte(gray); + } } break; case splashModeRGB8: case splashModeBGR8: imgData.lookup = (SplashColorPtr)gmallocn(n, 3); - for (i = 0; i < n; ++i) { - pix = (Guchar)i; - colorMap->getRGB(&pix, &rgb); - imgData.lookup[3*i] = colToByte(rgb.r); - imgData.lookup[3*i+1] = colToByte(rgb.g); - imgData.lookup[3*i+2] = colToByte(rgb.b); + if (likely(imgData.lookup != nullptr)) { + for (i = 0; i < n; ++i) { + pix = (Guchar)i; + colorMap->getRGB(&pix, &rgb); + imgData.lookup[3*i] = colToByte(rgb.r); + imgData.lookup[3*i+1] = colToByte(rgb.g); + imgData.lookup[3*i+2] = colToByte(rgb.b); + } } break; case splashModeXBGR8: @@ -4085,23 +4096,27 @@ void SplashOutputDev::drawSoftMaskedImage(GfxState *state, Object *ref, break; #ifdef SPLASH_CMYK case splashModeCMYK8: - imgData.lookup = (SplashColorPtr)gmallocn(n, 4); - for (i = 0; i < n; ++i) { - pix = (Guchar)i; - colorMap->getCMYK(&pix, &cmyk); - imgData.lookup[4*i] = colToByte(cmyk.c); - imgData.lookup[4*i+1] = colToByte(cmyk.m); - imgData.lookup[4*i+2] = colToByte(cmyk.y); - imgData.lookup[4*i+3] = colToByte(cmyk.k); + imgData.lookup = (SplashColorPtr)gmallocn_checkoverflow(n, 4); + if (likely(imgData.lookup != nullptr)) { + for (i = 0; i < n; ++i) { + pix = (Guchar)i; + colorMap->getCMYK(&pix, &cmyk); + imgData.lookup[4*i] = colToByte(cmyk.c); + imgData.lookup[4*i+1] = colToByte(cmyk.m); + imgData.lookup[4*i+2] = colToByte(cmyk.y); + imgData.lookup[4*i+3] = colToByte(cmyk.k); + } } break; case splashModeDeviceN8: - imgData.lookup = (SplashColorPtr)gmallocn(n, SPOT_NCOMPS+4); - for (i = 0; i < n; ++i) { - pix = (Guchar)i; - colorMap->getDeviceN(&pix, &deviceN); - for (int cp = 0; cp < SPOT_NCOMPS+4; cp++) - imgData.lookup[(SPOT_NCOMPS+4)*i + cp] = colToByte(deviceN.c[cp]); + imgData.lookup = (SplashColorPtr)gmallocn_checkoverflow(n, SPOT_NCOMPS+4); + if (likely(imgData.lookup != nullptr)) { + for (i = 0; i < n; ++i) { + pix = (Guchar)i; + colorMap->getDeviceN(&pix, &deviceN); + for (int cp = 0; cp < SPOT_NCOMPS+4; cp++) + imgData.lookup[(SPOT_NCOMPS+4)*i + cp] = colToByte(deviceN.c[cp]); + } } break; #endif _______________________________________________ poppler mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/poppler
