# HG changeset patch # User Santhoshini Sekar <santhosh...@multicorewareinc.com> # Date 1513927117 -19800 # Fri Dec 22 12:48:37 2017 +0530 # Branch stable # Node ID f41671b53ae084a8207e24a91d3a7cf2a96cd1ce # Parent f7498acb38746d9b799d1b45343a3db56cdba306 Fix possible NULL pointer dereferencing in cudata init
diff --git a/source/common/framedata.cpp b/source/common/framedata.cpp --- a/source/common/framedata.cpp +++ b/source/common/framedata.cpp @@ -41,9 +41,12 @@ if (param.rc.bStatWrite) m_spsrps = const_cast<RPS*>(sps.spsrps); - m_cuMemPool.create(0, param.internalCsp, sps.numCUsInFrame, param); - for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++) - m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr); + bool isallocated = m_cuMemPool.create(0, param.internalCsp, sps.numCUsInFrame, param); + if (isallocated) + for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++) + m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr); + else + return false; CHECKED_MALLOC_ZERO(m_cuStat, RCStatCU, sps.numCUsInFrame); CHECKED_MALLOC(m_rowStat, RCStatRow, sps.numCuInHeight); diff --git a/source/encoder/analysis.cpp b/source/encoder/analysis.cpp --- a/source/encoder/analysis.cpp +++ b/source/encoder/analysis.cpp @@ -101,15 +101,17 @@ { ModeDepth &md = m_modeDepth[depth]; - md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param); + ok &= md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param); ok &= md.fencYuv.create(cuSize, csp); - - for (int j = 0; j < MAX_PRED_TYPES; j++) + if (ok) { - md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param, j); - ok &= md.pred[j].predYuv.create(cuSize, csp); - ok &= md.pred[j].reconYuv.create(cuSize, csp); - md.pred[j].fencYuv = &md.fencYuv; + for (int j = 0; j < MAX_PRED_TYPES; j++) + { + md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param, j); + ok &= md.pred[j].predYuv.create(cuSize, csp); + ok &= md.pred[j].reconYuv.create(cuSize, csp); + md.pred[j].fencYuv = &md.fencYuv; + } } } if (m_param->sourceHeight >= 1080) diff --git a/source/encoder/api.cpp b/source/encoder/api.cpp --- a/source/encoder/api.cpp +++ b/source/encoder/api.cpp @@ -286,7 +286,7 @@ else if (pi_nal) *pi_nal = 0; - if (numEncoded && encoder->m_param->csvLogLevel) + if (numEncoded > 0 && encoder->m_param->csvLogLevel) x265_csvlog_frame(encoder->m_param, pic_out); if (numEncoded < 0) diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp --- a/source/encoder/encoder.cpp +++ b/source/encoder/encoder.cpp @@ -1342,7 +1342,9 @@ } else { - frameEnc->allocEncodeData(m_reconfigure ? m_latestParam : m_param, m_sps); + bool isallocated = frameEnc->allocEncodeData(m_reconfigure ? m_latestParam : m_param, m_sps); + if (!isallocated) + return -1; Slice* slice = frameEnc->m_encData->m_slice; slice->m_sps = &m_sps; slice->m_pps = &m_pps;
# HG changeset patch # User Santhoshini Sekar <santhosh...@multicorewareinc.com> # Date 1513927117 -19800 # Fri Dec 22 12:48:37 2017 +0530 # Branch stable # Node ID f41671b53ae084a8207e24a91d3a7cf2a96cd1ce # Parent f7498acb38746d9b799d1b45343a3db56cdba306 Fix possible NULL pointer dereferencing in cudata init diff --git a/source/common/framedata.cpp b/source/common/framedata.cpp --- a/source/common/framedata.cpp +++ b/source/common/framedata.cpp @@ -41,9 +41,12 @@ if (param.rc.bStatWrite) m_spsrps = const_cast<RPS*>(sps.spsrps); - m_cuMemPool.create(0, param.internalCsp, sps.numCUsInFrame, param); - for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++) - m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr); + bool isallocated = m_cuMemPool.create(0, param.internalCsp, sps.numCUsInFrame, param); + if (isallocated) + for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++) + m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr); + else + return false; CHECKED_MALLOC_ZERO(m_cuStat, RCStatCU, sps.numCUsInFrame); CHECKED_MALLOC(m_rowStat, RCStatRow, sps.numCuInHeight); diff --git a/source/encoder/analysis.cpp b/source/encoder/analysis.cpp --- a/source/encoder/analysis.cpp +++ b/source/encoder/analysis.cpp @@ -101,15 +101,17 @@ { ModeDepth &md = m_modeDepth[depth]; - md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param); + ok &= md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param); ok &= md.fencYuv.create(cuSize, csp); - - for (int j = 0; j < MAX_PRED_TYPES; j++) + if (ok) { - md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param, j); - ok &= md.pred[j].predYuv.create(cuSize, csp); - ok &= md.pred[j].reconYuv.create(cuSize, csp); - md.pred[j].fencYuv = &md.fencYuv; + for (int j = 0; j < MAX_PRED_TYPES; j++) + { + md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param, j); + ok &= md.pred[j].predYuv.create(cuSize, csp); + ok &= md.pred[j].reconYuv.create(cuSize, csp); + md.pred[j].fencYuv = &md.fencYuv; + } } } if (m_param->sourceHeight >= 1080) diff --git a/source/encoder/api.cpp b/source/encoder/api.cpp --- a/source/encoder/api.cpp +++ b/source/encoder/api.cpp @@ -286,7 +286,7 @@ else if (pi_nal) *pi_nal = 0; - if (numEncoded && encoder->m_param->csvLogLevel) + if (numEncoded > 0 && encoder->m_param->csvLogLevel) x265_csvlog_frame(encoder->m_param, pic_out); if (numEncoded < 0) diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp --- a/source/encoder/encoder.cpp +++ b/source/encoder/encoder.cpp @@ -1342,7 +1342,9 @@ } else { - frameEnc->allocEncodeData(m_reconfigure ? m_latestParam : m_param, m_sps); + bool isallocated = frameEnc->allocEncodeData(m_reconfigure ? m_latestParam : m_param, m_sps); + if (!isallocated) + return -1; Slice* slice = frameEnc->m_encData->m_slice; slice->m_sps = &m_sps; slice->m_pps = &m_pps;
_______________________________________________ x265-devel mailing list x265-devel@videolan.org https://mailman.videolan.org/listinfo/x265-devel