Fix undefined shift in opj_get_all_encoding_parameters
The value 1u << (l_pdx + l_level_no) is only used to calculate a minimum,
so skip it when the shift doesn't even fit unsigned integer. Also use the uint
min version since all values being considered are unsigned anyways.
Bug: chromium:666892
Change-Id: I79c6e52022aa894033c5cdabec29c4b8313e293b
Reviewed-on: https://pdfium-review.googlesource.com/4891
Reviewed-by: dsinclair <dsinclair@chromium.org>
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Commit-Queue: Nicolás Peña <npm@chromium.org>
diff --git a/third_party/libopenjpeg20/0030-undefined-shift-opj_get_all_encoding_parameters.patch b/third_party/libopenjpeg20/0030-undefined-shift-opj_get_all_encoding_parameters.patch
new file mode 100644
index 0000000..3ba3a2f
--- /dev/null
+++ b/third_party/libopenjpeg20/0030-undefined-shift-opj_get_all_encoding_parameters.patch
@@ -0,0 +1,29 @@
+diff --git a/third_party/libopenjpeg20/pi.c b/third_party/libopenjpeg20/pi.c
+index 083674222..6af38d0ca 100644
+--- a/third_party/libopenjpeg20/pi.c
++++ b/third_party/libopenjpeg20/pi.c
+@@ -782,18 +782,18 @@ static void opj_get_all_encoding_parameters( const opj_image_t *p_image,
+ /* use custom size for precincts*/
+ l_level_no = l_tccp->numresolutions - 1;
+ for (resno = 0; resno < l_tccp->numresolutions; ++resno) {
+- OPJ_UINT32 l_dx, l_dy;
+-
+ /* precinct width and height*/
+ l_pdx = l_tccp->prcw[resno];
+ l_pdy = l_tccp->prch[resno];
+ *lResolutionPtr++ = l_pdx;
+ *lResolutionPtr++ = l_pdy;
+- l_dx = l_img_comp->dx * (1u << (l_pdx + l_level_no));
+- l_dy = l_img_comp->dy * (1u << (l_pdy + l_level_no));
+ /* take the minimum size for l_dx for each comp and resolution*/
+- *p_dx_min = (OPJ_UINT32)opj_int_min((OPJ_INT32)*p_dx_min, (OPJ_INT32)l_dx);
+- *p_dy_min = (OPJ_UINT32)opj_int_min((OPJ_INT32)*p_dy_min, (OPJ_INT32)l_dy);
++ if (l_pdx + l_level_no < 32) {
++ *p_dx_min = opj_uint_min(*p_dx_min, l_img_comp->dx * (1u << (l_pdx + l_level_no)));
++ }
++ if (l_pdy + l_level_no < 32) {
++ *p_dy_min = opj_uint_min(*p_dy_min, l_img_comp->dy * (1u << (l_pdy + l_level_no)));
++ }
+
+ /* various calculations of extents*/
+ l_rx0 = opj_int_ceildivpow2(l_tcx0, (OPJ_INT32)l_level_no);
diff --git a/third_party/libopenjpeg20/README.pdfium b/third_party/libopenjpeg20/README.pdfium
index 9ce0c05..a75de2c 100644
--- a/third_party/libopenjpeg20/README.pdfium
+++ b/third_party/libopenjpeg20/README.pdfium
@@ -39,4 +39,5 @@
0027-undefined-shift-opj_t1_decode_cblk.patch: upstream fix for a ubsan bug.
0028-upstream-check-size-in-opj_j2k_read_siz.patch: upstream patch in j2k.c.
0029-avoid-division-by-0: fix some /0 and %0 in pi.c (caused by bad shifts).
+0030-undefined-shift-opj_get_all_encoding_parameters.patch: fix undefined shift in pi.c method.
TODO(thestig): List all the other patches.
diff --git a/third_party/libopenjpeg20/pi.c b/third_party/libopenjpeg20/pi.c
index 0836742..6af38d0 100644
--- a/third_party/libopenjpeg20/pi.c
+++ b/third_party/libopenjpeg20/pi.c
@@ -782,18 +782,18 @@
/* use custom size for precincts*/
l_level_no = l_tccp->numresolutions - 1;
for (resno = 0; resno < l_tccp->numresolutions; ++resno) {
- OPJ_UINT32 l_dx, l_dy;
-
/* precinct width and height*/
l_pdx = l_tccp->prcw[resno];
l_pdy = l_tccp->prch[resno];
*lResolutionPtr++ = l_pdx;
*lResolutionPtr++ = l_pdy;
- l_dx = l_img_comp->dx * (1u << (l_pdx + l_level_no));
- l_dy = l_img_comp->dy * (1u << (l_pdy + l_level_no));
/* take the minimum size for l_dx for each comp and resolution*/
- *p_dx_min = (OPJ_UINT32)opj_int_min((OPJ_INT32)*p_dx_min, (OPJ_INT32)l_dx);
- *p_dy_min = (OPJ_UINT32)opj_int_min((OPJ_INT32)*p_dy_min, (OPJ_INT32)l_dy);
+ if (l_pdx + l_level_no < 32) {
+ *p_dx_min = opj_uint_min(*p_dx_min, l_img_comp->dx * (1u << (l_pdx + l_level_no)));
+ }
+ if (l_pdy + l_level_no < 32) {
+ *p_dy_min = opj_uint_min(*p_dy_min, l_img_comp->dy * (1u << (l_pdy + l_level_no)));
+ }
/* various calculations of extents*/
l_rx0 = opj_int_ceildivpow2(l_tcx0, (OPJ_INT32)l_level_no);