Fix additional issues reported by UB sanitizers
Most of these involved overrunning the signed 32-bit JLONG type whenever
building libjpeg-turbo with a 32-bit compiler. These issues are not
believed to represent actual security threats, but eliminating them
makes it easier to detect such threats should they arise in the future.
diff --git a/ChangeLog.txt b/ChangeLog.txt
index 4a6f488..a064973 100644
--- a/ChangeLog.txt
+++ b/ChangeLog.txt
@@ -29,6 +29,12 @@
[6] The MIPS DSPr2 SIMD code can now be compiled to support either FR=0 or FR=1
FPUs.
+[7] Fixed additional negative left shifts and other issues reported by the GCC
+and Clang undefined behavior sanitizers. Most of these issues affected only
+32-bit code, and none of them was known to pose a security threat, but removing
+the warnings makes it easier to detect actual security issues, should they
+arise in the future.
+
1.4.2
=====
diff --git a/jcdctmgr.c b/jcdctmgr.c
index cef4b5e..1a0d406 100644
--- a/jcdctmgr.c
+++ b/jcdctmgr.c
@@ -6,7 +6,7 @@
* libjpeg-turbo Modifications:
* Copyright (C) 1999-2006, MIYASAKA Masaru.
* Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
- * Copyright (C) 2011, 2014-2015 D. R. Commander
+ * Copyright (C) 2011, 2014-2015, D. R. Commander.
* For conditions of distribution and use, see the accompanying README.ijg
* file.
*
@@ -209,7 +209,11 @@
dtbl[DCTSIZE2 * 0] = (DCTELEM) fq; /* reciprocal */
dtbl[DCTSIZE2 * 1] = (DCTELEM) c; /* correction + roundfactor */
+#ifdef WITH_SIMD
dtbl[DCTSIZE2 * 2] = (DCTELEM) (1 << (sizeof(DCTELEM)*8*2 - r)); /* scale */
+#else
+ dtbl[DCTSIZE2 * 2] = 1;
+#endif
dtbl[DCTSIZE2 * 3] = (DCTELEM) r - sizeof(DCTELEM)*8; /* shift */
if(r <= 16) return 0;
diff --git a/jcphuff.c b/jcphuff.c
index 3b29e02..656fc67 100644
--- a/jcphuff.c
+++ b/jcphuff.c
@@ -231,7 +231,7 @@
/* Emit some bits, unless we are in gather mode */
{
/* This routine is heavily used, so it's worth coding tightly. */
- register JLONG put_buffer = (JLONG) code;
+ register size_t put_buffer = (size_t) code;
register int put_bits = entropy->put_bits;
/* if size is 0, caller used an invalid Huffman table entry */
diff --git a/jdcolor.c b/jdcolor.c
index 6a5d819..af7c575 100644
--- a/jdcolor.c
+++ b/jdcolor.c
@@ -604,7 +604,7 @@
*/
#define DITHER_MASK 0x3
-#define DITHER_ROTATE(x) (((x) << 24) | (((x) >> 8) & 0x00FFFFFF))
+#define DITHER_ROTATE(x) ((((x) & 0xFF) << 24) | (((x) >> 8) & 0x00FFFFFF))
static const JLONG dither_matrix[4] = {
0x0008020A,
0x0C040E06,
diff --git a/jdhuff.h b/jdhuff.h
index 46d1916..422a7a1 100644
--- a/jdhuff.h
+++ b/jdhuff.h
@@ -74,12 +74,12 @@
#if SIZEOF_SIZE_T==8 || defined(_WIN64)
-typedef size_t bit_buf_type; /* type of bit-extraction buffer */
+typedef size_t bit_buf_type; /* type of bit-extraction buffer */
#define BIT_BUF_SIZE 64 /* size of buffer in bits */
#else
-typedef JLONG bit_buf_type; /* type of bit-extraction buffer */
+typedef unsigned long bit_buf_type; /* type of bit-extraction buffer */
#define BIT_BUF_SIZE 32 /* size of buffer in bits */
#endif
diff --git a/jdmerge.c b/jdmerge.c
index 9b2fdf8..629b871 100644
--- a/jdmerge.c
+++ b/jdmerge.c
@@ -456,7 +456,7 @@
*/
#define DITHER_MASK 0x3
-#define DITHER_ROTATE(x) (((x) << 24) | (((x) >> 8) & 0x00FFFFFF))
+#define DITHER_ROTATE(x) ((((x) & 0xFF) << 24) | (((x) >> 8) & 0x00FFFFFF))
static const JLONG dither_matrix[4] = {
0x0008020A,
0x0C040E06,