Fix memory issues in BIO_*printf functions

The internal |fmtstr| function used in processing a "%s" format string
in the BIO_*printf functions could overflow while calculating the length
of a string and cause an OOB read when printing very long strings.

Additionally the internal |doapr_outch| function can attempt to write to
an OOB memory location (at an offset from the NULL pointer) in the event of
a memory allocation failure. In 1.0.2 and below this could be caused where
the size of a buffer to be allocated is greater than INT_MAX. E.g. this
could be in processing a very long "%s" format string. Memory leaks can also
occur.

These issues will only occur on certain platforms where sizeof(size_t) >
sizeof(int). E.g. many 64 bit systems. The first issue may mask the second
issue dependent on compiler behaviour.

These problems could enable attacks where large amounts of untrusted data
is passed to the BIO_*printf functions. If applications use these functions
in this way then they could be vulnerable. OpenSSL itself uses these
functions when printing out human-readable dumps of ASN.1 data. Therefore
applications that print this data could be vulnerable if the data is from
untrusted sources. OpenSSL command line applications could also be
vulnerable where they print out ASN.1 data, or if untrusted data is passed
as command line arguments.

Libssl is not considered directly vulnerable. Additionally certificates etc
received via remote connections via libssl are also unlikely to be able to
trigger these issues because of message size limits enforced within libssl.

CVE-2016-0799

Issue reported by Guido Vranken.

(cherry-picked from commit 578b956fe741bf8e84055547b1e83c28dd902c73)

Bug: 29060514
Change-Id: I8bba4cc655dd01e5a384d15f3cf10eff3ca530e5
(cherry picked from commit 93efbca025e7e441b7991c380f79d864ce81dec5)
diff --git a/lib/openssl/crypto/bio/b_print.c b/lib/openssl/crypto/bio/b_print.c
index 143a7cf..a8c5dc9 100644
--- a/lib/openssl/crypto/bio/b_print.c
+++ b/lib/openssl/crypto/bio/b_print.c
@@ -125,14 +125,14 @@
 #define LLONG long
 #endif
 
-static void fmtstr     (char **, char **, size_t *, size_t *,
+static int fmtstr(char **, char **, size_t *, size_t *,
 			const char *, int, int, int);
-static void fmtint     (char **, char **, size_t *, size_t *,
+static int fmtint(char **, char **, size_t *, size_t *,
 			LLONG, int, int, int, int);
-static void fmtfp      (char **, char **, size_t *, size_t *,
+static int fmtfp(char **, char **, size_t *, size_t *,
 			LDOUBLE, int, int, int);
-static void doapr_outch (char **, char **, size_t *, size_t *, int);
-static void _dopr(char **sbuffer, char **buffer,
+static int doapr_outch(char **, char **, size_t *, size_t *, int);
+static int _dopr(char **sbuffer, char **buffer,
 		  size_t *maxlen, size_t *retlen, int *truncated,
 		  const char *format, va_list args);
 
@@ -165,7 +165,7 @@
 #define char_to_int(p) (p - '0')
 #define OSSL_MAX(p,q) ((p >= q) ? p : q)
 
-static void
+static int
 _dopr(
     char **sbuffer,
     char **buffer,
@@ -200,7 +200,8 @@
             if (ch == '%')
                 state = DP_S_FLAGS;
             else
-                doapr_outch(sbuffer,buffer, &currlen, maxlen, ch);
+                if (!doapr_outch(sbuffer, buffer, &currlen, maxlen, ch))
+			return 0;
             ch = *format++;
             break;
         case DP_S_FLAGS:
@@ -306,8 +307,9 @@
                     value = va_arg(args, int);
                     break;
                 }
-                fmtint(sbuffer, buffer, &currlen, maxlen,
-                       value, 10, min, max, flags);
+                if (!fmtint(sbuffer, buffer, &currlen, maxlen, value, 10, min,
+                            max, flags))
+                    return 0;
                 break;
             case 'X':
                 flags |= DP_F_UP;
@@ -332,17 +334,19 @@
                         unsigned int);
                     break;
                 }
-                fmtint(sbuffer, buffer, &currlen, maxlen, value,
+                if (!fmtint(sbuffer, buffer, &currlen, maxlen, value,
                        ch == 'o' ? 8 : (ch == 'u' ? 10 : 16),
-                       min, max, flags);
+                            min, max, flags))
+                    return 0;
                 break;
             case 'f':
                 if (cflags == DP_C_LDOUBLE)
                     fvalue = va_arg(args, LDOUBLE);
                 else
                     fvalue = va_arg(args, double);
-                fmtfp(sbuffer, buffer, &currlen, maxlen,
-                      fvalue, min, max, flags);
+                if (!fmtfp(sbuffer, buffer, &currlen, maxlen, fvalue, min, max,
+                           flags))
+                    return 0;
                 break;
             case 'E':
                 flags |= DP_F_UP;
@@ -361,8 +365,9 @@
                     fvalue = va_arg(args, double);
                 break;
             case 'c':
-                doapr_outch(sbuffer, buffer, &currlen, maxlen,
-                    va_arg(args, int));
+                if(!doapr_outch(sbuffer, buffer, &currlen, maxlen,
+                            va_arg(args, int)))
+                    return 0;
                 break;
             case 's':
                 strvalue = va_arg(args, char *);
@@ -372,13 +377,15 @@
 		    else
 			max = *maxlen;
 		}
-                fmtstr(sbuffer, buffer, &currlen, maxlen, strvalue,
-                       flags, min, max);
+                if (!fmtstr(sbuffer, buffer, &currlen, maxlen, strvalue,
+                            flags, min, max))
+                    return 0;
                 break;
             case 'p':
                 value = (long)va_arg(args, void *);
-                fmtint(sbuffer, buffer, &currlen, maxlen,
-                    value, 16, min, max, flags|DP_F_NUM);
+                if (!fmtint(sbuffer, buffer, &currlen, maxlen,
+                            value, 16, min, max, flags | DP_F_NUM))
+                    return 0;
                 break;
             case 'n': /* XXX */
                 if (cflags == DP_C_SHORT) {
@@ -400,7 +407,8 @@
                 }
                 break;
             case '%':
-                doapr_outch(sbuffer, buffer, &currlen, maxlen, ch);
+                if(!doapr_outch(sbuffer, buffer, &currlen, maxlen, ch))
+                    return 0;
                 break;
             case 'w':
                 /* not supported yet, treat as next char */
@@ -424,12 +432,13 @@
     *truncated = (currlen > *maxlen - 1);
     if (*truncated)
         currlen = *maxlen - 1;
-    doapr_outch(sbuffer, buffer, &currlen, maxlen, '\0');
+    if(!doapr_outch(sbuffer, buffer, &currlen, maxlen, '\0'))
+        return 0;
     *retlen = currlen - 1;
-    return;
+    return 1;
 }
 
-static void
+static int
 fmtstr(
     char **sbuffer,
     char **buffer,
@@ -440,36 +449,44 @@
     int min,
     int max)
 {
-    int padlen, strln;
+    int padlen;
+    size_t strln;
     int cnt = 0;
 
     if (value == 0)
         value = "<NULL>";
-    for (strln = 0; value[strln]; ++strln)
-        ;
+
+    strln = strlen(value);
+    if (strln > INT_MAX)
+        strln = INT_MAX;
+
     padlen = min - strln;
-    if (padlen < 0)
+    if (min < 0 || padlen < 0)
         padlen = 0;
     if (flags & DP_F_MINUS)
         padlen = -padlen;
 
     while ((padlen > 0) && (cnt < max)) {
-        doapr_outch(sbuffer, buffer, currlen, maxlen, ' ');
+        if(!doapr_outch(sbuffer, buffer, currlen, maxlen, ' '))
+            return 0;
         --padlen;
         ++cnt;
     }
     while (*value && (cnt < max)) {
-        doapr_outch(sbuffer, buffer, currlen, maxlen, *value++);
+        if(!doapr_outch(sbuffer, buffer, currlen, maxlen, *value++))
+            return 0;
         ++cnt;
     }
     while ((padlen < 0) && (cnt < max)) {
-        doapr_outch(sbuffer, buffer, currlen, maxlen, ' ');
+        if(!doapr_outch(sbuffer, buffer, currlen, maxlen, ' '))
+            return 0;
         ++padlen;
         ++cnt;
     }
+    return 1;
 }
 
-static void
+static int
 fmtint(
     char **sbuffer,
     char **buffer,
@@ -533,37 +550,44 @@
 
     /* spaces */
     while (spadlen > 0) {
-        doapr_outch(sbuffer, buffer, currlen, maxlen, ' ');
+        if(!doapr_outch(sbuffer, buffer, currlen, maxlen, ' '))
+            return 0;
         --spadlen;
     }
 
     /* sign */
     if (signvalue)
-        doapr_outch(sbuffer, buffer, currlen, maxlen, signvalue);
+        if(!doapr_outch(sbuffer, buffer, currlen, maxlen, signvalue))
+            return 0;
 
     /* prefix */
     while (*prefix) {
-	doapr_outch(sbuffer, buffer, currlen, maxlen, *prefix);
+        if(!doapr_outch(sbuffer, buffer, currlen, maxlen, *prefix))
+            return 0;
 	prefix++;
     }
 
     /* zeros */
     if (zpadlen > 0) {
         while (zpadlen > 0) {
-            doapr_outch(sbuffer, buffer, currlen, maxlen, '0');
+            if(!doapr_outch(sbuffer, buffer, currlen, maxlen, '0'))
+                return 0;
             --zpadlen;
         }
     }
     /* digits */
-    while (place > 0)
-        doapr_outch(sbuffer, buffer, currlen, maxlen, convert[--place]);
+    while (place > 0) {
+        if (!doapr_outch(sbuffer, buffer, currlen, maxlen, convert[--place]))
+            return 0;
+    }
 
     /* left justified spaces */
     while (spadlen < 0) {
-        doapr_outch(sbuffer, buffer, currlen, maxlen, ' ');
+        if (!doapr_outch(sbuffer, buffer, currlen, maxlen, ' '))
+            return 0;
         ++spadlen;
     }
-    return;
+    return 1;
 }
 
 static LDOUBLE
@@ -597,7 +621,7 @@
     return intpart;
 }
 
-static void
+static int
 fmtfp(
     char **sbuffer,
     char **buffer,
@@ -616,7 +640,6 @@
     int fplace = 0;
     int padlen = 0;
     int zpadlen = 0;
-    int caps = 0;
     long intpart;
     long fracpart;
     long max10;
@@ -650,9 +673,7 @@
 
     /* convert integer part */
     do {
-        iconvert[iplace++] =
-            (caps ? "0123456789ABCDEF"
-              : "0123456789abcdef")[intpart % 10];
+        iconvert[iplace++] = "0123456789"[intpart % 10];
         intpart = (intpart / 10);
     } while (intpart && (iplace < (int)sizeof(iconvert)));
     if (iplace == sizeof iconvert)
@@ -661,9 +682,7 @@
 
     /* convert fractional part */
     do {
-        fconvert[fplace++] =
-            (caps ? "0123456789ABCDEF"
-              : "0123456789abcdef")[fracpart % 10];
+        fconvert[fplace++] = "0123456789"[fracpart % 10];
         fracpart = (fracpart / 10);
     } while (fplace < max);
     if (fplace == sizeof fconvert)
@@ -682,47 +701,61 @@
 
     if ((flags & DP_F_ZERO) && (padlen > 0)) {
         if (signvalue) {
-            doapr_outch(sbuffer, buffer, currlen, maxlen, signvalue);
+            if (!doapr_outch(sbuffer, buffer, currlen, maxlen, signvalue))
+                return 0;
             --padlen;
             signvalue = 0;
         }
         while (padlen > 0) {
-            doapr_outch(sbuffer, buffer, currlen, maxlen, '0');
+            if (!doapr_outch(sbuffer, buffer, currlen, maxlen, '0'))
+                return 0;
             --padlen;
         }
     }
     while (padlen > 0) {
-        doapr_outch(sbuffer, buffer, currlen, maxlen, ' ');
+        if (!doapr_outch(sbuffer, buffer, currlen, maxlen, ' '))
+            return 0;
         --padlen;
     }
-    if (signvalue)
-        doapr_outch(sbuffer, buffer, currlen, maxlen, signvalue);
+    if (signvalue && !doapr_outch(sbuffer, buffer, currlen, maxlen, signvalue))
+        return 0;
 
-    while (iplace > 0)
-        doapr_outch(sbuffer, buffer, currlen, maxlen, iconvert[--iplace]);
+    while (iplace > 0) {
+        if (!doapr_outch(sbuffer, buffer, currlen, maxlen, iconvert[--iplace]))
+            return 0;
+    }
 
     /*
      * Decimal point. This should probably use locale to find the correct
      * char to print out.
      */
     if (max > 0 || (flags & DP_F_NUM)) {
-        doapr_outch(sbuffer, buffer, currlen, maxlen, '.');
+        if (!doapr_outch(sbuffer, buffer, currlen, maxlen, '.'))
+            return 0;
 
-        while (fplace > 0)
-            doapr_outch(sbuffer, buffer, currlen, maxlen, fconvert[--fplace]);
+        while (fplace > 0) {
+            if(!doapr_outch(sbuffer, buffer, currlen, maxlen,
+                            fconvert[--fplace]))
+                return 0;
+        }
     }
     while (zpadlen > 0) {
-        doapr_outch(sbuffer, buffer, currlen, maxlen, '0');
+        if (!doapr_outch(sbuffer, buffer, currlen, maxlen, '0'))
+            return 0;
         --zpadlen;
     }
 
     while (padlen < 0) {
-        doapr_outch(sbuffer, buffer, currlen, maxlen, ' ');
+        if (!doapr_outch(sbuffer, buffer, currlen, maxlen, ' '))
+            return 0;
         ++padlen;
     }
+    return 1;
 }
 
-static void
+#define BUFFER_INC  1024
+
+static int
 doapr_outch(
     char **sbuffer,
     char **buffer,
@@ -733,24 +766,30 @@
     /* If we haven't at least one buffer, someone has doe a big booboo */
     assert(*sbuffer != NULL || buffer != NULL);
 
-    if (buffer) {
-	while (*currlen >= *maxlen) {
-	    if (*buffer == NULL) {
-		if (*maxlen == 0)
-		    *maxlen = 1024;
+    /* |currlen| must always be <= |*maxlen| */
+    assert(*currlen <= *maxlen);
+
+    if (buffer && *currlen == *maxlen) {
+        if (*maxlen > INT_MAX - BUFFER_INC)
+            return 0;
+
+        *maxlen += BUFFER_INC;
+	if (*buffer == NULL) {
 		*buffer = OPENSSL_malloc(*maxlen);
+		if (*buffer == NULL)
+		    return 0;
 		if (*currlen > 0) {
 		    assert(*sbuffer != NULL);
 		    memcpy(*buffer, *sbuffer, *currlen);
 		}
 		*sbuffer = NULL;
-	    } else {
-		*maxlen += 1024;
-		*buffer = OPENSSL_realloc(*buffer, *maxlen);
-	    }
+	} else {
+            char *tmpbuf;
+            tmpbuf = OPENSSL_realloc(*buffer, *maxlen);
+            if (tmpbuf == NULL)
+                return 0;
+            *buffer = tmpbuf;
 	}
-	/* What to do if *buffer is NULL? */
-	assert(*sbuffer != NULL || *buffer != NULL);
     }
 
     if (*currlen < *maxlen) {
@@ -760,7 +799,7 @@
 	    (*buffer)[(*currlen)++] = (char)c;
     }
 
-    return;
+    return 1;
 }
 
 /***************************************************************************/
@@ -792,11 +831,15 @@
 
 	dynbuf = NULL;
 	CRYPTO_push_info("doapr()");
-	_dopr(&hugebufp, &dynbuf, &hugebufsize,
-		&retlen, &ignored, format, args);
+	if (!_dopr(&hugebufp, &dynbuf, &hugebufsize, &retlen, &ignored, format,
+                args))
+		{
+		OPENSSL_free(dynbuf);
+		return -1;
+		}
 	if (dynbuf)
 		{
-		ret=BIO_write(bio, dynbuf, (int)retlen);
+		ret = BIO_write(bio, dynbuf, (int)retlen);
 		OPENSSL_free(dynbuf);
 		}
 	else
@@ -829,7 +872,8 @@
 	size_t retlen;
 	int truncated;
 
-	_dopr(&buf, NULL, &n, &retlen, &truncated, format, args);
+	if(!_dopr(&buf, NULL, &n, &retlen, &truncated, format, args))
+	    return -1;
 
 	if (truncated)
 		/* In case of truncation, return -1 like traditional snprintf.