Roll back "die on malloc failure" behaviour a bit

After recent change, select(2^31-1, NULL, NULL, NULL)
would make strace exit. This change caps fdsize so that
it is always in [0, 1025*1024], IOW: we will try to allocate at most
1 megabyte, which in practice will almost always work,
unlike malloc(2Gig).

* desc.c (decode_select): Cap fdsize to 1024*1024.
* pathtrace.c (pathtrace_match): Cap fdsize to 1024*1024.
* file.c (sys_getdents): Cap len to 1024*1024.
(sys_getdents64): Cap len to 1024*1024.
* util.c (dumpiov): Refuse to process iov with more than 1024*1024
elements. Don't die on malloc failure.
(dumpstr): Don't die on malloc failure.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
diff --git a/desc.c b/desc.c
index 9a85866..c0e8ca9 100644
--- a/desc.c
+++ b/desc.c
@@ -490,12 +490,19 @@
 decode_select(struct tcb *tcp, long *args, enum bitness_t bitness)
 {
 	int i, j, nfds;
-	unsigned int fdsize = ((((args[0] + 7) / 8) + sizeof(long) - 1)
-			       & -sizeof(long));
+	unsigned nfds, fdsize;
 	fd_set *fds;
 	const char *sep;
 	long arg;
 
+	fdsize = args[0];
+	/* Beware of select(2^31-1, NULL, NULL, NULL) and similar... */
+	if (args[0] > 1024*1024)
+		fdsize = 1024*1024;
+	if (args[0] < 0)
+		fdsize = 0;
+	fdsize = (((fdsize + 7) / 8) + sizeof(long)-1) & -sizeof(long);
+
 	if (entering(tcp)) {
 		fds = malloc(fdsize);
 		if (!fds)
diff --git a/file.c b/file.c
index 96e36a2..b1d97c7 100644
--- a/file.c
+++ b/file.c
@@ -2421,6 +2421,11 @@
 		return 0;
 	}
 	len = tcp->u_rval;
+	/* Beware of insanely large or negative values in tcp->u_rval */
+	if (tcp->u_rval > 1024*1024)
+		len = 1024*1024;
+	if (tcp->u_rval < 0)
+		len = 0;
 	buf = len ? malloc(len) : NULL;
 	if (len && !buf)
 		die_out_of_memory();
@@ -2502,10 +2507,17 @@
 		tprintf("%#lx, %lu", tcp->u_arg[1], tcp->u_arg[2]);
 		return 0;
 	}
+
 	len = tcp->u_rval;
+	/* Beware of insanely large or negative tcp->u_rval */
+	if (tcp->u_rval > 1024*1024)
+		len = 1024*1024;
+	if (tcp->u_rval < 0)
+		len = 0;
 	buf = len ? malloc(len) : NULL;
 	if (len && !buf)
 		die_out_of_memory();
+
 	if (umoven(tcp, tcp->u_arg[1], len, buf) < 0) {
 		tprintf("%#lx, %lu", tcp->u_arg[1], tcp->u_arg[2]);
 		free(buf);
@@ -2573,10 +2585,17 @@
 		tprintf("%#lx, %lu, %#lx", tcp->u_arg[1], tcp->u_arg[2], tcp->u_arg[3]);
 		return 0;
 	}
+
 	len = tcp->u_rval;
+	/* Beware of insanely large or negative tcp->u_rval */
+	if (tcp->u_rval > 1024*1024)
+		len = 1024*1024;
+	if (tcp->u_rval < 0)
+		len = 0;
 	buf = malloc(len);
 	if (!buf)
 		die_out_of_memory();
+
 	if (umoven(tcp, tcp->u_arg[1], len, buf) < 0) {
 		tprintf("%#lx, %lu, %#lx", tcp->u_arg[1], tcp->u_arg[2], tcp->u_arg[3]);
 		free(buf);
diff --git a/pathtrace.c b/pathtrace.c
index 23ab47e..380363c 100644
--- a/pathtrace.c
+++ b/pathtrace.c
@@ -269,7 +269,8 @@
 	    s->sys_func == sys_oldselect ||
 	    s->sys_func == sys_pselect6)
 	{
-		int     i, j, nfds;
+		int     i, j;
+		unsigned nfds;
 		long   *args, oldargs[5];
 		unsigned fdsize;
 		fd_set *fds;
@@ -286,10 +287,14 @@
 			args = tcp->u_arg;
 
 		nfds = args[0];
+		/* Beware of select(2^31-1, NULL, NULL, NULL) and similar... */
+		if (args[0] > 1024*1024)
+			nfds = 1024*1024;
+		if (args[0] < 0)
+			nfds = 0;
 		fdsize = ((((nfds + 7) / 8) + sizeof(long) - 1)
 			  & -sizeof(long));
 		fds = malloc(fdsize);
-
 		if (!fds)
 			die_out_of_memory();
 
diff --git a/util.c b/util.c
index 7d1af70..20311fc 100644
--- a/util.c
+++ b/util.c
@@ -686,12 +686,14 @@
 #define iov_iov_len(i) iov[i].iov_len
 #endif
 	int i;
-	unsigned long size;
+	unsigned size;
 
-	size = sizeof_iov * (unsigned long) len;
-	if (size / sizeof_iov != len /* overflow? */
+	size = sizeof_iov * len;
+	/* Assuming no sane program has millions of iovs */
+	if ((unsigned)len > 1024*1024 /* insane or negative size? */
 	    || (iov = malloc(size)) == NULL) {
-		die_out_of_memory();
+		fprintf(stderr, "Out of memory\n");
+		return;
 	}
 	if (umoven(tcp, addr, size, (char *) iov) >= 0) {
 		for (i = 0; i < len; i++) {
@@ -703,7 +705,7 @@
 				iov_iov_len(i));
 		}
 	}
-	free((char *) iov);
+	free(iov);
 #undef sizeof_iov
 #undef iov_iov_base
 #undef iov_iov_len
@@ -722,8 +724,11 @@
 	if (strsize < len) {
 		free(str);
 		str = malloc(len);
-		if (!str)
-			die_out_of_memory();
+		if (!str) {
+			strsize = -1;
+			fprintf(stderr, "Out of memory\n");
+			return;
+		}
 		strsize = len;
 	}