integer overflow in XListHosts() [CVE-2013-1981 5/13]

If the reported number of host entries is too large, the calculations
to allocate memory for them may overflow, leaving us writing beyond the
bounds of the allocation.

Reported-by: Ilja Van Sprundel <ivansprundel@ioactive.com>
Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>
Reviewed-by: Matthieu Herrb <matthieu.herrb@laas.fr>
Signed-off-by: Julien Cristau <jcristau@debian.org>
Backported-to-NX-by: Ulrich Sibiller <uli42@gmx.de>

1 files changed, 15 insertions, 7 deletions

diff --git a/nx-X11/lib/X11/LiHosts.c b/nx-X11/lib/X11/LiHosts.c
index 63ecc508a..6ec956e72 100644
--- a/nx-X11/lib/X11/LiHosts.c
+++ b/nx-X11/lib/X11/LiHosts.c
@@ -62,6 +62,8 @@ X Window System is a trademark of The Open Group.
 #include <config.h>
 #endif
 #include "Xlibint.h"
+#include <limits.h>
+
 /*
  * can be freed using XFree.
  */
@@ -73,7 +75,6 @@ XHostAddress *XListHosts (
 {
     register XHostAddress *outbuf = NULL, *op;
     xListHostsReply reply;
-    long nbytes;
     unsigned char *buf, *bp;
     register unsigned i;
     register xListHostsReq *req;
@@ -90,19 +91,26 @@ XHostAddress *XListHosts (
     }
 
     if (reply.nHosts) {
-	nbytes = reply.length << 2;	/* compute number of bytes in reply */
+	unsigned long nbytes = reply.length << 2; /* number of bytes in reply */
+	const unsigned long max_hosts = INT_MAX /
+	    (sizeof(XHostAddress) + sizeof(XServerInterpretedAddress));
+
+	if (reply.nHosts < max_hosts) {
+	    unsigned long hostbytes = reply.nHosts *
+		(sizeof(XHostAddress) + sizeof(XServerInterpretedAddress));
 
-	op = outbuf = (XHostAddress *)
-	    Xmalloc((unsigned) (nbytes +
-	      (reply.nHosts * sizeof(XHostAddress)) +
-	      (reply.nHosts * sizeof(XServerInterpretedAddress))));
+	    if (reply.length < (INT_MAX >> 2) &&
+		(hostbytes >> 2) < ((INT_MAX >> 2) - reply.length))
+		outbuf = Xmalloc(nbytes + hostbytes);
+	}
 
 	if (! outbuf) {
-	    _XEatData(dpy, (unsigned long) nbytes);
+	    _XEatDataWords(dpy, reply.length);
 	    UnlockDisplay(dpy);
 	    SyncHandle();
 	    return (XHostAddress *) NULL;
 	}
+	op = outbuf;
 	sip = (XServerInterpretedAddress *)
 	 (((unsigned char  *) outbuf) + (reply.nHosts * sizeof(XHostAddress)));
 	bp = buf = ((unsigned char  *) sip)