glx: Be more strict about rejecting invalid image sizes [CVE-2014-8093 2/6]

Before this we'd just clamp the image size to 0, which was just
hideously stupid; if the parameters were such that they'd overflow an
integer, you'd allocate a small buffer, then pass huge values into (say)
ReadPixels, and now you're scribbling over arbitrary server memory.

v2: backport to nx-libs 3.6.x (Mike DePaulo)

Reviewed-by: Keith Packard <keithp@keithp.com>
Reviewed-by: Julien Cristau <jcristau@debian.org>
Reviewed-by: Michal Srb <msrb@suse.com>
Reviewed-by: Andy Ritger <aritger@nvidia.com>
Signed-off-by: Adam Jackson <ajax@redhat.com>
Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>
Signed-off-by: Fedora X Ninjas <x@fedoraproject.org>
Signed-off-by: Dave Airlie <airlied@redhat.com>

2 files changed, 14 insertions, 14 deletions

diff --git a/nx-X11/programs/Xserver/GL/glx/singlepix.c b/nx-X11/programs/Xserver/GL/glx/singlepix.c
index 845c46a64..be804d82e 100644
--- a/nx-X11/programs/Xserver/GL/glx/singlepix.c
+++ b/nx-X11/programs/Xserver/GL/glx/singlepix.c
@@ -70,7 +70,7 @@ int __glXDisp_ReadPixels(__GLXclientState *cl, GLbyte *pc)
     swapBytes = *(GLboolean *)(pc + 24);
     lsbFirst = *(GLboolean *)(pc + 25);
     compsize = __glReadPixels_size(format,type,width,height);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     glPixelStorei(GL_PACK_LSB_FIRST, lsbFirst);
@@ -130,7 +130,7 @@ int __glXDisp_GetTexImage(__GLXclientState *cl, GLbyte *pc)
      * are illegal, but then width, height, and depth would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,level,format,type,width,height,depth);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -227,7 +227,7 @@ int __glXDisp_GetSeparableFilter(__GLXclientState *cl, GLbyte *pc)
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
     compsize2 = __glGetTexImage_size(target,1,format,type,height,1,1);
 
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
     if (compsize2 < 0) compsize2 = 0;
     compsize = __GLX_PAD(compsize);
     compsize2 = __GLX_PAD(compsize2);
@@ -291,7 +291,7 @@ int __glXDisp_GetConvolutionFilter(__GLXclientState *cl, GLbyte *pc)
      * are illegal, but then width and height would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,height,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -346,7 +346,7 @@ int __glXDisp_GetHistogram(__GLXclientState *cl, GLbyte *pc)
      * are illegal, but then width would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -389,7 +389,7 @@ int __glXDisp_GetMinmax(__GLXclientState *cl, GLbyte *pc)
     reset = *(GLboolean *)(pc + 13);
 
     compsize = __glGetTexImage_size(target,1,format,type,2,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -436,7 +436,7 @@ int __glXDisp_GetColorTable(__GLXclientState *cl, GLbyte *pc)
      * are illegal, but then width would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
diff --git a/nx-X11/programs/Xserver/GL/glx/singlepixswap.c b/nx-X11/programs/Xserver/GL/glx/singlepixswap.c
index ff68ecedf..cdc6f164e 100644
--- a/nx-X11/programs/Xserver/GL/glx/singlepixswap.c
+++ b/nx-X11/programs/Xserver/GL/glx/singlepixswap.c
@@ -79,7 +79,7 @@ int __glXDispSwap_ReadPixels(__GLXclientState *cl, GLbyte *pc)
     swapBytes = *(GLboolean *)(pc + 24);
     lsbFirst = *(GLboolean *)(pc + 25);
     compsize = __glReadPixels_size(format,type,width,height);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     glPixelStorei(GL_PACK_LSB_FIRST, lsbFirst);
@@ -148,7 +148,7 @@ int __glXDispSwap_GetTexImage(__GLXclientState *cl, GLbyte *pc)
      * are illegal, but then width, height, and depth would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,level,format,type,width,height,depth);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -257,7 +257,7 @@ int __glXDispSwap_GetSeparableFilter(__GLXclientState *cl, GLbyte *pc)
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
     compsize2 = __glGetTexImage_size(target,1,format,type,height,1,1);
 
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
     if (compsize2 < 0) compsize2 = 0;
     compsize = __GLX_PAD(compsize);
     compsize2 = __GLX_PAD(compsize2);
@@ -328,7 +328,7 @@ int __glXDispSwap_GetConvolutionFilter(__GLXclientState *cl, GLbyte *pc)
      * are illegal, but then width and height would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,height,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -390,7 +390,7 @@ int __glXDispSwap_GetHistogram(__GLXclientState *cl, GLbyte *pc)
      * are illegal, but then width would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -439,7 +439,7 @@ int __glXDispSwap_GetMinmax(__GLXclientState *cl, GLbyte *pc)
     reset = *(GLboolean *)(pc + 13);
 
     compsize = __glGetTexImage_size(target,1,format,type,2,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);
@@ -491,7 +491,7 @@ int __glXDispSwap_GetColorTable(__GLXclientState *cl, GLbyte *pc)
      * are illegal, but then width would still be zero anyway.
      */
     compsize = __glGetTexImage_size(target,1,format,type,width,1,1);
-    if (compsize < 0) compsize = 0;
+    if (compsize < 0) return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer,cl,compsize,1);