Compext.c: fix comparisons of 16bit sequence numbers

rep->generic.sequenceNumber is of type CARD16
state->sequence is of type unsigned long

Converting state->sequence to an int as it has been done since the
first version of nxcomp I know of (1.3.0-18 from 2003) is wrong here
because for numbers > INT_MAX this will result in a negative number,
which, after applying the 16bit modulo, will not match
rep->generic.sequenceNumber.

Example with numbers:

CARD16 c = 24565
unsigned long u = 3179110389

c % 65536 = 24565
u % 65536 = 24565

(int)(u) = -1115856907
(int)(u) % 65536 = -40971

-40971 will not match 24565

To fix this we need to ensure the number stays positive. We use CARD16
for this to match the type in the request which is a 16bit number. On
my system CARD16 is unsigned short which is guaranteed to contain _at
least_ the 0-65,535 range. As there is no upper limit of the range we
cannot drop the modulo because we need this value to be 16bit and not
more.

Thanks to Norm Green for providing log after log until we could
finally identify the reason for him seeing "Xlib: unexpected async
reply (sequence 0x94b01439)!" when pasting stopped working.

1 files changed, 12 insertions, 12 deletions

diff --git a/nx-X11/programs/Xserver/hw/nxagent/compext/Compext.c b/nx-X11/programs/Xserver/hw/nxagent/compext/Compext.c
index 7e5a9799d..dcb91348b 100644
--- a/nx-X11/programs/Xserver/hw/nxagent/compext/Compext.c
+++ b/nx-X11/programs/Xserver/hw/nxagent/compext/Compext.c
@@ -3451,10 +3451,10 @@ static Bool _NXCollectImageHandler(Display *dpy, xReply *rep, char *buf,
   state = (_NXCollectImageState *) data;
 
   #ifdef DEBUG_IMAGE
-  fprintf(stderr, "******%s: sequence: received [%ld] expected [%ld]  16bit received [%d] expected [%d]\n",
+  fprintf(stderr, "******%s: sequence: received [%hu] expected [%lu]  16bit received [%hu] expected [%hu]\n",
           __func__,
-          (long) rep -> generic.sequenceNumber, (long)state -> sequence,
-          rep -> generic.sequenceNumber % 65536, (int)(state -> sequence) % 65536);
+          rep -> generic.sequenceNumber, state -> sequence,
+          rep -> generic.sequenceNumber % 65536, (CARD16)(state -> sequence) % 65536);
   #endif
 
   if ((rep -> generic.sequenceNumber % 65536) !=
@@ -3846,10 +3846,10 @@ static Bool _NXCollectPropertyHandler(Display *dpy, xReply *rep, char *buf,
   state = (_NXCollectPropertyState *) data;
 
   #ifdef DEBUG_PROPERTY
-  fprintf(stderr, "******%s: sequence: received [%ld] expected [%ld]  16bit received [%d] expected [%d]\n",
+  fprintf(stderr, "******%s: sequence: received [%hu] expected [%lu]  16bit received [%hu] expected [%hu]\n",
           __func__,
-          (long) rep -> generic.sequenceNumber, (long)state -> sequence,
-          rep -> generic.sequenceNumber % 65536, (int)(state -> sequence) % 65536);
+          rep -> generic.sequenceNumber, state -> sequence,
+          rep -> generic.sequenceNumber % 65536, (CARD16)(state -> sequence) % 65536);
   #endif
 
   if ((rep -> generic.sequenceNumber % 65536) !=
@@ -4219,10 +4219,10 @@ static Bool _NXCollectGrabPointerHandler(Display *dpy, xReply *rep, char *buf,
   state = (_NXCollectGrabPointerState *) data;
 
   #ifdef DEBUG_POINTER
-  fprintf(stderr, "******%s: sequence: received [%ld] expected [%ld]  16bit received [%d] expected [%d]\n",
+  fprintf(stderr, "******%s: sequence: received [%hu] expected [%lu]  16bit received [%hu] expected [%hu]\n",
           __func__,
-          (long) rep -> generic.sequenceNumber, (long)state -> sequence,
-          rep -> generic.sequenceNumber % 65536, (int)(state -> sequence) % 65536);
+          rep -> generic.sequenceNumber, state -> sequence,
+          rep -> generic.sequenceNumber % 65536, (CARD16)(state -> sequence) % 65536);
   #endif
 
   if ((rep -> generic.sequenceNumber % 65536) !=
@@ -4511,10 +4511,10 @@ static Bool _NXCollectInputFocusHandler(Display *dpy, xReply *rep, char *buf,
   state = (_NXCollectInputFocusState *) data;
 
   #ifdef DEBUG_INPUT
-  fprintf(stderr, "******%s: sequence: received [%ld] expected [%ld]  16bit received [%d] expected [%d]\n",
+  fprintf(stderr, "******%s: sequence: received [%hu] expected [%lu]  16bit received [%hu] expected [%hu]\n",
           __func__,
-          (long) rep -> generic.sequenceNumber, (long)state -> sequence,
-          rep -> generic.sequenceNumber % 65536, (int)(state -> sequence) % 65536);
+          rep -> generic.sequenceNumber, state -> sequence,
+          rep -> generic.sequenceNumber % 65536, (CARD16)(state -> sequence) % 65536);
   #endif
 
   if ((rep -> generic.sequenceNumber % 65536) !=