record: avoid crash when calling RecordFlushReplyBuffer recursively

Backported from X.Org:

commit 0801afbd7c2c644c672b37f8463f1a0cbadebd2e
Author: Erkki Seppälä <erkki.seppala@vincit.fi>
Date:   Thu Feb 10 15:35:14 2011 +0200

    record: avoid crash when calling RecordFlushReplyBuffer recursively

    RecordFlushReplyBuffer can call itself recursively through
    WriteClient->CallCallbacks->_CallCallbacks->RecordFlushAllContexts
    when the recording client's buffer cannot be completely emptied in one
    WriteClient. When a such a recursion occurs, it will not be broken out
    of which results in segmentation fault when the stack is exhausted.

    This patch adds a counter (a flag, really) that guards against this
    situation, to break out of the recursion.

    One alternative to this change would be to change _CallCallbacks to
    check the corresponding counter before the callback loop, but that
    might affect existing behavior, which may be relied upon.

    Reviewed-by: Rami Ylimäki <rami.ylimaki@vincit.fi>
    Signed-off-by: Erkki Seppälä <erkki.seppala@vincit.fi>
    Signed-off-by: Keith Packard <keithp@keithp.com>

Backported-to-NX-by: Mihai Moldovan <ionic@ionic.de>

Fixes: ArcticaProject/nx-libs#417.

1 files changed, 5 insertions, 1 deletions

diff --git a/nx-X11/programs/Xserver/record/record.c b/nx-X11/programs/Xserver/record/record.c
index df9419f22..48b1964e6 100644
--- a/nx-X11/programs/Xserver/record/record.c
+++ b/nx-X11/programs/Xserver/record/record.c
@@ -74,6 +74,7 @@ typedef struct {
     char	bufCategory;	   /* category of protocol in replyBuffer */
     int		numBufBytes;	   /* number of bytes in replyBuffer */
     char	replyBuffer[REPLY_BUF_SIZE]; /* buffered recorded protocol */
+    int		inFlush;	   /* are we inside RecordFlushReplyBuffer */
 } RecordContextRec, *RecordContextPtr;
 
 /*  RecordMinorOpRec - to hold minor opcode selections for extension requests
@@ -242,8 +243,9 @@ RecordFlushReplyBuffer(
     int len2
 )
 {
-    if (!pContext->pRecordingClient || pContext->pRecordingClient->clientGone) 
+    if (!pContext->pRecordingClient || pContext->pRecordingClient->clientGone || pContext->inFlush)
 	return;
+    ++pContext->inFlush;
     if (pContext->numBufBytes)
 	WriteToClient(pContext->pRecordingClient, pContext->numBufBytes,
 		      (char *)pContext->replyBuffer);
@@ -252,6 +254,7 @@ RecordFlushReplyBuffer(
 	WriteToClient(pContext->pRecordingClient, len1, data1);
     if (len2)
 	WriteToClient(pContext->pRecordingClient, len2, data2);
+    --pContext->inFlush;
 } /* RecordFlushReplyBuffer */
 
 
@@ -2039,6 +2042,7 @@ ProcRecordCreateContext(client)
     pContext->numBufBytes = 0;
     pContext->pBufClient = NULL;
     pContext->continuedReply = 0;
+    pContext->inFlush = 0;
 
     err = RecordRegisterClients(pContext, client,
 				(xRecordRegisterClientsReq *)stuff);