diff options
author | dscho <johannes.schindelin@gmx.de> | 2014-08-16 10:46:36 +0200 |
---|---|---|
committer | dscho <johannes.schindelin@gmx.de> | 2014-08-16 10:46:36 +0200 |
commit | ba710eb145bd2a9bb14bc316eb6ff645f004b06c (patch) | |
tree | 67f488e08d6bd6a89390c2840d96dd6ca909bcae /libvncclient | |
parent | 9453be42014b6b9eaa6ab6f8c2e8b6f4f01d15aa (diff) | |
parent | 85a778c0e45e87e35ee7199f1f25020648e8b812 (diff) | |
download | libtdevnc-ba710eb145bd2a9bb14bc316eb6ff645f004b06c.tar.gz libtdevnc-ba710eb145bd2a9bb14bc316eb6ff645f004b06c.zip |
Merge pull request #20 from newsoft/master
Fix integer overflow in MallocFrameBuffer()
Diffstat (limited to 'libvncclient')
-rw-r--r-- | libvncclient/rfbproto.c | 10 | ||||
-rw-r--r-- | libvncclient/vncviewer.c | 23 |
2 files changed, 28 insertions, 5 deletions
diff --git a/libvncclient/rfbproto.c b/libvncclient/rfbproto.c index b4d7156..f55c74f 100644 --- a/libvncclient/rfbproto.c +++ b/libvncclient/rfbproto.c @@ -1829,7 +1829,8 @@ HandleRFBServerMessage(rfbClient* client) client->updateRect.x = client->updateRect.y = 0; client->updateRect.w = client->width; client->updateRect.h = client->height; - client->MallocFrameBuffer(client); + if (!client->MallocFrameBuffer(client)) + return FALSE; SendFramebufferUpdateRequest(client, 0, 0, rect.r.w, rect.r.h, FALSE); rfbClientLog("Got new framebuffer size: %dx%d\n", rect.r.w, rect.r.h); continue; @@ -2290,7 +2291,9 @@ HandleRFBServerMessage(rfbClient* client) client->updateRect.x = client->updateRect.y = 0; client->updateRect.w = client->width; client->updateRect.h = client->height; - client->MallocFrameBuffer(client); + if (!client->MallocFrameBuffer(client)) + return FALSE; + SendFramebufferUpdateRequest(client, 0, 0, client->width, client->height, FALSE); rfbClientLog("Got new framebuffer size: %dx%d\n", client->width, client->height); break; @@ -2306,7 +2309,8 @@ HandleRFBServerMessage(rfbClient* client) client->updateRect.x = client->updateRect.y = 0; client->updateRect.w = client->width; client->updateRect.h = client->height; - client->MallocFrameBuffer(client); + if (!client->MallocFrameBuffer(client)) + return FALSE; SendFramebufferUpdateRequest(client, 0, 0, client->width, client->height, FALSE); rfbClientLog("Got new framebuffer size: %dx%d\n", client->width, client->height); break; diff --git a/libvncclient/vncviewer.c b/libvncclient/vncviewer.c index 3b16a6f..65b7412 100644 --- a/libvncclient/vncviewer.c +++ b/libvncclient/vncviewer.c @@ -82,9 +82,27 @@ static char* ReadPassword(rfbClient* client) { #endif } static rfbBool MallocFrameBuffer(rfbClient* client) { +uint64_t allocSize; + if(client->frameBuffer) free(client->frameBuffer); - client->frameBuffer=malloc(client->width*client->height*client->format.bitsPerPixel/8); + + /* SECURITY: promote 'width' into uint64_t so that the multiplication does not overflow + 'width' and 'height' are 16-bit integers per RFB protocol design + SIZE_MAX is the maximum value that can fit into size_t + */ + allocSize = (uint64_t)client->width * client->height * client->format.bitsPerPixel/8; + + if (allocSize >= SIZE_MAX) { + rfbClientErr("CRITICAL: cannot allocate frameBuffer, requested size is too large\n"); + return FALSE; + } + + client->frameBuffer=malloc( (size_t)allocSize ); + + if (client->frameBuffer == NULL) + rfbClientErr("CRITICAL: frameBuffer allocation failed, requested size too large or not enough memory?\n"); + return client->frameBuffer?TRUE:FALSE; } @@ -232,7 +250,8 @@ static rfbBool rfbInitConnection(rfbClient* client) client->width=client->si.framebufferWidth; client->height=client->si.framebufferHeight; - client->MallocFrameBuffer(client); + if (!client->MallocFrameBuffer(client)) + return FALSE; if (!SetFormatAndEncodings(client)) return FALSE; |