LibVNCClient: Add H.264 encoding for framebuffer updates

This patch implements support in LibVNCClient for framebuffer updates
encoded as H.264 frames. Hardware accelerated decoding is performed
using VA API.

This is experimental support to let the community explore the possibilities
offered by the potential bandwidth and latency reductions that H.264 encoding
allows. This may be particularly useful for use cases such as online gaming,
hosted desktops, hosted set top boxes...

This patch only provides the client side support and is meant to be used
with corresponding server-side support, as provided by an upcoming patch for
qemu ui/vnc module (to view the display of a virtual machine executing under
QEMU).

With this H.264-based encoding, if multiple framebuffer update messages
are generated for a single server framebuffer modification, the H.264
frame data is sent only with the first update message. Subsequent update
framebuffer messages will contain only the coordinates and size of the
additional updated regions.

Instructions/Requirements:
* The patch should be applied on top of the previous patch I submitted with
minor enhancements to the gtkvncviewer application:
http://sourceforge.net/mailarchive/message.php?msg_id=30323804
* Currently only works with libva 1.0: use branch "v1.0-branch" for libva and
intel-driver. Those can be built as follows:
   cd libva
   git checkout v1.0-branch
   ./autogen.sh
   make
   sudo make install
   cd ..
   git clone git://anongit.freedesktop.org/vaapi/intel-driver
   cd intel-driver
   git checkout v1.0-branch
   ./autogen.sh
   make
   sudo make install

Signed-off-by: David Verbeiren <david.verbeiren@intel.com>

1 files changed, 24 insertions, 0 deletions

diff --git a/libvncclient/rfbproto.c b/libvncclient/rfbproto.c
index 41afc6c..958780c 100644
--- a/libvncclient/rfbproto.c
+++ b/libvncclient/rfbproto.c
@@ -158,6 +158,10 @@ static void FillRectangle(rfbClient* client, int x, int y, int w, int h, uint32_
 static void CopyRectangle(rfbClient* client, uint8_t* buffer, int x, int y, int w, int h) {
   int j;
 
+  if (client->frameBuffer == NULL) {
+      return;
+  }
+
 #define COPY_RECT(BPP) \
   { \
     int rs = w * BPP / 8, rs2 = client->width * BPP / 8; \
@@ -260,6 +264,9 @@ static rfbBool HandleZRLE24Up(rfbClient* client, int rx, int ry, int rw, int rh)
 static rfbBool HandleZRLE24Down(rfbClient* client, int rx, int ry, int rw, int rh);
 static rfbBool HandleZRLE32(rfbClient* client, int rx, int ry, int rw, int rh);
 #endif
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+static rfbBool HandleH264 (rfbClient* client, int rx, int ry, int rw, int rh);
+#endif
 
 /*
  * Server Capability Functions
@@ -1344,6 +1351,10 @@ SetFormatAndEncodings(rfbClient* client)
 	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingCoRRE);
       } else if (strncasecmp(encStr,"rre",encStrLen) == 0) {
 	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingRRE);
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+      } else if (strncasecmp(encStr,"h264",encStrLen) == 0) {
+	encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingH264);
+#endif
       } else {
 	rfbClientLog("Unknown encoding '%.*s'\n",encStrLen,encStr);
       }
@@ -1412,6 +1423,10 @@ SetFormatAndEncodings(rfbClient* client)
       encs[se->nEncodings++] = rfbClientSwap32IfLE(client->appData.qualityLevel +
 					  rfbEncodingQualityLevel0);
     }
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+    encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingH264);
+    rfbClientLog("h264 encoding added\n");
+#endif
   }
 
 
@@ -2128,6 +2143,14 @@ HandleRFBServerMessage(rfbClient* client)
      }
 
 #endif
+#ifdef LIBVNCSERVER_CONFIG_LIBVA
+      case rfbEncodingH264:
+      {
+	if (!HandleH264(client, rect.r.x, rect.r.y, rect.r.w, rect.r.h))
+	  return FALSE;
+	break;
+      }
+#endif
 
       default:
 	 {
@@ -2361,6 +2384,7 @@ HandleRFBServerMessage(rfbClient* client)
 #define UNCOMP -8
 #include "zrle.c"
 #undef BPP
+#include "h264.c"
 
 
 /*