Merge pull request #161 from jlesage/master-base64-compat

websockets: Fixed compilation of websockets on systems where there is no implementation of base64 functions.

6 files changed, 335 insertions, 34 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 75d6470..16f235e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -174,28 +174,6 @@ check_function_exists(strdup          LIBVNCSERVER_HAVE_STRDUP)
 check_function_exists(strerror        LIBVNCSERVER_HAVE_STRERROR)
 check_function_exists(strstr          LIBVNCSERVER_HAVE_STRSTR)
 
-# On systems such as GNU/Linux with glibc, __b64_ntop is defined in a
-# separate library, libresolv. On some others, such as FreeBSD, it is
-# part of libc itself. We first check if __b64_ntop is found without
-# additional libraries, and then try looking for it with libresolv if
-# the first test fails.
-check_function_exists(__b64_ntop HAVE_B64_IN_LIBC)
-if(NOT HAVE_B64_IN_LIBC)
-    set(CMAKE_REQUIRED_LIBRARIES resolv)
-    check_function_exists(__b64_ntop HAVE_B64_IN_LIBRESOLV)
-    set(CMAKE_REQUIRED_LIBRARIES)
-
-    if(HAVE_B64_IN_LIBRESOLV)
-        set(RESOLV_LIB "resolv")
-    endif(HAVE_B64_IN_LIBRESOLV)
-
-    # the function check somehow fails for apple but the function is there
-    if(APPLE)
-        set(RESOLV_LIB "resolv")
-    endif(APPLE)
-
-endif(NOT HAVE_B64_IN_LIBC)
-
 if(Threads_FOUND)
   set(ADDITIONAL_LIBS ${ADDITIONAL_LIBS} ${CMAKE_THREAD_LIBS_INIT})
 endif(Threads_FOUND)
@@ -225,16 +203,16 @@ if(WITH_WEBSOCKETS AND LIBVNCSERVER_HAVE_SYS_UIO_H)
   if(GNUTLS_FOUND)
     set(LIBVNCSERVER_WITH_CLIENT_TLS 1)
     message(STATUS "Building websockets with GnuTLS")
-    set(WEBSOCKET_LIBRARIES ${RESOLV_LIB} ${GNUTLS_LIBRARIES})
+    set(WEBSOCKET_LIBRARIES ${GNUTLS_LIBRARIES})
     set(WSSRCS ${LIBVNCSERVER_DIR}/rfbssl_gnutls ${LIBVNCSERVER_DIR}/rfbcrypto_gnutls)
     include_directories(${GNUTLS_INCLUDE_DIR})
   elseif(OPENSSL_FOUND)
     message(STATUS "Building websockets with OpenSSL")
-    set(WEBSOCKET_LIBRARIES ${RESOLV_LIB} ${OPENSSL_LIBRARIES})
+    set(WEBSOCKET_LIBRARIES ${OPENSSL_LIBRARIES})
     set(WSSRCS ${LIBVNCSERVER_DIR}/rfbssl_openssl ${LIBVNCSERVER_DIR}/rfbcrypto_openssl)
   else()
     message(STATUS "Building websockets without SSL")
-    set(WEBSOCKET_LIBRARIES ${RESOLV_LIB})
+    set(WEBSOCKET_LIBRARIES)
     set(WSSRCS ${LIBVNCSERVER_DIR}/rfbssl_none.c ${LIBVNCSERVER_DIR}/rfbcrypto_included.c ${COMMON_DIR}/md5.c ${COMMON_DIR}/sha1.c)
   endif()
 endif(WITH_WEBSOCKETS AND LIBVNCSERVER_HAVE_SYS_UIO_H)
@@ -388,11 +366,11 @@ if(LIBVNCSERVER_WITH_WEBSOCKETS)
     ${LIBVNCSERVER_SOURCES}
     ${LIBVNCSERVER_DIR}/websockets.c
     ${LIBVNCSERVER_DIR}/ws_decode.c
+    ${COMMON_DIR}/base64.c
     ${WSSRCS}
   )
 endif(LIBVNCSERVER_WITH_WEBSOCKETS)
 
-
 add_library(vncclient ${LIBVNCCLIENT_SOURCES})
 add_library(vncserver ${LIBVNCSERVER_SOURCES})
 if(WIN32)
diff --git a/common/base64.c b/common/base64.c
new file mode 100644
index 0000000..4e3685a
--- /dev/null
+++ b/common/base64.c
@@ -0,0 +1,315 @@
+/*	$OpenBSD: base64.c,v 1.8 2015/01/16 16:48:51 deraadt Exp $	*/
+
+/*
+ * Copyright (c) 1996 by Internet Software Consortium.
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SOFTWARE CONSORTIUM DISCLAIMS
+ * ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL INTERNET SOFTWARE
+ * CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
+ * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
+ * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+ * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+
+/*
+ * Portions Copyright (c) 1995 by International Business Machines, Inc.
+ *
+ * International Business Machines, Inc. (hereinafter called IBM) grants
+ * permission under its copyrights to use, copy, modify, and distribute this
+ * Software with or without fee, provided that the above copyright notice and
+ * all paragraphs of this notice appear in all copies, and that the name of IBM
+ * not be used in connection with the marketing of any product incorporating
+ * the Software or modifications thereof, without specific, written prior
+ * permission.
+ *
+ * To the extent it has a right to do so, IBM grants an immunity from suit
+ * under its patents, if any, for the use, sale or manufacture of products to
+ * the extent that such products are used for performing Domain Name System
+ * dynamic updates in TCP/IP networks by means of the Software.  No immunity is
+ * granted for any product per se or for any other function of any product.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", AND IBM DISCLAIMS ALL WARRANTIES,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE.  IN NO EVENT SHALL IBM BE LIABLE FOR ANY SPECIAL,
+ * DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE, EVEN
+ * IF IBM IS APPRISED OF THE POSSIBILITY OF SUCH DAMAGES.
+ */
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <arpa/nameser.h>
+
+#include <ctype.h>
+#include <resolv.h>
+#include <stdio.h>
+
+#include <stdlib.h>
+#include <string.h>
+
+static const char Base64[] =
+	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+static const char Pad64 = '=';
+
+/* (From RFC1521 and draft-ietf-dnssec-secext-03.txt)
+   The following encoding technique is taken from RFC 1521 by Borenstein
+   and Freed.  It is reproduced here in a slightly edited form for
+   convenience.
+
+   A 65-character subset of US-ASCII is used, enabling 6 bits to be
+   represented per printable character. (The extra 65th character, "=",
+   is used to signify a special processing function.)
+
+   The encoding process represents 24-bit groups of input bits as output
+   strings of 4 encoded characters. Proceeding from left to right, a
+   24-bit input group is formed by concatenating 3 8-bit input groups.
+   These 24 bits are then treated as 4 concatenated 6-bit groups, each
+   of which is translated into a single digit in the base64 alphabet.
+
+   Each 6-bit group is used as an index into an array of 64 printable
+   characters. The character referenced by the index is placed in the
+   output string.
+
+                         Table 1: The Base64 Alphabet
+
+      Value Encoding  Value Encoding  Value Encoding  Value Encoding
+          0 A            17 R            34 i            51 z
+          1 B            18 S            35 j            52 0
+          2 C            19 T            36 k            53 1
+          3 D            20 U            37 l            54 2
+          4 E            21 V            38 m            55 3
+          5 F            22 W            39 n            56 4
+          6 G            23 X            40 o            57 5
+          7 H            24 Y            41 p            58 6
+          8 I            25 Z            42 q            59 7
+          9 J            26 a            43 r            60 8
+         10 K            27 b            44 s            61 9
+         11 L            28 c            45 t            62 +
+         12 M            29 d            46 u            63 /
+         13 N            30 e            47 v
+         14 O            31 f            48 w         (pad) =
+         15 P            32 g            49 x
+         16 Q            33 h            50 y
+
+   Special processing is performed if fewer than 24 bits are available
+   at the end of the data being encoded.  A full encoding quantum is
+   always completed at the end of a quantity.  When fewer than 24 input
+   bits are available in an input group, zero bits are added (on the
+   right) to form an integral number of 6-bit groups.  Padding at the
+   end of the data is performed using the '=' character.
+
+   Since all base64 input is an integral number of octets, only the
+         -------------------------------------------------
+   following cases can arise:
+
+       (1) the final quantum of encoding input is an integral
+           multiple of 24 bits; here, the final unit of encoded
+	   output will be an integral multiple of 4 characters
+	   with no "=" padding,
+       (2) the final quantum of encoding input is exactly 8 bits;
+           here, the final unit of encoded output will be two
+	   characters followed by two "=" padding characters, or
+       (3) the final quantum of encoding input is exactly 16 bits;
+           here, the final unit of encoded output will be three
+	   characters followed by one "=" padding character.
+   */
+
+int
+__b64_ntop(src, srclength, target, targsize)
+	u_char const *src;
+	size_t srclength;
+	char *target;
+	size_t targsize;
+{
+	size_t datalength = 0;
+	u_char input[3];
+	u_char output[4];
+	int i;
+
+	while (2 < srclength) {
+		input[0] = *src++;
+		input[1] = *src++;
+		input[2] = *src++;
+		srclength -= 3;
+
+		output[0] = input[0] >> 2;
+		output[1] = ((input[0] & 0x03) << 4) + (input[1] >> 4);
+		output[2] = ((input[1] & 0x0f) << 2) + (input[2] >> 6);
+		output[3] = input[2] & 0x3f;
+
+		if (datalength + 4 > targsize)
+			return (-1);
+		target[datalength++] = Base64[output[0]];
+		target[datalength++] = Base64[output[1]];
+		target[datalength++] = Base64[output[2]];
+		target[datalength++] = Base64[output[3]];
+	}
+
+	/* Now we worry about padding. */
+	if (0 != srclength) {
+		/* Get what's left. */
+		input[0] = input[1] = input[2] = '\0';
+		for (i = 0; i < srclength; i++)
+			input[i] = *src++;
+
+		output[0] = input[0] >> 2;
+		output[1] = ((input[0] & 0x03) << 4) + (input[1] >> 4);
+		output[2] = ((input[1] & 0x0f) << 2) + (input[2] >> 6);
+
+		if (datalength + 4 > targsize)
+			return (-1);
+		target[datalength++] = Base64[output[0]];
+		target[datalength++] = Base64[output[1]];
+		if (srclength == 1)
+			target[datalength++] = Pad64;
+		else
+			target[datalength++] = Base64[output[2]];
+		target[datalength++] = Pad64;
+	}
+	if (datalength >= targsize)
+		return (-1);
+	target[datalength] = '\0';	/* Returned value doesn't count \0. */
+	return (datalength);
+}
+
+/* skips all whitespace anywhere.
+   converts characters, four at a time, starting at (or after)
+   src from base - 64 numbers into three 8 bit bytes in the target area.
+   it returns the number of data bytes stored at the target, or -1 on error.
+ */
+
+int
+__b64_pton(src, target, targsize)
+	char const *src;
+	u_char *target;
+	size_t targsize;
+{
+	int tarindex, state, ch;
+	u_char nextbyte;
+	char *pos;
+
+	state = 0;
+	tarindex = 0;
+
+	while ((ch = (unsigned char)*src++) != '\0') {
+		if (isspace(ch))	/* Skip whitespace anywhere. */
+			continue;
+
+		if (ch == Pad64)
+			break;
+
+		pos = strchr(Base64, ch);
+		if (pos == 0) 		/* A non-base64 character. */
+			return (-1);
+
+		switch (state) {
+		case 0:
+			if (target) {
+				if (tarindex >= targsize)
+					return (-1);
+				target[tarindex] = (pos - Base64) << 2;
+			}
+			state = 1;
+			break;
+		case 1:
+			if (target) {
+				if (tarindex >= targsize)
+					return (-1);
+				target[tarindex]   |=  (pos - Base64) >> 4;
+				nextbyte = ((pos - Base64) & 0x0f) << 4;
+				if (tarindex + 1 < targsize)
+					target[tarindex+1] = nextbyte;
+				else if (nextbyte)
+					return (-1);
+			}
+			tarindex++;
+			state = 2;
+			break;
+		case 2:
+			if (target) {
+				if (tarindex >= targsize)
+					return (-1);
+				target[tarindex]   |=  (pos - Base64) >> 2;
+				nextbyte = ((pos - Base64) & 0x03) << 6;
+				if (tarindex + 1 < targsize)
+					target[tarindex+1] = nextbyte;
+				else if (nextbyte)
+					return (-1);
+			}
+			tarindex++;
+			state = 3;
+			break;
+		case 3:
+			if (target) {
+				if (tarindex >= targsize)
+					return (-1);
+				target[tarindex] |= (pos - Base64);
+			}
+			tarindex++;
+			state = 0;
+			break;
+		}
+	}
+
+	/*
+	 * We are done decoding Base-64 chars.  Let's see if we ended
+	 * on a byte boundary, and/or with erroneous trailing characters.
+	 */
+
+	if (ch == Pad64) {			/* We got a pad char. */
+		ch = (unsigned char)*src++;	/* Skip it, get next. */
+		switch (state) {
+		case 0:		/* Invalid = in first position */
+		case 1:		/* Invalid = in second position */
+			return (-1);
+
+		case 2:		/* Valid, means one byte of info */
+			/* Skip any number of spaces. */
+			for (; ch != '\0'; ch = (unsigned char)*src++)
+				if (!isspace(ch))
+					break;
+			/* Make sure there is another trailing = sign. */
+			if (ch != Pad64)
+				return (-1);
+			ch = (unsigned char)*src++;		/* Skip the = */
+			/* Fall through to "single trailing =" case. */
+			/* FALLTHROUGH */
+
+		case 3:		/* Valid, means two bytes of info */
+			/*
+			 * We know this char is an =.  Is there anything but
+			 * whitespace after it?
+			 */
+			for (; ch != '\0'; ch = (unsigned char)*src++)
+				if (!isspace(ch))
+					return (-1);
+
+			/*
+			 * Now make sure for cases 2 and 3 that the "extra"
+			 * bits that slopped past the last full byte were
+			 * zeros.  If we don't check them, they become a
+			 * subliminal channel.
+			 */
+			if (target && tarindex < targsize &&
+			    target[tarindex] != 0)
+				return (-1);
+		}
+	} else {
+		/*
+		 * We ended by seeing the end of the string.  Make sure we
+		 * have no partial bytes lying around.
+		 */
+		if (state != 0)
+			return (-1);
+	}
+
+	return (tarindex);
+}
diff --git a/common/base64.h b/common/base64.h
new file mode 100644
index 0000000..9b86fc1
--- /dev/null
+++ b/common/base64.h
@@ -0,0 +1,10 @@
+#ifndef _BASE64_H
+#define _BASE64_H
+
+extern int __b64_ntop(u_char const *src, size_t srclength, char *target, size_t targsize);
+extern int __b64_pton(char const *src, u_char *target, size_t targsize);
+
+#define rfbBase64NtoP __b64_ntop
+#define rfbBase64PtoN __b64_pton
+
+#endif /* _BASE64_H */
diff --git a/libvncserver/websockets.c b/libvncserver/websockets.c
index b9947c4..4ebff72 100644
--- a/libvncserver/websockets.c
+++ b/libvncserver/websockets.c
@@ -52,7 +52,7 @@
 #include "rfbssl.h"
 #include "rfbcrypto.h"
 #include "ws_decode.h"
-
+#include "base64.h"
 
 #if 0
 #include <sys/syscall.h>
@@ -117,8 +117,8 @@ static void webSocketsGenSha1Key(char *target, int size, char *key)
     iov[1].iov_base = GUID;
     iov[1].iov_len = sizeof(GUID) - 1;
     digestsha1(iov, 2, hash);
-    if (-1 == b64_ntop(hash, sizeof(hash), target, size))
-	rfbErr("b64_ntop failed\n");
+    if (-1 == rfbBase64NtoP(hash, sizeof(hash), target, size))
+	rfbErr("rfbBase64NtoP failed\n");
 }
 
 /*
@@ -412,7 +412,7 @@ webSocketsEncodeHybi(rfbClientPtr cl, const char *src, int len, char **dst)
     }
 
     if (wsctx->base64) {
-        if (-1 == (ret = b64_ntop((unsigned char *)src, len, wsctx->codeBufEncode + sz, sizeof(wsctx->codeBufEncode) - sz))) {
+        if (-1 == (ret = rfbBase64NtoP((unsigned char *)src, len, wsctx->codeBufEncode + sz, sizeof(wsctx->codeBufEncode) - sz))) {
             rfbErr("%s: Base 64 encode failed\n", __func__);
         } else {
           if (ret != blen)
diff --git a/libvncserver/ws_decode.c b/libvncserver/ws_decode.c
index 513fd4b..441ebc7 100644
--- a/libvncserver/ws_decode.c
+++ b/libvncserver/ws_decode.c
@@ -1,4 +1,5 @@
 #include "ws_decode.h"
+#include "base64.h"
 
 #include <string.h>
 #include <errno.h>
@@ -432,7 +433,7 @@ hybiReadAndDecode(ws_ctx_t *wsctx, char *dst, int len, int *sockRet, int nInBuf)
     case WS_OPCODE_TEXT_FRAME:
       data[toReturn] = '\0';
       ws_dbg("Initiate Base64 decoding in %p with max size %d and '\\0' at %p\n", data, bufsize, data + toReturn);
-      if (-1 == (wsctx->readlen = b64_pton((char *)data, data, bufsize))) {
+      if (-1 == (wsctx->readlen = rfbBase64PtoN((char *)data, data, bufsize))) {
         rfbErr("%s: Base64 decode error; %s\n", __func__, strerror(errno));
       }
       wsctx->writePos = hybiPayloadStart(wsctx);
diff --git a/libvncserver/ws_decode.h b/libvncserver/ws_decode.h
index 2923e3d..709477a 100644
--- a/libvncserver/ws_decode.h
+++ b/libvncserver/ws_decode.h
@@ -3,9 +3,6 @@
 
 #include <stdint.h>
 #include <rfb/rfb.h>
-#ifndef _MSC_VER
-#include <resolv.h> /* __b64_ntop */
-#endif
 
 #if defined(__APPLE__)