common: move tls calls to ssl_calls

6 files changed, 310 insertions, 332 deletions

diff --git a/common/Makefile.am b/common/Makefile.am
index 9feac9fb..8c7aa62c 100644
--- a/common/Makefile.am
+++ b/common/Makefile.am
@@ -39,8 +39,7 @@ libcommon_la_SOURCES = \
   os_calls.c \
   ssl_calls.c \
   thread_calls.c \
-  trans.c \
-  xrdp_tls.c
+  trans.c
 
 libcommon_la_LIBADD = \
   -lcrypto \
diff --git a/common/ssl_calls.c b/common/ssl_calls.c
index 134f5afd..966bbb15 100644
--- a/common/ssl_calls.c
+++ b/common/ssl_calls.c
@@ -2,6 +2,7 @@
  * xrdp: A Remote Desktop Protocol server.
  *
  * Copyright (C) Jay Sorg 2004-2014
+ * Copyright (C) Idan Freiberg 2013-2014
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,6 +32,7 @@
 #include "os_calls.h"
 #include "arch.h"
 #include "ssl_calls.h"
+#include "trans.h"
 
 #if defined(OPENSSL_VERSION_NUMBER) && (OPENSSL_VERSION_NUMBER >= 0x0090800f)
 #undef OLD_RSA_GEN1
@@ -524,3 +526,271 @@ ssl_gen_key_xrdp1(int key_size_in_bits, char *exp, int exp_len,
     return error;
 }
 #endif
+
+/*****************************************************************************/
+struct ssl_tls *
+APP_CC
+ssl_tls_create(struct trans *trans, const char *key, const char *cert)
+{
+    struct ssl_tls *self;
+    int pid;
+    char buf[1024];
+
+    self = (struct ssl_tls *) g_malloc(sizeof(struct ssl_tls), 1);
+    if (self != NULL)
+    {
+        self->trans = trans;
+        self->cert = (char *) cert;
+        self->key = (char *) key;
+        pid = g_getpid();
+        g_snprintf(buf, 1024, "xrdp_%8.8x_tls_rwo", pid);
+        self->rwo = g_create_wait_obj(buf);
+    }
+
+    return self;
+}
+
+/*****************************************************************************/
+int APP_CC
+ssl_tls_print_error(char *func, SSL *connection, int value)
+{
+    switch (SSL_get_error(connection, value))
+    {
+        case SSL_ERROR_ZERO_RETURN:
+            g_writeln("ssl_tls_print_error: %s: Server closed TLS connection",
+                      func);
+            return 1;
+
+        case SSL_ERROR_WANT_READ:
+            g_writeln("ssl_tls_print_error: SSL_ERROR_WANT_READ");
+            return 0;
+
+        case SSL_ERROR_WANT_WRITE:
+            g_writeln("ssl_tls_print_error: SSL_ERROR_WANT_WRITE");
+            return 0;
+
+        case SSL_ERROR_SYSCALL:
+            g_writeln("ssl_tls_print_error: %s: I/O error", func);
+            return 1;
+
+        case SSL_ERROR_SSL:
+            g_writeln("ssl_tls_print_error: %s: Failure in SSL library (protocol error?)",
+                      func);
+            return 1;
+
+        default:
+            g_writeln("ssl_tls_print_error: %s: Unknown error", func);
+            return 1;
+    }
+}
+
+/*****************************************************************************/
+int APP_CC
+ssl_tls_accept(struct ssl_tls *self)
+{
+    int connection_status;
+    long options = 0;
+
+    /**
+     * SSL_OP_NO_SSLv2:
+     *
+     * We only want SSLv3 and TLSv1, so disable SSLv2.
+     * SSLv3 is used by, eg. Microsoft RDC for Mac OS X.
+     */
+    options |= SSL_OP_NO_SSLv2;
+
+#if defined(SSL_OP_NO_COMPRESSION)
+    /**
+     * SSL_OP_NO_COMPRESSION:
+     *
+     * The Microsoft RDP server does not advertise support
+     * for TLS compression, but alternative servers may support it.
+     * This was observed between early versions of the FreeRDP server
+     * and the FreeRDP client, and caused major performance issues,
+     * which is why we're disabling it.
+     */
+    options |= SSL_OP_NO_COMPRESSION;
+#endif
+
+    /**
+     * SSL_OP_TLS_BLOCK_PADDING_BUG:
+     *
+     * The Microsoft RDP server does *not* support TLS padding.
+     * It absolutely needs to be disabled otherwise it won't work.
+     */
+    options |= SSL_OP_TLS_BLOCK_PADDING_BUG;
+
+    /**
+     * SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS:
+     *
+     * Just like TLS padding, the Microsoft RDP server does not
+     * support empty fragments. This needs to be disabled.
+     */
+    options |= SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS;
+
+    self->ctx = SSL_CTX_new(SSLv23_server_method());
+    /* set context options */
+    SSL_CTX_set_mode(self->ctx,
+                     SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER |
+                     SSL_MODE_ENABLE_PARTIAL_WRITE);
+    SSL_CTX_set_options(self->ctx, options);
+    SSL_CTX_set_read_ahead(self->ctx, 1);
+
+    if (self->ctx == NULL)
+    {
+        g_writeln("ssl_tls_accept: SSL_CTX_new failed");
+        return 1;
+    }
+
+    if (SSL_CTX_use_RSAPrivateKey_file(self->ctx, self->key, SSL_FILETYPE_PEM)
+            <= 0)
+    {
+        g_writeln("ssl_tls_accept: SSL_CTX_use_RSAPrivateKey_file failed");
+        return 1;
+    }
+
+    self->ssl = SSL_new(self->ctx);
+
+    if (self->ssl == NULL)
+    {
+        g_writeln("ssl_tls_accept: SSL_new failed");
+        return 1;
+    }
+
+    if (SSL_use_certificate_file(self->ssl, self->cert, SSL_FILETYPE_PEM) <= 0)
+    {
+        g_writeln("ssl_tls_accept: SSL_use_certificate_file failed");
+        return 1;
+    }
+
+    if (SSL_set_fd(self->ssl, self->trans->sck) < 1)
+    {
+        g_writeln("ssl_tls_accept: SSL_set_fd failed");
+        return 1;
+    }
+
+    connection_status = SSL_accept(self->ssl);
+
+    if (connection_status <= 0)
+    {
+        if (ssl_tls_print_error("SSL_accept", self->ssl, connection_status))
+        {
+            return 1;
+        }
+    }
+
+    g_writeln("ssl_tls_accept: TLS connection accepted");
+
+    return 0;
+}
+
+/*****************************************************************************/
+int APP_CC
+ssl_tls_disconnect(struct ssl_tls *self)
+{
+    int status = SSL_shutdown(self->ssl);
+    while (status != 1)
+    {
+        status = SSL_shutdown(self->ssl);
+
+        if (status <= 0)
+        {
+            if (ssl_tls_print_error("SSL_shutdown", self->ssl, status))
+            {
+                return 1;
+            }
+        }
+    }
+    return 0;
+}
+
+/*****************************************************************************/
+void APP_CC
+ssl_tls_delete(struct ssl_tls *self)
+{
+    if (self != NULL)
+    {
+        if (self->ssl)
+            SSL_free(self->ssl);
+
+        if (self->ctx)
+            SSL_CTX_free(self->ctx);
+
+        g_delete_wait_obj(self->rwo);
+
+        g_free(self);
+    }
+}
+
+/*****************************************************************************/
+int APP_CC
+ssl_tls_read(struct ssl_tls *tls, char *data, int length)
+{
+    int status;
+
+    status = SSL_read(tls->ssl, data, length);
+
+    switch (SSL_get_error(tls->ssl, status))
+    {
+        case SSL_ERROR_NONE:
+            break;
+
+        case SSL_ERROR_WANT_READ:
+        case SSL_ERROR_WANT_WRITE:
+            status = 0;
+            break;
+
+        default:
+            ssl_tls_print_error("SSL_read", tls->ssl, status);
+            status = -1;
+            break;
+    }
+
+    if (SSL_pending(tls->ssl) > 0)
+    {
+        g_set_wait_obj(tls->rwo);
+    }
+
+    return status;
+}
+
+/*****************************************************************************/
+int APP_CC
+ssl_tls_write(struct ssl_tls *tls, const char *data, int length)
+{
+    int status;
+
+    status = SSL_write(tls->ssl, data, length);
+
+    switch (SSL_get_error(tls->ssl, status))
+    {
+        case SSL_ERROR_NONE:
+            break;
+
+        case SSL_ERROR_WANT_READ:
+        case SSL_ERROR_WANT_WRITE:
+            status = 0;
+            break;
+
+        default:
+            ssl_tls_print_error("SSL_write", tls->ssl, status);
+            status = -1;
+            break;
+    }
+
+    return status;
+}
+
+/*****************************************************************************/
+/* returns boolean */
+int APP_CC
+ssl_tls_can_recv(struct ssl_tls *tls, int sck, int millis)
+{
+    if (SSL_pending(tls->ssl) > 0)
+    {
+        return 1;
+    }
+    g_reset_wait_obj(tls->rwo);
+    return g_tcp_can_recv(sck, millis);
+}
+
diff --git a/common/ssl_calls.h b/common/ssl_calls.h
index 40acfb5b..6cfe73a3 100644
--- a/common/ssl_calls.h
+++ b/common/ssl_calls.h
@@ -2,6 +2,7 @@
  * xrdp: A Remote Desktop Protocol server.
  *
  * Copyright (C) Jay Sorg 2004-2014
+ * Copyright (C) Idan Freiberg 2013-2014
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -80,4 +81,31 @@ int APP_CC
 ssl_gen_key_xrdp1(int key_size_in_bits, char* exp, int exp_len,
                   char* mod, int mod_len, char* pri, int pri_len);
 
+/* ssl_tls */
+struct ssl_tls
+{
+    void *ssl; /* SSL * */
+    void *ctx; /* SSL_CTX * */
+    char *cert;
+    char *key;
+    struct trans *trans;
+    tintptr rwo; /* wait obj */
+};
+
+/* xrdp_tls.c */
+struct ssl_tls *APP_CC
+ssl_tls_create(struct trans *trans, const char *key, const char *cert);
+int APP_CC
+ssl_tls_accept(struct ssl_tls *self);
+int APP_CC
+ssl_tls_disconnect(struct ssl_tls *self);
+void APP_CC
+ssl_tls_delete(struct ssl_tls *self);
+int APP_CC
+ssl_tls_read(struct ssl_tls *tls, char *data, int length);
+int APP_CC
+ssl_tls_write(struct ssl_tls *tls, const char *data, int length);
+int APP_CC
+ssl_tls_can_recv(struct ssl_tls *tls, int sck, int millis);
+
 #endif
diff --git a/common/trans.c b/common/trans.c
index 5f435bcd..3828a174 100644
--- a/common/trans.c
+++ b/common/trans.c
@@ -22,6 +22,7 @@
 #include "trans.h"
 #include "arch.h"
 #include "parse.h"
+#include "ssl_calls.h"
 
 /*****************************************************************************/
 int APP_CC
@@ -31,7 +32,7 @@ trans_tls_recv(struct trans *self, void *ptr, int len)
     {
         return 1;
     }
-    return xrdp_tls_read(self->tls, ptr, len);
+    return ssl_tls_read(self->tls, ptr, len);
 }
 
 /*****************************************************************************/
@@ -42,7 +43,7 @@ trans_tls_send(struct trans *self, const void *data, int len)
     {
         return 1;
     }
-    return xrdp_tls_write(self->tls, data, len);
+    return ssl_tls_write(self->tls, data, len);
 }
 
 /*****************************************************************************/
@@ -53,7 +54,7 @@ trans_tls_can_recv(struct trans *self, int sck, int millis)
     {
         return 1;
     }
-    return xrdp_tls_can_recv(self->tls, sck, millis);
+    return ssl_tls_can_recv(self->tls, sck, millis);
 }
 
 /*****************************************************************************/
@@ -130,7 +131,7 @@ trans_delete(struct trans *self)
 
     if (self->tls != 0)
     {
-        xrdp_tls_delete(self->tls);
+        ssl_tls_delete(self->tls);
     }
 
     g_free(self);
@@ -721,16 +722,16 @@ trans_get_out_s(struct trans *self, int size)
 int APP_CC
 trans_set_tls_mode(struct trans *self, const char *key, const char *cert)
 {
-    self->tls = xrdp_tls_create(self, key, cert);
+    self->tls = ssl_tls_create(self, key, cert);
     if (self->tls == NULL)
     {
-        g_writeln("trans_set_tls_mode: xrdp_tls_create malloc error");
+        g_writeln("trans_set_tls_mode: ssl_tls_create malloc error");
         return 1;
     }
 
-    if (xrdp_tls_accept(self->tls) != 0)
+    if (ssl_tls_accept(self->tls) != 0)
     {
-        g_writeln("trans_set_tls_mode: xrdp_tls_accept failed");
+        g_writeln("trans_set_tls_mode: ssl_tls_accept failed");
         return 1;
     }
 
@@ -748,7 +749,7 @@ trans_shutdown_tls_mode(struct trans *self)
 {
     if (self->tls != NULL)
     {
-        return xrdp_tls_disconnect(self->tls);
+        return ssl_tls_disconnect(self->tls);
     }
 
     /* assign callback back to tcp cal */
diff --git a/common/trans.h b/common/trans.h
index fbf08f01..c2a10762 100644
--- a/common/trans.h
+++ b/common/trans.h
@@ -64,39 +64,12 @@ struct trans
     char port[256];
     int no_stream_init_on_data_in;
     int extra_flags; /* user defined */
-    struct xrdp_tls *tls;
+    struct ssl_tls *tls;
     trans_recv_proc trans_recv;
     trans_send_proc trans_send;
     trans_can_recv_proc trans_can_recv;
 };
 
-/* xrdp_tls */
-struct xrdp_tls
-{
-    void *ssl; /* SSL * */
-    void *ctx; /* SSL_CTX * */
-    char *cert;
-    char *key;
-    struct trans *trans;
-    tintptr rwo; /* wait obj */
-};
-
-/* xrdp_tls.c */
-struct xrdp_tls *APP_CC
-xrdp_tls_create(struct trans *trans, const char *key, const char *cert);
-int APP_CC
-xrdp_tls_accept(struct xrdp_tls *self);
-int APP_CC
-xrdp_tls_disconnect(struct xrdp_tls *self);
-void APP_CC
-xrdp_tls_delete(struct xrdp_tls *self);
-int APP_CC
-xrdp_tls_read(struct xrdp_tls *tls, char *data, int length);
-int APP_CC
-xrdp_tls_write(struct xrdp_tls *tls, const char *data, int length);
-int APP_CC
-xrdp_tls_can_recv(struct xrdp_tls *tls, int sck, int millis);
-
 struct trans* APP_CC
 trans_create(int mode, int in_size, int out_size);
 void APP_CC
diff --git a/common/xrdp_tls.c b/common/xrdp_tls.c
deleted file mode 100644
index 3c74c47a..00000000
--- a/common/xrdp_tls.c
+++ /dev/null
@@ -1,293 +0,0 @@
-/**
- * xrdp: A Remote Desktop Protocol server.
- *
- * Copyright (C) Idan Freiberg 2013-2014
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * transport layer security
- */
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <openssl/ssl.h>
-
-#include "os_calls.h"
-#include "trans.h"
-#include "ssl_calls.h"
-
-
-/*****************************************************************************/
-struct xrdp_tls *
-APP_CC
-xrdp_tls_create(struct trans *trans, const char *key, const char *cert)
-{
-    struct xrdp_tls *self;
-    int pid;
-    char buf[1024];
-
-    self = (struct xrdp_tls *) g_malloc(sizeof(struct xrdp_tls), 1);
-    if (self != NULL)
-    {
-        self->trans = trans;
-        self->cert = (char *) cert;
-        self->key = (char *) key;
-        pid = g_getpid();
-        g_snprintf(buf, 1024, "xrdp_%8.8x_tls_rwo", pid);
-        self->rwo = g_create_wait_obj(buf);
-    }
-
-    return self;
-}
-
-/*****************************************************************************/
-int APP_CC
-xrdp_tls_print_error(char *func, SSL *connection, int value)
-{
-    switch (SSL_get_error(connection, value))
-    {
-        case SSL_ERROR_ZERO_RETURN:
-            g_writeln("xrdp_tls_print_error: %s: Server closed TLS connection",
-                      func);
-            return 1;
-
-        case SSL_ERROR_WANT_READ:
-            g_writeln("xrdp_tls_print_error: SSL_ERROR_WANT_READ");
-            return 0;
-
-        case SSL_ERROR_WANT_WRITE:
-            g_writeln("xrdp_tls_print_error: SSL_ERROR_WANT_WRITE");
-            return 0;
-
-        case SSL_ERROR_SYSCALL:
-            g_writeln("xrdp_tls_print_error: %s: I/O error", func);
-            return 1;
-
-        case SSL_ERROR_SSL:
-            g_writeln("xrdp_tls_print_error: %s: Failure in SSL library (protocol error?)",
-                      func);
-            return 1;
-
-        default:
-            g_writeln("xrdp_tls_print_error: %s: Unknown error", func);
-            return 1;
-    }
-}
-
-/*****************************************************************************/
-int APP_CC
-xrdp_tls_accept(struct xrdp_tls *self)
-{
-    int connection_status;
-    long options = 0;
-
-    /**
-     * SSL_OP_NO_SSLv2:
-     *
-     * We only want SSLv3 and TLSv1, so disable SSLv2.
-     * SSLv3 is used by, eg. Microsoft RDC for Mac OS X.
-     */
-    options |= SSL_OP_NO_SSLv2;
-
-#if defined(SSL_OP_NO_COMPRESSION)
-    /**
-     * SSL_OP_NO_COMPRESSION:
-     *
-     * The Microsoft RDP server does not advertise support
-     * for TLS compression, but alternative servers may support it.
-     * This was observed between early versions of the FreeRDP server
-     * and the FreeRDP client, and caused major performance issues,
-     * which is why we're disabling it.
-     */
-    options |= SSL_OP_NO_COMPRESSION;
-#endif
-
-    /**
-     * SSL_OP_TLS_BLOCK_PADDING_BUG:
-     *
-     * The Microsoft RDP server does *not* support TLS padding.
-     * It absolutely needs to be disabled otherwise it won't work.
-     */
-    options |= SSL_OP_TLS_BLOCK_PADDING_BUG;
-
-    /**
-     * SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS:
-     *
-     * Just like TLS padding, the Microsoft RDP server does not
-     * support empty fragments. This needs to be disabled.
-     */
-    options |= SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS;
-
-    self->ctx = SSL_CTX_new(SSLv23_server_method());
-    /* set context options */
-    SSL_CTX_set_mode(self->ctx,
-                     SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER |
-                     SSL_MODE_ENABLE_PARTIAL_WRITE);
-    SSL_CTX_set_options(self->ctx, options);
-    SSL_CTX_set_read_ahead(self->ctx, 1);
-
-    if (self->ctx == NULL)
-    {
-        g_writeln("xrdp_tls_accept: SSL_CTX_new failed");
-        return 1;
-    }
-
-    if (SSL_CTX_use_RSAPrivateKey_file(self->ctx, self->key, SSL_FILETYPE_PEM)
-            <= 0)
-    {
-        g_writeln("xrdp_tls_accept: SSL_CTX_use_RSAPrivateKey_file failed");
-        return 1;
-    }
-
-    self->ssl = SSL_new(self->ctx);
-
-    if (self->ssl == NULL)
-    {
-        g_writeln("xrdp_tls_accept: SSL_new failed");
-        return 1;
-    }
-
-    if (SSL_use_certificate_file(self->ssl, self->cert, SSL_FILETYPE_PEM) <= 0)
-    {
-        g_writeln("xrdp_tls_accept: SSL_use_certificate_file failed");
-        return 1;
-    }
-
-    if (SSL_set_fd(self->ssl, self->trans->sck) < 1)
-    {
-        g_writeln("xrdp_tls_accept: SSL_set_fd failed");
-        return 1;
-    }
-
-    connection_status = SSL_accept(self->ssl);
-
-    if (connection_status <= 0)
-    {
-        if (xrdp_tls_print_error("SSL_accept", self->ssl, connection_status))
-        {
-            return 1;
-        }
-    }
-
-    g_writeln("xrdp_tls_accept: TLS connection accepted");
-
-    return 0;
-}
-/*****************************************************************************/
-int APP_CC
-xrdp_tls_disconnect(struct xrdp_tls *self)
-{
-    int status = SSL_shutdown(self->ssl);
-    while (status != 1)
-    {
-        status = SSL_shutdown(self->ssl);
-
-        if (status <= 0)
-        {
-            if (xrdp_tls_print_error("SSL_shutdown", self->ssl, status))
-            {
-                return 1;
-            }
-        }
-    }
-    return 0;
-}
-/*****************************************************************************/
-void APP_CC
-xrdp_tls_delete(struct xrdp_tls *self)
-{
-    if (self != NULL)
-    {
-        if (self->ssl)
-            SSL_free(self->ssl);
-
-        if (self->ctx)
-            SSL_CTX_free(self->ctx);
-
-        g_delete_wait_obj(self->rwo);
-
-        g_free(self);
-    }
-}
-/*****************************************************************************/
-int APP_CC
-xrdp_tls_read(struct xrdp_tls *tls, char *data, int length)
-{
-    int status;
-
-    status = SSL_read(tls->ssl, data, length);
-
-    switch (SSL_get_error(tls->ssl, status))
-    {
-        case SSL_ERROR_NONE:
-            break;
-
-        case SSL_ERROR_WANT_READ:
-        case SSL_ERROR_WANT_WRITE:
-            status = 0;
-            break;
-
-        default:
-            xrdp_tls_print_error("SSL_read", tls->ssl, status);
-            status = -1;
-            break;
-    }
-
-    if (SSL_pending(tls->ssl) > 0)
-    {
-        g_set_wait_obj(tls->rwo);
-    }
-
-    return status;
-}
-/*****************************************************************************/
-int APP_CC
-xrdp_tls_write(struct xrdp_tls *tls, const char *data, int length)
-{
-    int status;
-
-    status = SSL_write(tls->ssl, data, length);
-
-    switch (SSL_get_error(tls->ssl, status))
-    {
-        case SSL_ERROR_NONE:
-            break;
-
-        case SSL_ERROR_WANT_READ:
-        case SSL_ERROR_WANT_WRITE:
-            status = 0;
-            break;
-
-        default:
-            xrdp_tls_print_error("SSL_write", tls->ssl, status);
-            status = -1;
-            break;
-    }
-
-    return status;
-}
-/*****************************************************************************/
-/* returns boolean */
-int APP_CC
-xrdp_tls_can_recv(struct xrdp_tls *tls, int sck, int millis)
-{
-    if (SSL_pending(tls->ssl) > 0)
-    {
-        return 1;
-    }
-    g_reset_wait_obj(tls->rwo);
-    return g_tcp_can_recv(sck, millis);
-}
-