From e67e2e12e61a5f5e111afc445b2f3256b8034c9b Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Tue, 3 Jul 2018 15:02:11 -0700 Subject: [PATCH] obs-outputs: Add support for and use mbedTLS for SSL This diff adds mbedTLS support to the obs-outputs plugin. PolarSSL and mbedTLS have grown so different between 2015-or-so when libRTMP was written, and now it's no longer feasible to just use the USE_POLARSSL flag. This commit adds a WITH_RTMPS tri-state CMake variable (auto/on/off), set to "Auto" by default. "Auto" will use RTMPS if mbedTLS is found, otherwise will disable RTMPS. "On" will make it require mbedTLS, otherwise fails configuration, and "Off" disables RTMPS support altogether. Closes obsproject/obs-studio#1360 --- cmake/Modules/FindMbedTLS.cmake | 143 ++++++++++++ cmake/Modules/FindSSL.cmake | 77 ------- plugins/obs-outputs/CMakeLists.txt | 43 +++- plugins/obs-outputs/data/locale/en-US.ini | 1 + plugins/obs-outputs/librtmp/dh.h | 56 ++++- plugins/obs-outputs/librtmp/handshake.h | 34 ++- plugins/obs-outputs/librtmp/hashswf.c | 44 +++- plugins/obs-outputs/librtmp/rtmp.c | 254 ++++++++++++++++++++-- plugins/obs-outputs/librtmp/rtmp_sys.h | 72 +++++- plugins/obs-outputs/rtmp-stream.c | 9 + plugins/obs-outputs/rtmp-windows.c | 8 +- 11 files changed, 622 insertions(+), 119 deletions(-) create mode 100644 cmake/Modules/FindMbedTLS.cmake delete mode 100644 cmake/Modules/FindSSL.cmake diff --git a/cmake/Modules/FindMbedTLS.cmake b/cmake/Modules/FindMbedTLS.cmake new file mode 100644 index 000000000..954df18e8 --- /dev/null +++ b/cmake/Modules/FindMbedTLS.cmake @@ -0,0 +1,143 @@ +# Once done these will be defined: +# +# LIBMBEDTLS_FOUND +# LIBMBEDTLS_INCLUDE_DIRS +# LIBMBEDTLS_LIBRARIES +# +# For use in OBS: +# +# MBEDTLS_INCLUDE_DIR + +find_package(PkgConfig QUIET) +if (PKG_CONFIG_FOUND) + pkg_check_modules(_MBEDTLS QUIET mbedtls) +endif() + +if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(_lib_suffix 64) +else() + set(_lib_suffix 32) +endif() + +# If we're on MacOS or Linux, please try to statically-link mbedtls. +if(STATIC_MBEDTLS AND (APPLE OR UNIX)) + set(_MBEDTLS_LIBRARIES libmbedtls.a) + set(_MBEDCRYPTO_LIBRARIES libmbedcrypto.a) + set(_MBEDX509_LIBRARIES libmbedx509.a) +endif() + +find_path(MBEDTLS_INCLUDE_DIR + NAMES mbedtls/ssl.h + HINTS + ENV mbedtlsPath${_lib_suffix} + ENV mbedtlsPath + ENV DepsPath${_lib_suffix} + ENV DepsPath + ${mbedtlsPath${_lib_suffix}} + ${mbedtlsPath} + ${DepsPath${_lib_suffix}} + ${DepsPath} + ${_MBEDTLS_INCLUDE_DIRS} + PATHS + /usr/include /usr/local/include /opt/local/include /sw/include + PATH_SUFFIXES + include) + +find_library(MBEDTLS_LIB + NAMES ${_MBEDTLS_LIBRARIES} mbedtls libmbedtls + HINTS + ENV mbedtlsPath${_lib_suffix} + ENV mbedtlsPath + ENV DepsPath${_lib_suffix} + ENV DepsPath + ${mbedtlsPath${_lib_suffix}} + ${mbedtlsPath} + ${DepsPath${_lib_suffix}} + ${DepsPath} + ${_MBEDTLS_LIBRARY_DIRS} + PATHS + /usr/lib /usr/local/lib /opt/local/lib /sw/lib + PATH_SUFFIXES + lib${_lib_suffix} lib + libs${_lib_suffix} libs + bin${_lib_suffix} bin + ../lib${_lib_suffix} ../lib + ../libs${_lib_suffix} ../libs + ../bin${_lib_suffix} ../bin) + +find_library(MBEDCRYPTO_LIB + NAMES ${_MBEDCRYPTO_LIBRARIES} mbedcrypto libmbedcrypto + HINTS + ENV mbedcryptoPath${_lib_suffix} + ENV mbedcryptoPath + ENV DepsPath${_lib_suffix} + ENV DepsPath + ${mbedcryptoPath${_lib_suffix}} + ${mbedcryptoPath} + ${DepsPath${_lib_suffix}} + ${DepsPath} + ${_MBEDCRYPTO_LIBRARY_DIRS} + PATHS + /usr/lib /usr/local/lib /opt/local/lib /sw/lib + PATH_SUFFIXES + lib${_lib_suffix} lib + libs${_lib_suffix} libs + bin${_lib_suffix} bin + ../lib${_lib_suffix} ../lib + ../libs${_lib_suffix} ../libs + ../bin${_lib_suffix} ../bin) + +find_library(MBEDX509_LIB + NAMES ${_MBEDX509_LIBRARIES} mbedx509 libmbedx509 + HINTS + ENV mbedx509Path${_lib_suffix} + ENV mbedx509Path + ENV DepsPath${_lib_suffix} + ENV DepsPath + ${mbedx509Path${_lib_suffix}} + ${mbedx509Path} + ${DepsPath${_lib_suffix}} + ${DepsPath} + ${_MBEDX509_LIBRARY_DIRS} + PATHS + /usr/lib /usr/local/lib /opt/local/lib /sw/lib + PATH_SUFFIXES + lib${_lib_suffix} lib + libs${_lib_suffix} libs + bin${_lib_suffix} bin + ../lib${_lib_suffix} ../lib + ../libs${_lib_suffix} ../libs + ../bin${_lib_suffix} ../bin) + +# Sometimes mbedtls is split between three libs, and sometimes it isn't. +# If it isn't, let's check if the symbols we need are all in MBEDTLS_LIB. +if(MBEDTLS_LIB AND NOT MBEDCRYPTO_LIB AND NOT MBEDX509_LIB) + set(CMAKE_REQUIRED_LIBRARIES ${MBEDTLS_LIB}) + set(CMAKE_REQUIRED_INCLUDES ${MBEDTLS_INCLUDE_DIR}) + check_symbol_exists(mbedtls_x509_crt_init "mbedtls/x509_crt.h" MBEDTLS_INCLUDES_X509) + check_symbol_exists(mbedtls_sha256_init "mbedtls/sha256.h" MBEDTLS_INCLUDES_CRYPTO) + unset(CMAKE_REQUIRED_INCLUDES) + unset(CMAKE_REQUIRED_LIBRARIES) +endif() + +# If we find all three libraries, then go ahead. +if(MBEDTLS_LIB AND MBEDCRYPTO_LIB AND MBEDX509_LIB) + set(LIBMBEDTLS_INCLUDE_DIRS ${MBEDTLS_INCLUDE_DIR}) + set(LIBMBEDTLS_LIBRARIES ${MBEDTLS_LIB} ${MBEDCRYPTO_LIB} ${MBEDX509_LIB}) + set(MBEDTLS_INCLUDE_DIRS ${LIBMBEDTLS_INCLUDE_DIRS}) + set(MBEDTLS_LIBRARIES ${LIBMBEDTLS_LIBRARIES}) + +# Otherwise, if we find MBEDTLS_LIB, and it has both CRYPTO and x509 +# within the single lib (i.e. a windows build environment), then also +# feel free to go ahead. +elseif(MBEDTLS_LIB AND MBEDTLS_INCLUDES_CRYPTO AND MBEDTLS_INCLUDES_X509) + set(LIBMBEDTLS_INCLUDE_DIRS ${MBEDTLS_INCLUDE_DIR}) + set(LIBMBEDTLS_LIBRARIES ${MBEDTLS_LIB}) + set(MBEDTLS_INCLUDE_DIRS ${LIBMBEDTLS_INCLUDE_DIRS}) + set(MBEDTLS_LIBRARIES ${LIBMBEDTLS_LIBRARIES}) +endif() + +# Now we've accounted for the 3-vs-1 library case: +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Libmbedtls DEFAULT_MSG MBEDTLS_LIBRARIES MBEDTLS_INCLUDE_DIRS) +mark_as_advanced(MBEDTLS_INCLUDE_DIR MBEDTLS_LIBRARIES MBEDTLS_INCLUDE_DIRS) diff --git a/cmake/Modules/FindSSL.cmake b/cmake/Modules/FindSSL.cmake deleted file mode 100644 index 5f5d50eb1..000000000 --- a/cmake/Modules/FindSSL.cmake +++ /dev/null @@ -1,77 +0,0 @@ -# Once done these will be defined: -# -# SSL_FOUND -# SSL_INCLUDE_DIRS -# SSL_LIBRARIES -# -# For use in OBS: -# -# SSL_INCLUDE_DIR - -find_package(PkgConfig QUIET) -if (PKG_CONFIG_FOUND) - pkg_check_modules(_CRYPTO QUIET libcrypto) - pkg_check_modules(_SSL QUIET libssl) -endif() - -if(CMAKE_SIZEOF_VOID_P EQUAL 8) - set(_lib_suffix 64) -else() - set(_lib_suffix 32) -endif() - -set(_SSL_BASE_HINTS - ENV sslPath${_lib_suffix} - ENV sslPath - ENV DepsPath${_lib_suffix} - ENV DepsPath - ${sslPath${_lib_suffix}} - ${sslPath} - ${DepsPath${_lib_suffix}} - ${DepsPath}) - -set(_SSL_LIB_SUFFIXES - lib${_lib_suffix} lib - libs${_lib_suffix} libs - bin${_lib_suffix} bin - ../lib${_lib_suffix} ../lib - ../libs${_lib_suffix} ../libs - ../bin${_lib_suffix} ../bin) - -find_path(SSL_INCLUDE_DIR - NAMES openssl/ssl.h - HINTS - ${_SSL_BASE_HINTS} - ${_CRYPTO_INCLUDE_DIRS} - ${_SSL_INCLUDE_DIRS} - PATHS - /usr/include /usr/local/include /opt/local/include /sw/include - PATH_SUFFIXES - include) - -find_library(_SSL_LIB - NAMES ${_SSL_LIBRARIES} ssleay32 ssl - HINTS - ${_SSL_BASE_HINTS} - ${_SSL_LIBRARY_DIRS} - PATHS - /usr/lib /usr/local/lib /opt/local/lib /sw/lib - PATH_SUFFIXES ${_SSL_LIB_SUFFIXES}) - -find_library(_CRYPTO_LIB - NAMES ${_CRYPTO_LIBRARIES} libeay32 crypto - HINTS - ${_SSL_BASE_HINTS} - ${_CRYPTO_LIBRARY_DIRS} - PATHS - /usr/lib /usr/local/lib /opt/local/lib /sw/lib - PATH_SUFFIXES ${_SSL_LIB_SUFFIXES}) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ssl DEFAULT_MSG _SSL_LIB _CRYPTO_LIB SSL_INCLUDE_DIR) -mark_as_advanced(SSL_INCLUDE_DIR _SSL_LIB _CRYPTO_LIB) - -if(SSL_FOUND) - set(SSL_INCLUDE_DIRS ${SSL_INCLUDE_DIR}) - set(SSL_LIBRARIES ${_SSL_LIB} ${_CRYPTO_LIB}) -endif() diff --git a/plugins/obs-outputs/CMakeLists.txt b/plugins/obs-outputs/CMakeLists.txt index 1bde00a58..0c373cdb3 100644 --- a/plugins/obs-outputs/CMakeLists.txt +++ b/plugins/obs-outputs/CMakeLists.txt @@ -1,20 +1,25 @@ project(obs-outputs) -option(USE_SSL "Enable rtmps support with OpenSSL" OFF) +set(WITH_RTMPS AUTO CACHE STRING "Enable RTMPS support with mbedTLS") +set_property(CACHE WITH_RTMPS PROPERTY STRINGS AUTO ON OFF) -if (USE_SSL) - find_package(SSL QUIET) +option(STATIC_MBEDTLS "Statically link mbedTLS into binary" OFF) + +if (WITH_RTMPS OR (WITH_RTMPS STREQUAL "AUTO")) + find_package(MbedTLS QUIET) find_package(ZLIB QUIET) endif() -if (SSL_FOUND AND ZLIB_FOUND) - add_definitions(-DCRYPTO -DUSE_OPENSSL) - include_directories(${SSL_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS}) +if (LIBMBEDTLS_FOUND AND ZLIB_FOUND) + add_definitions(-DCRYPTO -DUSE_MBEDTLS) + include_directories(${LIBMBEDTLS_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS}) else() - if (USE_SSL) - message(WARNING "SSL enabled by user, but OpenSSL was not found") + if(WITH_RTMPS STREQUAL "AUTO") + message(WARNING "mbedTLS was not found, RTMPS will be auto-disabled") + elseif (WITH_RTMPS) + message(FATAL_ERROR "RTMPS enabled by user, but mbedTLS was not found") endif() - unset(SSL_LIBRARIES) + unset(LIBMBEDTLS_LIBRARIES) unset(ZLIB_LIBRARIES) add_definitions(-DNO_CRYPTO) endif() @@ -84,6 +89,12 @@ if(WIN32) ws2_32 winmm Iphlpapi) + + if (WITH_RTMPS OR (WITH_RTMPS STREQUAL "AUTO")) + SET(obs-outputs_PLATFORM_DEPS + ${obs-outputs_PLATFORM_DEPS} + crypt32) + endif() endif() if(MSVC) @@ -92,6 +103,16 @@ if(MSVC) w32-pthreads) endif() +if(APPLE AND (WITH_RTMPS OR (WITH_RTMPS STREQUAL "AUTO"))) + find_library(FOUNDATION_FRAMEWORK Foundation) + find_library(SECURITY_FRAMEWORK Security) + + set(obs-outputs_PLATFORM_DEPS + ${obs-outputs_PLATFORM_DEPS} + ${FOUNDATION_FRAMEWORK} + ${SECURITY_FRAMEWORK}) +endif() + set(obs-outputs_librtmp_HEADERS librtmp/amf.h librtmp/bytes.h @@ -133,7 +154,7 @@ set(obs-outputs_SOURCES flv-output.c flv-mux.c net-if.c) - + add_library(obs-outputs MODULE ${ftl_SOURCES} ${ftl_HEADERS} @@ -143,7 +164,7 @@ add_library(obs-outputs MODULE ${obs-outputs_librtmp_HEADERS}) target_link_libraries(obs-outputs libobs - ${SSL_LIBRARIES} + ${LIBMBEDTLS_LIBRARIES} ${ZLIB_LIBRARIES} ${ftl_IMPORTS} ${obs-outputs_PLATFORM_DEPS}) diff --git a/plugins/obs-outputs/data/locale/en-US.ini b/plugins/obs-outputs/data/locale/en-US.ini index 58596b131..3a915ce78 100644 --- a/plugins/obs-outputs/data/locale/en-US.ini +++ b/plugins/obs-outputs/data/locale/en-US.ini @@ -11,3 +11,4 @@ ConnectionReset="The connection was reset by the peer. This usually indicates in HostNotFound="Hostname not found. Make sure you entered a valid streaming server and your internet connection / DNS are working correctly." NoData="Hostname found, but no data of the requested type. This can occur if you have bound to an IPv6 address and your streaming service only has IPv4 addresses (see Settings / Advanced)." AddressNotAvailable="Address not available. You may have tried to bind to an invalid IP address (see Settings / Advanced)." +SSLCertVerifyFailed="The RTMP server sent an invalid SSL certificate." diff --git a/plugins/obs-outputs/librtmp/dh.h b/plugins/obs-outputs/librtmp/dh.h index d43ef8073..e32fe94c7 100644 --- a/plugins/obs-outputs/librtmp/dh.h +++ b/plugins/obs-outputs/librtmp/dh.h @@ -21,7 +21,59 @@ * http://www.gnu.org/copyleft/lgpl.html */ -#ifdef USE_POLARSSL +#if defined(USE_MBEDTLS) +#include +#include +typedef mbedtls_mpi* MP_t; +#define MP_new(m) m = malloc(sizeof(mbedtls_mpi)); mbedtls_mpi_init(m) +#define MP_set_w(mpi, w) mbedtls_mpi_lset(mpi, w) +#define MP_cmp(u, v) mbedtls_mpi_cmp_mpi(u, v) +#define MP_set(u, v) mbedtls_mpi_copy(u, v) +#define MP_sub_w(mpi, w) mbedtls_mpi_sub_int(mpi, mpi, w) +#define MP_cmp_1(mpi) mbedtls_mpi_cmp_int(mpi, 1) +#define MP_modexp(r, y, q, p) mbedtls_mpi_exp_mod(r, y, q, p, NULL) +#define MP_free(mpi) mbedtls_mpi_free(mpi); free(mpi) +#define MP_gethex(u, hex, res) MP_new(u); res = mbedtls_mpi_read_string(u, 16, hex) == 0 +#define MP_bytes(u) mbedtls_mpi_size(u) +#define MP_setbin(u,buf,len) mbedtls_mpi_write_binary(u,buf,len) +#define MP_getbin(u,buf,len) MP_new(u); mbedtls_mpi_read_binary(u,buf,len) + +typedef struct MDH +{ + MP_t p; + MP_t g; + MP_t pub_key; + MP_t priv_key; + long length; + mbedtls_dhm_context ctx; +} MDH; + +#define MDH_new() calloc(1,sizeof(MDH)) +#define MDH_free(vp) {MDH *_dh = vp; mbedtls_dhm_free(&_dh->ctx); MP_free(_dh->p); MP_free(_dh->g); MP_free(_dh->pub_key); MP_free(_dh->priv_key); free(_dh);} + +static int MDH_generate_key(MDH *dh) +{ + unsigned char out[2]; + MP_set(&dh->ctx.P, dh->p); + MP_set(&dh->ctx.G, dh->g); + dh->ctx.len = 128; + mbedtls_dhm_make_public(&dh->ctx, 1024, out, 1, mbedtls_ctr_drbg_random, &RTMP_TLS_ctx->ctr_drbg); + MP_new(dh->pub_key); + MP_new(dh->priv_key); + MP_set(dh->pub_key, &dh->ctx.GX); + MP_set(dh->priv_key, &dh->ctx.X); + return 1; +} + +static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh) +{ + MP_set(&dh->ctx.GY, pub); + size_t olen; + mbedtls_dhm_calc_secret(&dh->ctx, secret, len, &olen, NULL, NULL); + return 0; +} + +#elif defined(USE_POLARSSL) #include typedef mpi * MP_t; #define MP_new(m) m = malloc(sizeof(mpi)); mpi_init(m) @@ -271,7 +323,7 @@ DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen) if (!dh || !dh->pub_key) return 0; - len = MP_bytes(dh->pub_key); + len = (int)MP_bytes(dh->pub_key); if (len <= 0 || len > (int) nPubkeyLen) return 0; diff --git a/plugins/obs-outputs/librtmp/handshake.h b/plugins/obs-outputs/librtmp/handshake.h index 163626113..8cd3e3224 100644 --- a/plugins/obs-outputs/librtmp/handshake.h +++ b/plugins/obs-outputs/librtmp/handshake.h @@ -24,7 +24,28 @@ /* This file is #included in rtmp.c, it is not meant to be compiled alone */ -#ifdef USE_POLARSSL +#if defined(USE_MBEDTLS) +#include +#include +#ifndef SHA256_DIGEST_LENGTH +#define SHA256_DIGEST_LENGTH 32 +#endif +typedef mbedtls_md_context_t *HMAC_CTX; +#define HMAC_setup(ctx, key, len) ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \ + mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \ + mbedtls_md_hmac_starts(ctx, (const unsigned char *)key, len) +#define HMAC_crunch(ctx, buf, len) mbedtls_md_hmac_update(ctx, buf, len) +#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; mbedtls_md_hmac_finish(ctx, dig) +#define HMAC_close(ctx) mbedtls_md_free(ctx); free(ctx); ctx = NULL + +typedef mbedtls_arc4_context* RC4_handle; +#define RC4_alloc(h) *h = malloc(sizeof(mbedtls_arc4_context)); mbedtls_arc4_init(*h) +#define RC4_setkey(h,l,k) mbedtls_arc4_setup(h,k,l) +#define RC4_encrypt(h,l,d) mbedtls_arc4_crypt(h,l,(unsigned char *)d,(unsigned char *)d) +#define RC4_encrypt2(h,l,s,d) mbedtls_arc4_crypt(h,l,(unsigned char *)s,(unsigned char *)d) +#define RC4_free(h) mbedtls_arc4_free(h); free(h); h = NULL + +#elif defined(USE_POLARSSL) #include #include #ifndef SHA256_DIGEST_LENGTH @@ -148,6 +169,8 @@ typedef unsigned int (getoff)(uint8_t *buf, unsigned int len); static unsigned int GetDHOffset2(uint8_t *handshake, unsigned int len) { + (void) len; + unsigned int offset = 0; uint8_t *ptr = handshake + 768; unsigned int res; @@ -177,6 +200,8 @@ GetDHOffset2(uint8_t *handshake, unsigned int len) static unsigned int GetDigestOffset2(uint8_t *handshake, unsigned int len) { + (void) len; + unsigned int offset = 0; uint8_t *ptr = handshake + 772; unsigned int res; @@ -206,6 +231,8 @@ GetDigestOffset2(uint8_t *handshake, unsigned int len) static unsigned int GetDHOffset1(uint8_t *handshake, unsigned int len) { + (void) len; + unsigned int offset = 0; uint8_t *ptr = handshake + 1532; unsigned int res; @@ -235,6 +262,8 @@ GetDHOffset1(uint8_t *handshake, unsigned int len) static unsigned int GetDigestOffset1(uint8_t *handshake, unsigned int len) { + (void) len; + unsigned int offset = 0; uint8_t *ptr = handshake + 8; unsigned int res; @@ -1128,6 +1157,7 @@ HandShake(RTMP * r, int FP9HandShake) __FUNCTION__); } } + // TODO(mgoulet): Should this have a HMAC_finish here? RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__); return TRUE; @@ -1482,6 +1512,8 @@ SHandShake(RTMP * r) } } + // TODO(mgoulet): Should this have an Rc4_free? + RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__); return TRUE; } diff --git a/plugins/obs-outputs/librtmp/hashswf.c b/plugins/obs-outputs/librtmp/hashswf.c index b8d5a9ed0..b6c2b61fb 100644 --- a/plugins/obs-outputs/librtmp/hashswf.c +++ b/plugins/obs-outputs/librtmp/hashswf.c @@ -30,7 +30,20 @@ #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif -#ifdef USE_POLARSSL +#if defined(USE_MBEDTLS) +#include +#ifndef SHA256_DIGEST_LENGTH +#define SHA256_DIGEST_LENGTH 32 +#endif +typedef mbedtls_md_context_t *HMAC_CTX; +#define HMAC_setup(ctx, key, len) ctx = malloc(sizeof(mbedtls_md_context_t)); mbedtls_md_init(ctx); \ + mbedtls_md_setup(ctx, mbedtls_md_info_from_type(MBEDTLS_MD_SHA256), 1); \ + mbedtls_md_hmac_starts(ctx, (const unsigned char *)key, len) +#define HMAC_crunch(ctx, buf, len) mbedtls_md_hmac_update(ctx, buf, len) +#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; mbedtls_md_hmac_finish(ctx, dig) +#define HMAC_close(ctx) free(ctx); mbedtls_md_free(ctx); ctx = NULL + +#elif defined(USE_POLARSSL) #include #ifndef SHA256_DIGEST_LENGTH #define SHA256_DIGEST_LENGTH 32 @@ -40,6 +53,7 @@ #define HMAC_crunch(ctx, buf, len) sha2_hmac_update(&ctx, buf, len) #define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig) #define HMAC_close(ctx) + #elif defined(USE_GNUTLS) #include #ifndef SHA256_DIGEST_LENGTH @@ -51,6 +65,7 @@ #define HMAC_crunch(ctx, buf, len) hmac_sha256_update(&ctx, len, buf) #define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) #define HMAC_close(ctx) + #else /* USE_OPENSSL */ #include #include @@ -161,8 +176,17 @@ HTTP_get(struct HTTP_ctx *http, const char *url, HTTP_read_callback *cb) goto leave; #else TLS_client(RTMP_TLS_ctx, sb.sb_ssl); + +#if defined(USE_MBEDTLS) + mbedtls_net_context *server_fd = &RTMP_TLS_ctx->net; + server_fd->fd = sb.sb_socket; + TLS_setfd(sb.sb_ssl, server_fd); +#else TLS_setfd(sb.sb_ssl, sb.sb_socket); - if (TLS_connect(sb.sb_ssl) < 0) +#endif + + int connect_return = TLS_connect(sb.sb_ssl); + if (connect_return < 0) { RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__); ret = HTTPRES_LOST_CONNECTION; @@ -318,21 +342,21 @@ swfcrunch(void *ptr, size_t size, size_t nmemb, void *stream) { unsigned char out[CHUNK]; i->zs->next_in = (unsigned char *)p; - i->zs->avail_in = len; + i->zs->avail_in = (uInt)len; do { i->zs->avail_out = CHUNK; i->zs->next_out = out; inflate(i->zs, Z_NO_FLUSH); len = CHUNK - i->zs->avail_out; - i->size += len; + i->size += (int)len; HMAC_crunch(i->ctx, out, len); } while (i->zs->avail_out == 0); } else { - i->size += len; + i->size += (int)len; HMAC_crunch(i->ctx, (unsigned char *)p, len); } return size * nmemb; @@ -469,7 +493,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash, home.av_val = "\\UserData"; #else hpre.av_val = getenv("HOMEDRIVE"); - hpre.av_len = strlen(hpre.av_val); + hpre.av_len = (int)strlen(hpre.av_val); home.av_val = getenv("HOMEPATH"); #endif #define DIRSEP "\\" @@ -482,7 +506,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash, #endif if (!home.av_val) home.av_val = "."; - home.av_len = strlen(home.av_val); + home.av_len = (int)strlen(home.av_val); /* SWF hash info is cached in a fixed-format file. * url: @@ -528,7 +552,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash, if (strncmp(buf + 5, url, hlen)) continue; r1 = strrchr(buf, '/'); - i = strlen(r1); + i = (int)strlen(r1); r1[--i] = '\0'; if (strncmp(r1, file, i)) continue; @@ -543,7 +567,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash, else if (!strncmp(buf, "hash: ", 6)) { unsigned char *ptr = hash, *in = (unsigned char *)buf + 6; - int l = strlen((char *)in) - 1; + int l = (int)strlen((char *)in) - 1; for (i = 0; i < l; i += 2) *ptr++ = (HEX2BIN(in[i]) << 4) | HEX2BIN(in[i + 1]); got++; @@ -625,7 +649,7 @@ RTMP_HashSWF(const char *url, unsigned int *size, unsigned char *hash, if (q) i = q - url; else - i = strlen(url); + i = (int)strlen(url); fprintf(f, "url: %.*s\n", i, url); } diff --git a/plugins/obs-outputs/librtmp/rtmp.c b/plugins/obs-outputs/librtmp/rtmp.c index 1c77efc9b..cb2aa765f 100644 --- a/plugins/obs-outputs/librtmp/rtmp.c +++ b/plugins/obs-outputs/librtmp/rtmp.c @@ -40,7 +40,20 @@ #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif -#ifdef USE_POLARSSL +#if defined(USE_MBEDTLS) +#if defined(_WIN32) +#include +#include +#elif defined(__APPLE__) +#include +#endif + +#include +#include +#include +#define MD5_DIGEST_LENGTH 16 + +#elif defined(USE_POLARSSL) #include #include #include @@ -57,7 +70,6 @@ static const char *my_dhm_P = "E8A700D60B7F1200FA8E77B0A979DABF"; static const char *my_dhm_G = "4"; - #elif defined(USE_GNUTLS) #include #define MD5_DIGEST_LENGTH 16 @@ -70,7 +82,8 @@ static const char *my_dhm_G = "4"; #include #include #endif -TLS_CTX RTMP_TLS_ctx; + +TLS_CTX RTMP_TLS_ctx = NULL; #endif #define RTMP_SIG_SIZE 1536 @@ -265,11 +278,101 @@ RTMP_LibVersion() return RTMP_LIB_VERSION; } +void +RTMP_TLS_LoadCerts() { +#ifdef USE_MBEDTLS + mbedtls_x509_crt *chain = RTMP_TLS_ctx->cacert = calloc(1, sizeof(struct mbedtls_x509_crt)); + mbedtls_x509_crt_init(chain); + +#if defined(_WIN32) + HCERTSTORE hCertStore; + PCCERT_CONTEXT pCertContext = NULL; + + if (!(hCertStore = CertOpenSystemStore((HCRYPTPROV)NULL, L"ROOT"))) { + goto error; + } + + while (pCertContext = CertEnumCertificatesInStore(hCertStore, pCertContext)) { + mbedtls_x509_crt_parse_der(chain, + (unsigned char *)pCertContext->pbCertEncoded, + pCertContext->cbCertEncoded); + } + + CertFreeCertificateContext(pCertContext); + CertCloseStore(hCertStore, 0); +#elif defined(__APPLE__) + SecKeychainRef keychain_ref; + CFMutableDictionaryRef search_settings_ref; + CFArrayRef result_ref; + + if (SecKeychainOpen("/System/Library/Keychains/SystemRootCertificates.keychain", + &keychain_ref) + != errSecSuccess) { + goto error; + } + + search_settings_ref = CFDictionaryCreateMutable(NULL, 0, NULL, NULL); + CFDictionarySetValue(search_settings_ref, kSecClass, kSecClassCertificate); + CFDictionarySetValue(search_settings_ref, kSecMatchLimit, kSecMatchLimitAll); + CFDictionarySetValue(search_settings_ref, kSecReturnRef, kCFBooleanTrue); + CFDictionarySetValue(search_settings_ref, kSecMatchSearchList, + CFArrayCreate(NULL, (const void **)&keychain_ref, 1, NULL)); + + if (SecItemCopyMatching(search_settings_ref, (CFTypeRef *)&result_ref) + != errSecSuccess) { + goto error; + } + + for (CFIndex i = 0; i < CFArrayGetCount(result_ref); i++) { + SecCertificateRef item_ref = (SecCertificateRef) + CFArrayGetValueAtIndex(result_ref, i); + CFDataRef data_ref; + + if ((data_ref = SecCertificateCopyData(item_ref))) { + mbedtls_x509_crt_parse_der(chain, + (unsigned char *)CFDataGetBytePtr(data_ref), + CFDataGetLength(data_ref)); + CFRelease(data_ref); + } + } + + CFRelease(keychain_ref); +#elif defined(__linux__) + if (mbedtls_x509_crt_parse_path(chain, "/etc/ssl/certs/") != 0) { + goto error; + } +#endif + + mbedtls_ssl_conf_ca_chain(&RTMP_TLS_ctx->conf, chain, NULL); + return; + +error: + mbedtls_x509_crt_free(chain); + free(chain); + RTMP_TLS_ctx->cacert = NULL; +#endif /* USE_MBEDTLS */ +} + void RTMP_TLS_Init() { #ifdef CRYPTO -#ifdef USE_POLARSSL +#if defined(USE_MBEDTLS) + const char * pers = "RTMP_TLS"; + RTMP_TLS_ctx = calloc(1,sizeof(struct tls_ctx)); + + mbedtls_ssl_config_init(&RTMP_TLS_ctx->conf); + mbedtls_ctr_drbg_init(&RTMP_TLS_ctx->ctr_drbg); + mbedtls_entropy_init(&RTMP_TLS_ctx->entropy); + + mbedtls_ctr_drbg_seed(&RTMP_TLS_ctx->ctr_drbg, + mbedtls_entropy_func, + &RTMP_TLS_ctx->entropy, + (const unsigned char *)pers, + strlen(pers)); + + RTMP_TLS_LoadCerts(); +#elif defined(USE_POLARSSL) /* Do this regardless of NO_SSL, we use havege for rtmpe too */ RTMP_TLS_ctx = calloc(1,sizeof(struct tls_ctx)); havege_init(&RTMP_TLS_ctx->hs); @@ -293,6 +396,26 @@ RTMP_TLS_Init() SSL_CTX_set_options(RTMP_TLS_ctx, SSL_OP_ALL); SSL_CTX_set_default_verify_paths(RTMP_TLS_ctx); #endif +#else +#endif +} + +void +RTMP_TLS_Free() { +#ifdef USE_MBEDTLS + mbedtls_ssl_config_free(&RTMP_TLS_ctx->conf); + mbedtls_ctr_drbg_free(&RTMP_TLS_ctx->ctr_drbg); + mbedtls_entropy_free(&RTMP_TLS_ctx->entropy); + + if (RTMP_TLS_ctx->cacert) { + mbedtls_x509_crt_free(RTMP_TLS_ctx->cacert); + free(RTMP_TLS_ctx->cacert); + RTMP_TLS_ctx->cacert = NULL; + } + + // NO mbedtls_net_free() BECAUSE WE SET IT UP BY HAND! + free(RTMP_TLS_ctx); + RTMP_TLS_ctx = NULL; #endif } @@ -303,7 +426,27 @@ RTMP_TLS_AllocServerContext(const char* cert, const char* key) #ifdef CRYPTO if (!RTMP_TLS_ctx) RTMP_TLS_Init(); -#ifdef USE_POLARSSL +#if defined(USE_MBEDTLS) + tls_server_ctx *tc = ctx = calloc(1, sizeof(struct tls_server_ctx)); + tc->conf = &RTMP_TLS_ctx->conf; + tc->ctr_drbg = &RTMP_TLS_ctx->ctr_drbg; + + mbedtls_x509_crt_init(&tc->cert); + if (mbedtls_x509_crt_parse_file(&tc->cert, cert)) + { + free(tc); + return NULL; + } + + mbedtls_pk_init(&tc->key); + if (mbedtls_pk_parse_keyfile(&tc->key, key, NULL)) + { + mbedtls_x509_crt_free(&tc->cert); + mbedtls_pk_free(&tc->key); + free(tc); + return NULL; + } +#elif defined(USE_POLARSSL) tls_server_ctx *tc = ctx = calloc(1, sizeof(struct tls_server_ctx)); tc->dhm_P = my_dhm_P; tc->dhm_G = my_dhm_G; @@ -350,7 +493,11 @@ void RTMP_TLS_FreeServerContext(void *ctx) { #ifdef CRYPTO -#ifdef USE_POLARSSL +#if defined(USE_MBEDTLS) + mbedtls_x509_crt_free(&((tls_server_ctx*)ctx)->cert); + mbedtls_pk_free(&((tls_server_ctx*)ctx)->key); + free(ctx); +#elif defined(USE_POLARSSL) x509_free(&((tls_server_ctx*)ctx)->cert); rsa_free(&((tls_server_ctx*)ctx)->key); free(ctx); @@ -374,6 +521,10 @@ RTMP_Alloc() void RTMP_Free(RTMP *r) { +#if defined(CRYPTO) && defined(USE_MBEDTLS) + if (RTMP_TLS_ctx) + RTMP_TLS_Free(); +#endif free(r); } @@ -850,9 +1001,20 @@ int RTMP_TLS_Accept(RTMP *r, void *ctx) { #if defined(CRYPTO) && !defined(NO_SSL) - TLS_server(ctx, r->m_sb.sb_ssl); + tls_server_ctx *srv_ctx = ctx; + TLS_server(srv_ctx, r->m_sb.sb_ssl); + +#if defined(USE_MBEDTLS) + mbedtls_net_context *client_fd = &RTMP_TLS_ctx->net; + mbedtls_net_init(client_fd); + client_fd->fd = r->m_sb.sb_socket; + TLS_setfd(r->m_sb.sb_ssl, client_fd); +#else TLS_setfd(r->m_sb.sb_ssl, r->m_sb.sb_socket); - if (TLS_accept(r->m_sb.sb_ssl) < 0) +#endif + + int connect_return = TLS_connect(r->m_sb.sb_ssl); + if (connect_return < 0) { RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__); return FALSE; @@ -872,10 +1034,59 @@ RTMP_Connect1(RTMP *r, RTMPPacket *cp) { #if defined(CRYPTO) && !defined(NO_SSL) TLS_client(RTMP_TLS_ctx, r->m_sb.sb_ssl); + +#if defined(USE_MBEDTLS) + mbedtls_net_context *server_fd = &RTMP_TLS_ctx->net; + server_fd->fd = r->m_sb.sb_socket; + TLS_setfd(r->m_sb.sb_ssl, server_fd); + + // make sure we verify the certificate hostname + char hostname[MBEDTLS_SSL_MAX_HOST_NAME_LEN + 1]; + + if (r->Link.hostname.av_len >= MBEDTLS_SSL_MAX_HOST_NAME_LEN) + return FALSE; + + memcpy(hostname, r->Link.hostname.av_val, r->Link.hostname.av_len); + hostname[r->Link.hostname.av_len] = 0; + + if (mbedtls_ssl_set_hostname(r->m_sb.sb_ssl, hostname)) + return FALSE; +#else TLS_setfd(r->m_sb.sb_ssl, r->m_sb.sb_socket); - if (TLS_connect(r->m_sb.sb_ssl) < 0) +#endif + + int connect_return = TLS_connect(r->m_sb.sb_ssl); + if (connect_return < 0) { - RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed", __FUNCTION__); +#if defined(USE_MBEDTLS) + if (connect_return == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED) + { + r->last_error_code = connect_return; + + // show a more detailed error in the log if possible + int verify_result = mbedtls_ssl_get_verify_result(r->m_sb.sb_ssl); + if (verify_result) + { + char err[256], *e; + if (mbedtls_x509_crt_verify_info(err, sizeof(err), "", verify_result) > 0) + { + e = strchr(err, '\n'); + if (e) + *e = '\0'; + } + else + { + strcpy(err, "unknown error"); + } + RTMP_Log(RTMP_LOGERROR, "%s, Cert verify failed: %d (%s)", __FUNCTION__, verify_result, err); + RTMP_Close(r); + return FALSE; + } + } +#endif + // output the error in a format that matches mbedTLS + connect_return = abs(connect_return); + RTMP_Log(RTMP_LOGERROR, "%s, TLS_Connect failed: -0x%x", __FUNCTION__, connect_return); RTMP_Close(r); return FALSE; } @@ -883,7 +1094,6 @@ RTMP_Connect1(RTMP *r, RTMPPacket *cp) RTMP_Log(RTMP_LOGERROR, "%s, no SSL/TLS support", __FUNCTION__); RTMP_Close(r); return FALSE; - #endif } if (r->Link.protocol & RTMP_FEATURE_HTTP) @@ -2402,7 +2612,19 @@ b64enc(const unsigned char *input, int length, char *output, int maxsize) { (void)maxsize; -#ifdef USE_POLARSSL +#if defined(USE_MBEDTLS) + size_t osize; + if(mbedtls_base64_encode((unsigned char *) output, maxsize, &osize, input, length) == 0) + { + output[osize] = '\0'; + return 1; + } + else + { + RTMP_Log(RTMP_LOGDEBUG, "%s, error", __FUNCTION__); + return 0; + } +#elif defined(USE_POLARSSL) size_t buf_size = maxsize; if(base64_encode((unsigned char *) output, &buf_size, input, length) == 0) { @@ -2461,7 +2683,13 @@ b64enc(const unsigned char *input, int length, char *output, int maxsize) return 1; } -#ifdef USE_POLARSSL +#if defined(USE_MBEDTLS) +typedef mbedtls_md5_context MD5_CTX; +#define MD5_Init(ctx) mbedtls_md5_init(ctx); mbedtls_md5_starts(ctx) +#define MD5_Update(ctx,data,len) mbedtls_md5_update(ctx,(unsigned char *)data,len) +#define MD5_Final(dig,ctx) mbedtls_md5_finish(ctx,dig); mbedtls_md5_free(ctx) + +#elif defined(USE_POLARSSL) #define MD5_CTX md5_context #define MD5_Init(ctx) md5_starts(ctx) #define MD5_Update(ctx,data,len) md5_update(ctx,(unsigned char *)data,len) diff --git a/plugins/obs-outputs/librtmp/rtmp_sys.h b/plugins/obs-outputs/librtmp/rtmp_sys.h index 098478ccd..1e8e4dc86 100644 --- a/plugins/obs-outputs/librtmp/rtmp_sys.h +++ b/plugins/obs-outputs/librtmp/rtmp_sys.h @@ -80,7 +80,76 @@ #include "rtmp.h" -#ifdef USE_POLARSSL +#if defined(USE_MBEDTLS) +#include +#include +#include +#include +#include + +#define my_dhm_P \ + "E4004C1F94182000103D883A448B3F80" \ + "2CE4B44A83301270002C20D0321CFD00" \ + "11CCEF784C26A400F43DFB901BCA7538" \ + "F2C6B176001CF5A0FD16D2C48B1D0C1C" \ + "F6AC8E1DA6BCC3B4E1F96B0564965300" \ + "FFA1D0B601EB2800F489AA512C4B248C" \ + "01F76949A60BB7F00A40B1EAB64BDD48" \ + "E8A700D60B7F1200FA8E77B0A979DABF" + +#define my_dhm_G "4" + +#define SSL_SET_SESSION(S,resume,timeout,ctx) mbedtls_ssl_set_session(S,ctx) + +typedef struct tls_ctx +{ + mbedtls_entropy_context entropy; + mbedtls_ctr_drbg_context ctr_drbg; + mbedtls_ssl_config conf; + mbedtls_ssl_session ssn; + mbedtls_x509_crt *cacert; + mbedtls_net_context net; +} tls_ctx; + +typedef struct tls_server_ctx +{ + mbedtls_ssl_config *conf; + mbedtls_ctr_drbg_context *ctr_drbg; + mbedtls_pk_context key; + mbedtls_x509_crt cert; +} tls_server_ctx; + +typedef tls_ctx *TLS_CTX; + +#define TLS_client(ctx,s) \ + s = malloc(sizeof(mbedtls_ssl_context));\ + mbedtls_ssl_init(s);\ + mbedtls_ssl_setup(s, &ctx->conf);\ + mbedtls_ssl_config_defaults(&ctx->conf, MBEDTLS_SSL_IS_CLIENT, MBEDTLS_SSL_TRANSPORT_STREAM, MBEDTLS_SSL_PRESET_DEFAULT);\ + mbedtls_ssl_conf_authmode(&ctx->conf, MBEDTLS_SSL_VERIFY_REQUIRED);\ + mbedtls_ssl_conf_rng(&ctx->conf, mbedtls_ctr_drbg_random, &ctx->ctr_drbg) + +#define TLS_server(ctx,s)\ + s = malloc(sizeof(mbedtls_ssl_context));\ + mbedtls_ssl_init(s);\ + mbedtls_ssl_setup(s, ctx->conf);\ + mbedtls_ssl_conf_endpoint(ctx->conf, MBEDTLS_SSL_IS_SERVER);\ + mbedtls_ssl_conf_authmode(ctx->conf, MBEDTLS_SSL_VERIFY_REQUIRED);\ + mbedtls_ssl_conf_rng(ctx->conf, mbedtls_ctr_drbg_random, ctx->ctr_drbg);\ + mbedtls_ssl_conf_own_cert(ctx->conf, &ctx->cert, &ctx->key);\ + mbedtls_ssl_conf_dh_param_bin(ctx->conf,\ + (const unsigned char *)my_dhm_P, strlen(my_dhm_P),\ + (const unsigned char *)my_dhm_G, strlen(my_dhm_G)) + +#define TLS_setfd(s,fd) mbedtls_ssl_set_bio(s, fd, mbedtls_net_send, mbedtls_net_recv, NULL) +#define TLS_connect(s) mbedtls_ssl_handshake(s) +#define TLS_accept(s) mbedtls_ssl_handshake(s) +#define TLS_read(s,b,l) mbedtls_ssl_read(s,(unsigned char *)b,l) +#define TLS_write(s,b,l) mbedtls_ssl_write(s,(unsigned char *)b,l) +#define TLS_shutdown(s) mbedtls_ssl_close_notify(s) +#define TLS_close(s) mbedtls_ssl_free(s); free(s) + +#elif defined(USE_POLARSSL) #include #include #include @@ -128,6 +197,7 @@ typedef struct tls_server_ctx #define TLS_shutdown(s) ssl_close_notify(s) #define TLS_close(s) ssl_free(s); free(s) + #elif defined(USE_GNUTLS) #include typedef struct tls_ctx diff --git a/plugins/obs-outputs/rtmp-stream.c b/plugins/obs-outputs/rtmp-stream.c index d33969e97..def27887e 100644 --- a/plugins/obs-outputs/rtmp-stream.c +++ b/plugins/obs-outputs/rtmp-stream.c @@ -438,6 +438,15 @@ static void set_output_error(struct rtmp_stream *stream) } #endif + // non platform-specific errors + if (!msg) { + switch (stream->rtmp.last_error_code) { + case -0x2700: + msg = obs_module_text("SSLCertVerifyFailed"); + break; + } + } + obs_output_set_last_error(stream->output, msg); } diff --git a/plugins/obs-outputs/rtmp-windows.c b/plugins/obs-outputs/rtmp-windows.c index 64e8ec0cb..cd0deb8c1 100644 --- a/plugins/obs-outputs/rtmp-windows.c +++ b/plugins/obs-outputs/rtmp-windows.c @@ -169,13 +169,13 @@ static enum data_ret write_data(struct rtmp_stream *stream, bool *can_write, size_t send_len = min(latency_packet_size, stream->write_buf_len); - ret = send(stream->rtmp.m_sb.sb_socket, + ret = RTMPSockBuf_Send(&stream->rtmp.m_sb, (const char *)stream->write_buf, - (int)send_len, 0); + (int)send_len); } else { - ret = send(stream->rtmp.m_sb.sb_socket, + ret = RTMPSockBuf_Send(&stream->rtmp.m_sb, (const char *)stream->write_buf, - (int)stream->write_buf_len, 0); + (int)stream->write_buf_len); } if (ret > 0) { -- GitLab