Update SSL call sites to compile with both OpenSSL 1.1.0 and BoringSSL

OpenSSL is making a lot of data structure opaque, so we can no longer directly access internal data structure. Fortunately, API methods are provided for this purpose.

BoringSSL is sharing the same API.

Bug: webrtc:8817
Change-Id: Ia5090200f0e7c352f82e8191720ac4c14fbb5a85
Reviewed-on: https://webrtc-review.googlesource.com/47321
Commit-Queue: Tommi <tommi@webrtc.org>
Reviewed-by: Justin Uberti <juberti@webrtc.org>
Reviewed-by: Emad Omara <emadomara@webrtc.org>
Reviewed-by: Tommi <tommi@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21895}

rtc_base/openssladapter.cc

@@ -76,27 +76,33 @@ static long socket_ctrl(BIO* h, int cmd, long arg1, void* arg2);
 static int socket_new(BIO* h);
 static int socket_free(BIO* data);
 
-// TODO(davidben): This should be const once BoringSSL is assumed.
+static BIO_METHOD* BIO_socket_method() {
-static BIO_METHOD methods_socket = {
+  static BIO_METHOD* methods = [] {
-    BIO_TYPE_BIO, "socket",   socket_write, socket_read, socket_puts, 0,
+    BIO_METHOD* methods = BIO_meth_new(BIO_TYPE_BIO, "socket");
-    socket_ctrl,  socket_new, socket_free,  nullptr,
+    BIO_meth_set_write(methods, socket_write);
-};
+    BIO_meth_set_read(methods, socket_read);
-
+    BIO_meth_set_puts(methods, socket_puts);
-static BIO_METHOD* BIO_s_socket2() { return(&methods_socket); }
+    BIO_meth_set_ctrl(methods, socket_ctrl);
+    BIO_meth_set_create(methods, socket_new);
+    BIO_meth_set_destroy(methods, socket_free);
+    return methods;
+  }();
+  return methods;
+}
 
 static BIO* BIO_new_socket(rtc::AsyncSocket* socket) {
-  BIO* ret = BIO_new(BIO_s_socket2());
+  BIO* ret = BIO_new(BIO_socket_method());
   if (ret == nullptr) {
     return nullptr;
   }
-  ret->ptr = socket;
+  BIO_set_data(ret, socket);
   return ret;
 }
 
 static int socket_new(BIO* b) {
-  b->shutdown = 0;
+  BIO_set_shutdown(b, 0);
-  b->init = 1;
+  BIO_set_init(b, 1);
-  b->ptr = 0;
+  BIO_set_data(b, 0);
   return 1;
 }
 
@@ -109,7 +115,7 @@ static int socket_free(BIO* b) {
 static int socket_read(BIO* b, char* out, int outl) {
   if (!out)
     return -1;
-  rtc::AsyncSocket* socket = static_cast<rtc::AsyncSocket*>(b->ptr);
+  rtc::AsyncSocket* socket = static_cast<rtc::AsyncSocket*>(BIO_get_data(b));
   BIO_clear_retry_flags(b);
   int result = socket->Recv(out, outl, nullptr);
   if (result > 0) {
@@ -123,7 +129,7 @@ static int socket_read(BIO* b, char* out, int outl) {
 static int socket_write(BIO* b, const char* in, int inl) {
   if (!in)
     return -1;
-  rtc::AsyncSocket* socket = static_cast<rtc::AsyncSocket*>(b->ptr);
+  rtc::AsyncSocket* socket = static_cast<rtc::AsyncSocket*>(BIO_get_data(b));
   BIO_clear_retry_flags(b);
   int result = socket->Send(in, inl);
   if (result > 0) {
@@ -178,55 +184,10 @@ static void LogSslError() {
 
 namespace rtc {
 
-#ifndef OPENSSL_IS_BORINGSSL
-
-// This array will store all of the mutexes available to OpenSSL.
-static MUTEX_TYPE* mutex_buf = nullptr;
-
-static void locking_function(int mode, int n, const char * file, int line) {
-  if (mode & CRYPTO_LOCK) {
-    MUTEX_LOCK(mutex_buf[n]);
-  } else {
-    MUTEX_UNLOCK(mutex_buf[n]);
-  }
-}
-
-static unsigned long id_function() {  // NOLINT
-  // Use old-style C cast because THREAD_ID's type varies with the platform,
-  // in some cases requiring static_cast, and in others requiring
-  // reinterpret_cast.
-  return (unsigned long)THREAD_ID; // NOLINT
-}
-
-static CRYPTO_dynlock_value* dyn_create_function(const char* file, int line) {
-  CRYPTO_dynlock_value* value = new CRYPTO_dynlock_value;
-  if (!value)
-    return nullptr;
-  MUTEX_SETUP(value->mutex);
-  return value;
-}
-
-static void dyn_lock_function(int mode, CRYPTO_dynlock_value* l,
-                              const char* file, int line) {
-  if (mode & CRYPTO_LOCK) {
-    MUTEX_LOCK(l->mutex);
-  } else {
-    MUTEX_UNLOCK(l->mutex);
-  }
-}
-
-static void dyn_destroy_function(CRYPTO_dynlock_value* l,
-                                 const char* file, int line) {
-  MUTEX_CLEANUP(l->mutex);
-  delete l;
-}
-
-#endif  // #ifndef OPENSSL_IS_BORINGSSL
-
 VerificationCallback OpenSSLAdapter::custom_verify_callback_ = nullptr;
 
 bool OpenSSLAdapter::InitializeSSL(VerificationCallback callback) {
-  if (!InitializeSSLThread() || !SSL_library_init())
+  if (!SSL_library_init())
     return false;
 #if !defined(ADDRESS_SANITIZER) || !defined(WEBRTC_MAC) || defined(WEBRTC_IOS)
   // Loading the error strings crashes mac_asan.  Omit this debugging aid there.
@@ -239,41 +200,7 @@ bool OpenSSLAdapter::InitializeSSL(VerificationCallback callback) {
   return true;
 }
 
-bool OpenSSLAdapter::InitializeSSLThread() {
-  // BoringSSL is doing the locking internally, so the callbacks are not used
-  // in this case (and are no-ops anyways).
-#ifndef OPENSSL_IS_BORINGSSL
-  mutex_buf = new MUTEX_TYPE[CRYPTO_num_locks()];
-  if (!mutex_buf)
-    return false;
-  for (int i = 0; i < CRYPTO_num_locks(); ++i)
-    MUTEX_SETUP(mutex_buf[i]);
-
-  // we need to cast our id_function to return an unsigned long -- pthread_t is
-  // a pointer
-  CRYPTO_set_id_callback(id_function);
-  CRYPTO_set_locking_callback(locking_function);
-  CRYPTO_set_dynlock_create_callback(dyn_create_function);
-  CRYPTO_set_dynlock_lock_callback(dyn_lock_function);
-  CRYPTO_set_dynlock_destroy_callback(dyn_destroy_function);
-#endif  // #ifndef OPENSSL_IS_BORINGSSL
-  return true;
-}
-
 bool OpenSSLAdapter::CleanupSSL() {
-#ifndef OPENSSL_IS_BORINGSSL
-  if (!mutex_buf)
-    return false;
-  CRYPTO_set_id_callback(nullptr);
-  CRYPTO_set_locking_callback(nullptr);
-  CRYPTO_set_dynlock_create_callback(nullptr);
-  CRYPTO_set_dynlock_lock_callback(nullptr);
-  CRYPTO_set_dynlock_destroy_callback(nullptr);
-  for (int i = 0; i < CRYPTO_num_locks(); ++i)
-    MUTEX_CLEANUP(mutex_buf[i]);
-  delete [] mutex_buf;
-  mutex_buf = nullptr;
-#endif  // #ifndef OPENSSL_IS_BORINGSSL
   return true;
 }
 
@@ -439,9 +366,11 @@ int OpenSSLAdapter::BeginSSL() {
     }
   }
 
+#ifdef OPENSSL_IS_BORINGSSL
   // Set a couple common TLS extensions; even though we don't use them yet.
   SSL_enable_ocsp_stapling(ssl_);
   SSL_enable_signed_cert_timestamps(ssl_);
+#endif
 
   if (!alpn_protocols_.empty()) {
     std::string tls_alpn_string = TransformAlpnProtocols(alpn_protocols_);
@@ -896,7 +825,8 @@ bool OpenSSLAdapter::VerifyServerName(SSL* ssl, const char* host,
   GENERAL_NAMES* names = reinterpret_cast<GENERAL_NAMES*>(
       X509_get_ext_d2i(certificate, NID_subject_alt_name, nullptr, nullptr));
   if (names) {
-    for (size_t i = 0; i < sk_GENERAL_NAME_num(names); i++) {
+    for (size_t i = 0; i < static_cast<size_t>(sk_GENERAL_NAME_num(names));
+         i++) {
       const GENERAL_NAME* name = sk_GENERAL_NAME_value(names, i);
       if (name->type != GEN_DNS)
         continue;

rtc_base/openssladapter.h

@@ -31,7 +31,6 @@ class OpenSSLAdapterFactory;
 class OpenSSLAdapter : public SSLAdapter, public MessageHandler {
  public:
   static bool InitializeSSL(VerificationCallback callback);
-  static bool InitializeSSLThread();
   static bool CleanupSSL();
 
   explicit OpenSSLAdapter(AsyncSocket* socket,

rtc_base/openssldigest.cc

@@ -16,16 +16,18 @@
 namespace rtc {
 
 OpenSSLDigest::OpenSSLDigest(const std::string& algorithm) {
-  EVP_MD_CTX_init(&ctx_);
+  ctx_ = EVP_MD_CTX_new();
+  RTC_CHECK(ctx_ != nullptr);
+  EVP_MD_CTX_init(ctx_);
   if (GetDigestEVP(algorithm, &md_)) {
-    EVP_DigestInit_ex(&ctx_, md_, nullptr);
+    EVP_DigestInit_ex(ctx_, md_, nullptr);
   } else {
     md_ = nullptr;
   }
 }
 
 OpenSSLDigest::~OpenSSLDigest() {
-  EVP_MD_CTX_cleanup(&ctx_);
+  EVP_MD_CTX_destroy(ctx_);
 }
 
 size_t OpenSSLDigest::Size() const {
@@ -39,7 +41,7 @@ void OpenSSLDigest::Update(const void* buf, size_t len) {
   if (!md_) {
     return;
   }
-  EVP_DigestUpdate(&ctx_, buf, len);
+  EVP_DigestUpdate(ctx_, buf, len);
 }
 
 size_t OpenSSLDigest::Finish(void* buf, size_t len) {
@@ -47,8 +49,8 @@ size_t OpenSSLDigest::Finish(void* buf, size_t len) {
     return 0;
   }
   unsigned int md_len;
-  EVP_DigestFinal_ex(&ctx_, static_cast<unsigned char*>(buf), &md_len);
+  EVP_DigestFinal_ex(ctx_, static_cast<unsigned char*>(buf), &md_len);
-  EVP_DigestInit_ex(&ctx_, md_, nullptr);  // prepare for future Update()s
+  EVP_DigestInit_ex(ctx_, md_, nullptr);  // prepare for future Update()s
   RTC_DCHECK(md_len == Size());
   return md_len;
 }

rtc_base/openssldigest.h

@@ -41,7 +41,7 @@ class OpenSSLDigest : public MessageDigest {
                             size_t* len);
 
  private:
-  EVP_MD_CTX ctx_;
+  EVP_MD_CTX* ctx_ = nullptr;
   const EVP_MD* md_;
 };
 

rtc_base/opensslidentity.cc

@@ -207,11 +207,7 @@ OpenSSLKeyPair* OpenSSLKeyPair::GetReference() {
 }
 
 void OpenSSLKeyPair::AddReference() {
-#if defined(OPENSSL_IS_BORINGSSL)
   EVP_PKEY_up_ref(pkey_);
-#else
-  CRYPTO_add(&pkey_->references, 1, CRYPTO_LOCK_EVP_PKEY);
-#endif
 }
 
 std::string OpenSSLKeyPair::PrivateKeyToPEMString() const {
@@ -329,7 +325,7 @@ OpenSSLCertificate* OpenSSLCertificate::FromPEMString(
 // and before CleanupSSL.
 bool OpenSSLCertificate::GetSignatureDigestAlgorithm(
     std::string* algorithm) const {
-  int nid = OBJ_obj2nid(x509_->sig_alg->algorithm);
+  int nid = X509_get_signature_nid(x509_);
   switch (nid) {
     case NID_md5WithRSA:
     case NID_md5WithRSAEncryption:
@@ -448,11 +444,7 @@ void OpenSSLCertificate::ToDER(Buffer* der_buffer) const {
 
 void OpenSSLCertificate::AddReference() const {
   RTC_DCHECK(x509_ != nullptr);
-#if defined(OPENSSL_IS_BORINGSSL)
   X509_up_ref(x509_);
-#else
-  CRYPTO_add(&x509_->references, 1, CRYPTO_LOCK_X509);
-#endif
 }
 
 bool OpenSSLCertificate::operator==(const OpenSSLCertificate& other) const {

rtc_base/opensslstreamadapter.cc

@@ -162,26 +162,34 @@ static long stream_ctrl(BIO* h, int cmd, long arg1, void* arg2);
 static int stream_new(BIO* h);
 static int stream_free(BIO* data);
 
-static const BIO_METHOD methods_stream = {
+static BIO_METHOD* BIO_stream_method() {
-    BIO_TYPE_BIO, "stream",   stream_write, stream_read, stream_puts, 0,
+  static BIO_METHOD* method = [] {
-    stream_ctrl,  stream_new, stream_free,  nullptr,
+    BIO_METHOD* method = BIO_meth_new(BIO_TYPE_BIO, "stream");
-};
+    BIO_meth_set_write(method, stream_write);
+    BIO_meth_set_read(method, stream_read);
+    BIO_meth_set_puts(method, stream_puts);
+    BIO_meth_set_ctrl(method, stream_ctrl);
+    BIO_meth_set_create(method, stream_new);
+    BIO_meth_set_destroy(method, stream_free);
+    return method;
+  }();
+  return method;
+}
 
 static BIO* BIO_new_stream(StreamInterface* stream) {
-  // TODO(davidben): Remove the const_cast when BoringSSL is assumed.
+  BIO* ret = BIO_new(BIO_stream_method());
-  BIO* ret = BIO_new(const_cast<BIO_METHOD*>(&methods_stream));
   if (ret == nullptr)
     return nullptr;
-  ret->ptr = stream;
+  BIO_set_data(ret, stream);
   return ret;
 }
 
 // bio methods return 1 (or at least non-zero) on success and 0 on failure.
 
 static int stream_new(BIO* b) {
-  b->shutdown = 0;
+  BIO_set_shutdown(b, 0);
-  b->init = 1;
+  BIO_set_init(b, 1);
-  b->ptr = 0;
+  BIO_set_data(b, 0);
   return 1;
 }
 
@@ -194,7 +202,7 @@ static int stream_free(BIO* b) {
 static int stream_read(BIO* b, char* out, int outl) {
   if (!out)
     return -1;
-  StreamInterface* stream = static_cast<StreamInterface*>(b->ptr);
+  StreamInterface* stream = static_cast<StreamInterface*>(BIO_get_data(b));
   BIO_clear_retry_flags(b);
   size_t read;
   int error;
@@ -210,7 +218,7 @@ static int stream_read(BIO* b, char* out, int outl) {
 static int stream_write(BIO* b, const char* in, int inl) {
   if (!in)
     return -1;
-  StreamInterface* stream = static_cast<StreamInterface*>(b->ptr);
+  StreamInterface* stream = static_cast<StreamInterface*>(BIO_get_data(b));
   BIO_clear_retry_flags(b);
   size_t written;
   int error;

rtc_base/ssladapter.cc

@@ -30,10 +30,6 @@ bool InitializeSSL(VerificationCallback callback) {
   return OpenSSLAdapter::InitializeSSL(callback);
 }
 
-bool InitializeSSLThread() {
-  return OpenSSLAdapter::InitializeSSLThread();
-}
-
 bool CleanupSSL() {
   return OpenSSLAdapter::CleanupSSL();
 }

rtc_base/ssladapter.h

@@ -85,12 +85,9 @@ class SSLAdapter : public AsyncSocketAdapter {
 typedef bool (*VerificationCallback)(void* cert);
 
 // Call this on the main thread, before using SSL.
-// Call CleanupSSLThread when finished with SSL.
+// Call CleanupSSL when finished with SSL.
 bool InitializeSSL(VerificationCallback callback = nullptr);
 
-// Call to initialize additional threads.
-bool InitializeSSLThread();
-
 // Call to cleanup additional threads, and also the main thread.
 bool CleanupSSL();