From 69402e275cd3715dee6fd4e0bd11fd111dc54a66 Mon Sep 17 00:00:00 2001
From: Stephen Birarda <commit@birarda.com>
Date: Tue, 14 Oct 2014 11:51:58 -0700
Subject: [PATCH] use DER format for generated keys in RSAKeypairGenerator

---
 interface/src/Application.cpp                 |  1 -
 .../networking/src/RSAKeypairGenerator.cpp    | 38 ++++++++++---------
 2 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp
index 4e5d3c9e96..9dc478b9ba 100644
--- a/interface/src/Application.cpp
+++ b/interface/src/Application.cpp
@@ -64,7 +64,6 @@
 #include <ParticlesScriptingInterface.h>
 #include <PerfStat.h>
 #include <ResourceCache.h>
-#include <RSAKeypairGenerator.h>
 #include <UserActivityLogger.h>
 #include <UUID.h>
 
diff --git a/libraries/networking/src/RSAKeypairGenerator.cpp b/libraries/networking/src/RSAKeypairGenerator.cpp
index 032bd62f06..ae61ac3eda 100644
--- a/libraries/networking/src/RSAKeypairGenerator.cpp
+++ b/libraries/networking/src/RSAKeypairGenerator.cpp
@@ -11,7 +11,7 @@
 
 #include <openssl/err.h>
 #include <openssl/rsa.h>
-#include <openssl/pem.h>
+#include <openssl/x509.h>
 
 #include <qdebug.h>
 
@@ -44,24 +44,28 @@ void RSAKeypairGenerator::generateKeypair() {
     BN_free(exponent);
     
     // grab the public key and private key from the file
-    BIO *privateKeyBIO = BIO_new(BIO_s_mem());
-    int privateWrite = PEM_write_bio_RSAPrivateKey(privateKeyBIO, keyPair, NULL, NULL, 0, NULL, NULL);
+    unsigned char* publicKeyDER = NULL;
+    int publicKeyLength = i2d_RSA_PUBKEY(keyPair, &publicKeyDER);
     
-    BIO *publicKeyBIO = BIO_new(BIO_s_mem());
-    int publicWrite = PEM_write_bio_RSAPublicKey(publicKeyBIO, keyPair);
+    unsigned char* privateKeyDER = NULL;
+    int privateKeyLength = i2d_RSAPrivateKey(keyPair, &privateKeyDER);
     
-    if (privateWrite == 0 || publicWrite == 0) {
-        // we had a error grabbing either the private or public key from the RSA
+    if (publicKeyLength <= 0 || privateKeyLength <= 0) {
+        qDebug() << "Error getting DER public or private key from RSA struct -" << ERR_get_error();
         
-        // bubble up our error
         emit errorGeneratingKeypair();
         
         // cleanup the RSA struct
         RSA_free(keyPair);
         
-        // cleanup the BIOs
-        BIO_free(privateKeyBIO);
-        BIO_free(publicKeyBIO);
+        // cleanup the public and private key DER data, if required
+        if (publicKeyLength > 0) {
+            delete publicKeyDER;
+        }
+        
+        if (privateKeyLength > 0) {
+            delete privateKeyDER;
+        }
         
         return;
     }
@@ -70,14 +74,12 @@ void RSAKeypairGenerator::generateKeypair() {
     // we can cleanup the RSA struct before we continue on
     RSA_free(keyPair);
     
-    char* publicKeyData;
-    int publicKeyLength = BIO_get_mem_data(publicKeyBIO, &publicKeyData);
+    QByteArray publicKeyArray(reinterpret_cast<char*>(publicKeyDER), publicKeyLength);
+    QByteArray privateKeyArray(reinterpret_cast<char*>(privateKeyDER), privateKeyLength);
     
-    char* privateKeyData;
-    int privateKeyLength = BIO_get_mem_data(privateKeyBIO, &privateKeyData);
-    
-    QByteArray publicKeyArray(publicKeyData, publicKeyLength);
-    QByteArray privateKeyArray(privateKeyData, privateKeyLength);
+    // cleanup the publicKeyDER and publicKeyDER data
+    delete publicKeyDER;
+    delete privateKeyDER;
     
     emit generatedKeypair(publicKeyArray, privateKeyArray);
 }
\ No newline at end of file