diff --git a/OpenSSL/crypto/crypto.c b/OpenSSL/crypto/crypto.c
index 8196be2..d9ce52e 100644
--- a/OpenSSL/crypto/crypto.c
+++ b/OpenSSL/crypto/crypto.c
@@ -61,6 +61,21 @@
     return nchars;
 }
 
+static PyObject *
+raise_current_error(void)
+{
+    if (PyErr_Occurred()) {
+        /*
+         * The python exception from callback is more informative than
+         * OpenSSL's error.
+         */
+        flush_error_queue();
+        return NULL;
+    }
+    exception_from_error_queue(crypto_Error);
+    return NULL;
+}
+
 static char crypto_load_privatekey_doc[] = "\n\
 Load a private key from a buffer\n\
 \n\
@@ -127,8 +142,7 @@
 
     if (pkey == NULL)
     {
-        exception_from_error_queue(crypto_Error);
-        return NULL;
+        return raise_current_error();
     }
 
     return (PyObject *)crypto_PKey_New(pkey, 1);
@@ -202,11 +216,6 @@
     {
         case X509_FILETYPE_PEM:
             ret = PEM_write_bio_PrivateKey(bio, pkey->pkey, cipher, NULL, 0, cb, cb_arg);
-            if (PyErr_Occurred())
-            {
-                BIO_free(bio);
-                return NULL;
-            }
             break;
 
         case X509_FILETYPE_ASN1:
@@ -228,8 +237,7 @@
     if (ret == 0)
     {
         BIO_free(bio);
-        exception_from_error_queue(crypto_Error);
-        return NULL;
+        return raise_current_error();
     }
 
     buf_len = BIO_get_mem_data(bio, &temp);
diff --git a/OpenSSL/test/test_crypto.py b/OpenSSL/test/test_crypto.py
index 56638e8..4438f8c 100644
--- a/OpenSSL/test/test_crypto.py
+++ b/OpenSSL/test/test_crypto.py
@@ -1990,6 +1990,18 @@
         self.assertTrue(isinstance(key, PKeyType))
 
 
+    def test_load_privatekey_passphrase_exception(self):
+        """
+        If the passphrase callback raises an exception, that exception is raised
+        by :py:obj:`load_privatekey`.
+        """
+        def cb(ignored):
+            raise ArithmeticError
+
+        self.assertRaises(ArithmeticError,
+            load_privatekey, FILETYPE_PEM, encryptedPrivateKeyPEM, cb)
+
+
     def test_load_privatekey_wrongPassphraseCallback(self):
         """
         :py:obj:`load_privatekey` raises :py:obj:`OpenSSL.crypto.Error` when it is passed an
@@ -2021,22 +2033,6 @@
         self.assertEqual(called, [False])
 
 
-    def test_load_privatekey_passphrase_exception(self):
-        """
-        An exception raised by the passphrase callback passed to
-        :py:obj:`load_privatekey` causes :py:obj:`OpenSSL.crypto.Error` to be raised.
-
-        This isn't as nice as just letting the exception pass through.  The
-        behavior might be changed to that eventually.
-        """
-        def broken(ignored):
-            raise RuntimeError("This is not working.")
-        self.assertRaises(
-            Error,
-            load_privatekey,
-            FILETYPE_PEM, encryptedPrivateKeyPEM, broken)
-
-
     def test_dump_privatekey_wrong_args(self):
         """
         :py:obj:`dump_privatekey` raises :py:obj:`TypeError` if called with the wrong number
@@ -2171,6 +2167,19 @@
         self.assertEqual(loadedKey.bits(), key.bits())
 
 
+    def test_dump_privatekey_passphrase_exception(self):
+        """
+        L{dump_privatekey} should not overwrite the exception raised
+        by the passphrase callback.
+        """
+        def cb(ignored):
+            raise ArithmeticError
+
+        key = load_privatekey(FILETYPE_PEM, cleartextPrivateKeyPEM)
+        self.assertRaises(ArithmeticError,
+            dump_privatekey, FILETYPE_PEM, key, "blowfish", cb)
+
+
     def test_load_pkcs7_data(self):
         """
         :py:obj:`load_pkcs7_data` accepts a PKCS#7 string and returns an instance of