diff --git a/cryptography/hazmat/backends/openssl/backend.py b/cryptography/hazmat/backends/openssl/backend.py
index e842f07..67b365f 100644
--- a/cryptography/hazmat/backends/openssl/backend.py
+++ b/cryptography/hazmat/backends/openssl/backend.py
@@ -177,6 +177,11 @@
 
         return self._ffi.buffer(buf)[:]
 
+    def _err_string(self, code):
+        err_buf = self._ffi.new("char[]", 256)
+        self._lib.ERR_error_string_n(code, err_buf, 256)
+        return self._ffi.string(err_buf, 256)[:]
+
     def _handle_error(self, mode):
         code = self._lib.ERR_get_error()
         if not code and isinstance(mode, GCM):
@@ -211,7 +216,10 @@
                     )
 
         raise InternalError(
-            "Unknown error code from OpenSSL, you should probably file a bug."
+            "Unknown error code {0} from OpenSSL, "
+            "you should probably file a bug. {1}".format(
+                code, self._err_string(code)
+            )
         )
 
 
diff --git a/tests/hazmat/backends/test_openssl.py b/tests/hazmat/backends/test_openssl.py
index b0a58c4..1e35322 100644
--- a/tests/hazmat/backends/test_openssl.py
+++ b/tests/hazmat/backends/test_openssl.py
@@ -116,6 +116,23 @@
 
         assert backend._lib.ERR_peek_error() == 0
 
+    def test_openssl_error_string(self):
+        backend._lib.ERR_put_error(
+            backend._lib.ERR_LIB_EVP,
+            backend._lib.EVP_F_EVP_DECRYPTFINAL_EX,
+            0,
+            b"test_openssl.py",
+            -1
+        )
+
+        with pytest.raises(InternalError) as exc:
+            backend._handle_error(None)
+
+        assert (
+            "digital envelope routines:"
+            "EVP_DecryptFinal_ex:digital envelope routines" in str(exc)
+        )
+
     def test_ssl_ciphers_registered(self):
         meth = backend._lib.TLSv1_method()
         ctx = backend._lib.SSL_CTX_new(meth)