diff --git a/src/cryptography/hazmat/backends/openssl/ciphers.py b/src/cryptography/hazmat/backends/openssl/ciphers.py
index 739ae19..cfd6c1b 100644
--- a/src/cryptography/hazmat/backends/openssl/ciphers.py
+++ b/src/cryptography/hazmat/backends/openssl/ciphers.py
@@ -13,6 +13,7 @@
 @utils.register_interface(ciphers.CipherContext)
 @utils.register_interface(ciphers.AEADCipherContext)
 @utils.register_interface(ciphers.AEADEncryptionContext)
+@utils.register_interface(ciphers.AEADDecryptionContext)
 class _CipherContext(object):
     _ENCRYPT = 1
     _DECRYPT = 0
@@ -78,21 +79,22 @@
                 len(iv_nonce), self._backend._ffi.NULL
             )
             self._backend.openssl_assert(res != 0)
-            if (
-                self._operation == self._DECRYPT and
-                self._backend._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 and
-                not self._backend._lib.CRYPTOGRAPHY_IS_LIBRESSL
-            ):
-                if mode.tag is None:
-                    raise NotImplementedError(
-                        "delayed passing of GCM tag requires OpenSSL >= 1.0.2."
-                        " To use this feature please update OpenSSL"
-                    )
+            if mode.tag is not None:
                 res = self._backend._lib.EVP_CIPHER_CTX_ctrl(
                     ctx, self._backend._lib.EVP_CTRL_GCM_SET_TAG,
                     len(mode.tag), mode.tag
                 )
                 self._backend.openssl_assert(res != 0)
+                self._tag = mode.tag
+            elif (
+                self._operation == self._DECRYPT and
+                self._backend._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 and
+                not self._backend._lib.CRYPTOGRAPHY_IS_LIBRESSL
+            ):
+                raise NotImplementedError(
+                    "delayed passing of GCM tag requires OpenSSL >= 1.0.2."
+                    " To use this feature please update OpenSSL"
+                )
 
         # pass key/iv
         res = self._backend._lib.EVP_CipherInit_ex(
@@ -146,21 +148,11 @@
         if (
             self._operation == self._DECRYPT and
             isinstance(self._mode, modes.ModeWithAuthenticationTag) and
-            (
-                not self._backend._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 or
-                self._backend._lib.CRYPTOGRAPHY_IS_LIBRESSL
-            )
+            self.tag is None
         ):
-            tag = self._mode.tag
-            if tag is None:
-                raise ValueError(
-                    "Authentication tag must be provided when decrypting."
-                )
-            res = self._backend._lib.EVP_CIPHER_CTX_ctrl(
-                self._ctx, self._backend._lib.EVP_CTRL_GCM_SET_TAG,
-                len(tag), tag
+            raise ValueError(
+                "Authentication tag must be provided when decrypting."
             )
-            self._backend.openssl_assert(res != 0)
 
         buf = self._backend._ffi.new("unsigned char[]", self._block_size_bytes)
         outlen = self._backend._ffi.new("int *")
@@ -203,6 +195,23 @@
         self._backend.openssl_assert(res == 1)
         return self._backend._ffi.buffer(buf)[:outlen[0]]
 
+    def finalize_with_tag(self, tag):
+        if (
+            self._backend._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 and
+            not self._backend._lib.CRYPTOGRAPHY_IS_LIBRESSL
+        ):
+            raise NotImplementedError(
+                "finalize_with_tag requires OpenSSL >= 1.0.2. To use this "
+                "method please update OpenSSL"
+            )
+        res = self._backend._lib.EVP_CIPHER_CTX_ctrl(
+            self._ctx, self._backend._lib.EVP_CTRL_GCM_SET_TAG,
+            len(tag), tag
+        )
+        self._backend.openssl_assert(res != 0)
+        self._tag = tag
+        return self.finalize()
+
     def authenticate_additional_data(self, data):
         outlen = self._backend._ffi.new("int *")
         res = self._backend._lib.EVP_CipherUpdate(
diff --git a/src/cryptography/hazmat/primitives/ciphers/__init__.py b/src/cryptography/hazmat/primitives/ciphers/__init__.py
index b5dd0ed..171b1c6 100644
--- a/src/cryptography/hazmat/primitives/ciphers/__init__.py
+++ b/src/cryptography/hazmat/primitives/ciphers/__init__.py
@@ -5,8 +5,8 @@
 from __future__ import absolute_import, division, print_function
 
 from cryptography.hazmat.primitives.ciphers.base import (
-    AEADCipherContext, AEADEncryptionContext, BlockCipherAlgorithm, Cipher,
-    CipherAlgorithm, CipherContext
+    AEADCipherContext, AEADDecryptionContext, AEADEncryptionContext,
+    BlockCipherAlgorithm, Cipher, CipherAlgorithm, CipherContext
 )
 
 
@@ -16,5 +16,6 @@
     "BlockCipherAlgorithm",
     "CipherContext",
     "AEADCipherContext",
+    "AEADDecryptionContext",
     "AEADEncryptionContext",
 ]
diff --git a/src/cryptography/hazmat/primitives/ciphers/base.py b/src/cryptography/hazmat/primitives/ciphers/base.py
index dd024fb..826e23e 100644
--- a/src/cryptography/hazmat/primitives/ciphers/base.py
+++ b/src/cryptography/hazmat/primitives/ciphers/base.py
@@ -221,17 +221,12 @@
         return data
 
     def finalize_with_tag(self, tag):
-        if (
-            self._ctx._backend.name == "openssl" and
-            self._ctx._backend._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_102 and
-            not self._ctx._backend._lib.CRYPTOGRAPHY_IS_LIBRESSL
-        ):
-            raise NotImplementedError(
-                "finalize_with_tag requires OpenSSL >= 1.0.2. To use this "
-                "method please update OpenSSL"
-            )
-        self._ctx._mode._set_tag(tag)
-        return self.finalize()
+        if self._ctx is None:
+            raise AlreadyFinalized("Context was already finalized.")
+        data = self._ctx.finalize_with_tag(tag)
+        self._tag = self._ctx.tag
+        self._ctx = None
+        return data
 
     def authenticate_additional_data(self, data):
         if self._ctx is None:
diff --git a/src/cryptography/hazmat/primitives/ciphers/modes.py b/src/cryptography/hazmat/primitives/ciphers/modes.py
index 5b28157..54670b7 100644
--- a/src/cryptography/hazmat/primitives/ciphers/modes.py
+++ b/src/cryptography/hazmat/primitives/ciphers/modes.py
@@ -164,9 +164,6 @@
         if not isinstance(initialization_vector, bytes):
             raise TypeError("initialization_vector must be bytes")
         self._initialization_vector = initialization_vector
-        self._set_tag(tag, min_tag_length)
-
-    def _set_tag(self, tag, min_tag_length=16):
         if tag is not None:
             if not isinstance(tag, bytes):
                 raise TypeError("tag must be bytes or None")
diff --git a/tests/hazmat/primitives/test_ciphers.py b/tests/hazmat/primitives/test_ciphers.py
index 7f51576..60faa0c 100644
--- a/tests/hazmat/primitives/test_ciphers.py
+++ b/tests/hazmat/primitives/test_ciphers.py
@@ -11,7 +11,7 @@
 
 import pytest
 
-from cryptography.exceptions import _Reasons
+from cryptography.exceptions import AlreadyFinalized, _Reasons
 from cryptography.hazmat.backends.interfaces import CipherBackend
 from cryptography.hazmat.primitives import ciphers
 from cryptography.hazmat.primitives.ciphers import modes
@@ -196,6 +196,28 @@
         assert res == len(pt)
         assert bytes(buf)[:res] == pt
 
+    @pytest.mark.supported(
+        only_if=lambda backend: backend.cipher_supported(
+            AES(b"\x00" * 16), modes.GCM(b"0" * 12)
+        ),
+        skip_message="Does not support AES GCM",
+    )
+    def test_finalize_with_tag_already_finalized(self, backend):
+        key = binascii.unhexlify(b"e98b72a9881a84ca6b76e0f43e68647a")
+        iv = binascii.unhexlify(b"8b23299fde174053f3d652ba")
+        encryptor = ciphers.Cipher(
+            AES(key), modes.GCM(iv), backend
+        ).encryptor()
+        ciphertext = encryptor.update(b"abc") + encryptor.finalize()
+
+        decryptor = ciphers.Cipher(
+            AES(key), modes.GCM(iv, tag=encryptor.tag), backend
+        ).decryptor()
+        decryptor.update(ciphertext)
+        decryptor.finalize()
+        with pytest.raises(AlreadyFinalized):
+            decryptor.finalize_with_tag(encryptor.tag)
+
     @pytest.mark.parametrize(
         "params",
         load_vectors_from_file(