diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py
index a0d062a..f9657f0 100644
--- a/Lib/email/encoders.py
+++ b/Lib/email/encoders.py
@@ -28,7 +28,7 @@
 
     Also, add an appropriate Content-Transfer-Encoding header.
     """
-    orig = msg.get_payload()
+    orig = msg.get_payload(decode=True)
     encdata = str(_bencode(orig), 'ascii')
     msg.set_payload(encdata)
     msg['Content-Transfer-Encoding'] = 'base64'
@@ -40,20 +40,16 @@
 
     Also, add an appropriate Content-Transfer-Encoding header.
     """
-    orig = msg.get_payload()
-    if isinstance(orig, str):
-        # If it is a string, the model data may have binary data encoded in via
-        # surrogateescape.  Convert back to bytes so we can CTE encode it.
-        orig = orig.encode('ascii', 'surrogateescape')
+    orig = msg.get_payload(decode=True)
     encdata = _qencode(orig)
-    msg.set_payload(encdata.decode('ascii', 'surrogateescape'))
+    msg.set_payload(encdata)
     msg['Content-Transfer-Encoding'] = 'quoted-printable'
 
 
 
 def encode_7or8bit(msg):
     """Set the Content-Transfer-Encoding header to 7bit or 8bit."""
-    orig = msg.get_payload()
+    orig = msg.get_payload(decode=True)
     if orig is None:
         # There's no payload.  For backwards compatibility we use 7bit
         msg['Content-Transfer-Encoding'] = '7bit'
@@ -75,16 +71,8 @@
             msg['Content-Transfer-Encoding'] = '8bit'
     else:
         msg['Content-Transfer-Encoding'] = '7bit'
-    if not isinstance(orig, str):
-        msg.set_payload(orig.decode('ascii', 'surrogateescape'))
 
 
 
 def encode_noop(msg):
     """Do nothing."""
-    # Well, not quite *nothing*: in Python3 we have to turn bytes into a string
-    # in our internal surrogateescaped form in order to keep the model
-    # consistent.
-    orig = msg.get_payload()
-    if not isinstance(orig, str):
-        msg.set_payload(orig.decode('ascii', 'surrogateescape'))
diff --git a/Lib/email/message.py b/Lib/email/message.py
index 3feab52..5020a03 100644
--- a/Lib/email/message.py
+++ b/Lib/email/message.py
@@ -275,6 +275,8 @@
         Optional charset sets the message's default character set.  See
         set_charset() for details.
         """
+        if isinstance(payload, bytes):
+            payload = payload.decode('ascii', 'surrogateescape')
         self._payload = payload
         if charset is not None:
             self.set_charset(charset)
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index 56be794..e11194b 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -593,6 +593,42 @@
             "attachment; filename*=utf-8''Fu%C3%9Fballer%20%5Bfilename%5D.ppt",
             msg['Content-Disposition'])
 
+    def test_binary_quopri_payload(self):
+        for charset in ('latin-1', 'ascii'):
+            msg = Message()
+            msg['content-type'] = 'text/plain; charset=%s' % charset
+            msg['content-transfer-encoding'] = 'quoted-printable'
+            msg.set_payload(b'foo=e6=96=87bar')
+            self.assertEqual(
+                msg.get_payload(decode=True),
+                b'foo\xe6\x96\x87bar',
+                'get_payload returns wrong result with charset %s.' % charset)
+
+    def test_binary_base64_payload(self):
+        for charset in ('latin-1', 'ascii'):
+            msg = Message()
+            msg['content-type'] = 'text/plain; charset=%s' % charset
+            msg['content-transfer-encoding'] = 'base64'
+            msg.set_payload(b'Zm9v5paHYmFy')
+            self.assertEqual(
+                msg.get_payload(decode=True),
+                b'foo\xe6\x96\x87bar',
+                'get_payload returns wrong result with charset %s.' % charset)
+
+    def test_binary_uuencode_payload(self):
+        for charset in ('latin-1', 'ascii'):
+            for encoding in ('x-uuencode', 'uuencode', 'uue', 'x-uue'):
+                msg = Message()
+                msg['content-type'] = 'text/plain; charset=%s' % charset
+                msg['content-transfer-encoding'] = encoding
+                msg.set_payload(b"begin 666 -\n)9F]OYI:'8F%R\n \nend\n")
+                self.assertEqual(
+                    msg.get_payload(decode=True),
+                    b'foo\xe6\x96\x87bar',
+                    str(('get_payload returns wrong result ',
+                         'with charset {0} and encoding {1}.')).\
+                        format(charset, encoding))
+
     def test_add_header_with_name_only_param(self):
         msg = Message()
         msg.add_header('Content-Disposition', 'inline', foo_bar=None)
diff --git a/Misc/NEWS b/Misc/NEWS
index d13b357..8e74408 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -66,6 +66,9 @@
 Library
 -------
 
+- Issue #18324: set_payload now correctly handles binary input.  This also
+  supersedes the previous fixes for #14360, #1717, and #16564.
+
 - Issue #17119: Fixed integer overflows when processing large strings and tuples
   in the tkinter module.