[3.7] bpo-34922: Fix integer overflow in the digest() and hexdigest() methods (GH-9751) (GH-9798)

for the SHAKE algorithm in the hashlib module.
(cherry picked from commit 9b8c2e767643256202bb11456ba8665593b9a500)
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index 5b7f408..fc0649d 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -230,6 +230,20 @@
                 self.assertIsInstance(h.digest(), bytes)
                 self.assertEqual(hexstr(h.digest()), h.hexdigest())
 
+    def test_digest_length_overflow(self):
+        # See issue #34922
+        large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10)
+        for cons in self.hash_constructors:
+            h = cons()
+            if h.name not in self.shakes:
+                continue
+            for digest in h.digest, h.hexdigest:
+                with self.assertRaises((ValueError, OverflowError)):
+                    digest(-10)
+                for length in large_sizes:
+                    with self.assertRaises((ValueError, OverflowError)):
+                        digest(length)
+
     def test_name_attribute(self):
         for cons in self.hash_constructors:
             h = cons()