diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 8f30fef..6320f75 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -383,7 +383,8 @@
 			goto error;
 		allocated = 1;
 	}
-        if (PyObject_AsCharBuffer(bufobj, &buf, &buflen) < 0) {
+	buf = PyUnicode_AsStringAndSize(bufobj, &buflen);
+	if (buf == NULL) {
 		goto error;
 	}
 	if (buflen > size) {