Handle non-ascii characters in identifiers
The style guide disallows this, but the JLS does not.
MOE_MIGRATED_REVID=137761780
diff --git a/java/com/google/turbine/parse/StreamLexer.java b/java/com/google/turbine/parse/StreamLexer.java
index f2850a6..4aef7a8 100644
--- a/java/com/google/turbine/parse/StreamLexer.java
+++ b/java/com/google/turbine/parse/StreamLexer.java
@@ -386,6 +386,10 @@
}
// does not fall through
default:
+ if (Character.isJavaIdentifierStart(ch)) {
+ // TODO(cushon): the style guide disallows non-ascii identifiers
+ return identifier();
+ }
throw error("unexpected input: %c", ch);
}
}
@@ -970,78 +974,14 @@
private Token identifier() {
readFrom();
eat();
- while (true) {
- switch (ch) {
- case 'a':
- case 'b':
- case 'c':
- case 'd':
- case 'e':
- case 'f':
- case 'g':
- case 'h':
- case 'i':
- case 'j':
- case 'k':
- case 'l':
- case 'm':
- case 'n':
- case 'o':
- case 'p':
- case 'q':
- case 'r':
- case 's':
- case 't':
- case 'u':
- case 'v':
- case 'w':
- case 'x':
- case 'y':
- case 'z':
- case 'A':
- case 'B':
- case 'C':
- case 'D':
- case 'E':
- case 'F':
- case 'G':
- case 'H':
- case 'I':
- case 'J':
- case 'K':
- case 'L':
- case 'M':
- case 'N':
- case 'O':
- case 'P':
- case 'Q':
- case 'R':
- case 'S':
- case 'T':
- case 'U':
- case 'V':
- case 'W':
- case 'X':
- case 'Y':
- case 'Z':
- case '_':
- case '$':
- case '0':
- case '1':
- case '2':
- case '3':
- case '4':
- case '5':
- case '6':
- case '7':
- case '8':
- case '9':
- eat();
- break;
- default:
- return makeIdent(stringValue());
+ // TODO(cushon): the style guide disallows non-ascii identifiers
+ while (Character.isJavaIdentifierPart(ch)) {
+ if (ch == ASCII_SUB && reader.done()) {
+ break;
}
+ eat();
}
+ return makeIdent(stringValue());
}
private Token makeIdent(String s) {
diff --git a/javatests/com/google/turbine/lower/LowerIntegrationTest.java b/javatests/com/google/turbine/lower/LowerIntegrationTest.java
index ff44533..cbb3190 100644
--- a/javatests/com/google/turbine/lower/LowerIntegrationTest.java
+++ b/javatests/com/google/turbine/lower/LowerIntegrationTest.java
@@ -248,6 +248,7 @@
"nonconst_unary_expression.test",
"type_anno_ambiguous.test",
"type_anno_ambiguous_param.test",
+ "unicode.test",
};
List<Object[]> tests =
ImmutableList.copyOf(testCases).stream().map(x -> new Object[] {x}).collect(toList());
diff --git a/javatests/com/google/turbine/lower/testdata/unicode.test b/javatests/com/google/turbine/lower/testdata/unicode.test
new file mode 100644
index 0000000..36a2864
--- /dev/null
+++ b/javatests/com/google/turbine/lower/testdata/unicode.test
@@ -0,0 +1,5 @@
+=== Test.java ===
+class Test {
+ public static final double π = 3.14;
+ public static final double _π = 3.14;
+}
\ No newline at end of file