Added shift and mask ops.
Allow numbers starting with a period.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ccc6d6d..009c193 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -85,6 +85,10 @@
"NOTEQUAL",
"LESSEQUAL",
"GREATEREQUAL",
+ "TILDE",
+ "CIRCUMFLEX",
+ "LEFTSHIFT",
+ "RIGHTSHIFT",
/* This table must match the #defines in token.h! */
"OP",
"<ERRORTOKEN>",
@@ -301,6 +305,8 @@
case '`': return BACKQUOTE;
case '{': return LBRACE;
case '}': return RBRACE;
+ case '^': return CIRCUMFLEX;
+ case '~': return TILDE;
default: return OP;
}
}
@@ -325,11 +331,13 @@
switch (c2) {
case '>': return NOTEQUAL;
case '=': return LESSEQUAL;
+ case '<': return LEFTSHIFT;
}
break;
case '>':
switch (c2) {
case '=': return GREATEREQUAL;
+ case '>': return RIGHTSHIFT;
}
break;
}
@@ -438,7 +446,7 @@
beginning or end of the file. (Will vi never die...?)
For Python it must be at the beginning of the file! */
int x;
- /* XXX The case to (unsigned char *) is needed by THINK C 3.0 */
+ /* XXX The cast to (unsigned char *) is needed by THINK C 3.0 */
if (sscanf(/*(unsigned char *)*/tok->cur,
" vi:set tabsize=%d:", &x) == 1 &&
x >= 1 && x <= 40) {
@@ -451,8 +459,10 @@
}
/* Check for EOF and errors now */
- if (c == EOF)
+ if (c == EOF) {
+ *p_start = *p_end = tok->cur;
return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
+ }
/* Identifier (most frequent token!) */
if (isalpha(c) || c == '_') {
@@ -473,6 +483,19 @@
return NEWLINE;
}
+ /* Period or number starting with period? */
+ if (c == '.') {
+ c = tok_nextc(tok);
+ if (isdigit(c)) {
+ goto fraction;
+ }
+ else {
+ tok_backup(tok, c);
+ *p_end = tok->cur;
+ return DOT;
+ }
+ }
+
/* Number */
if (isdigit(c)) {
if (c == '0') {