Index: test/unit/testanalysis.py
===================================================================
--- test/unit/testanalysis.py	(.../trunk)	(revision 0)
+++ test/unit/testanalysis.py	(.../branches/re2c)	(revision 115)
@@ -0,0 +1,63 @@
+import unittest
+from lucille.analysis import Token
+from lucille.analysis.standard import standard_tokenizer
+
+
+class TestStandardTokenizer(unittest.TestCase):
+
+	def assertTokens(self, input, tokens):
+		tokenizer = standard_tokenizer(input)
+		for i, token in enumerate(tokenizer):
+			self.assertEquals(token, tokens[i])
+		self.assertRaises(StopIteration, tokenizer.next)
+
+	def testInputTypes(self):
+		self.assertRaises(TypeError, standard_tokenizer, 'asdf') # XXX should fix this
+		self.assertRaises(TypeError, standard_tokenizer, 1234)
+		self.assertRaises(TypeError, standard_tokenizer, (u'hi', u'there'))
+		self.assertRaises(TypeError, standard_tokenizer, Token(u'asdf', 0, 4))
+
+	def testEmpty(self):
+		self.assertTokens(u'', [])
+		self.assertTokens(u'  \n\t\r ', [])
+	
+	def testAll(self):
+		self.assertTokens(u"   ABC R2D2 3NF O'Reilly's U.S.S.R. AT&T Excite@Home d.j-c@localhost.localdomain localhost.localdomain 1.2e5  ", 
+				[Token(u'ABC', 3, 6, 'ALPHANUM'), 
+				 Token(u'R2D2', 7, 11, 'ALPHANUM'), 
+				 Token(u'3NF', 12, 15, 'ALPHANUM'), 
+				 Token(u"O'Reilly's", 16, 26, 'APOSTROPHE'), 
+				 Token(u'U.S.S.R.', 27, 35, 'ACRONYM'), 
+				 Token(u'AT&T', 36, 40, 'COMPANY'), 
+				 Token(u'Excite@Home', 41, 52, 'COMPANY'), 
+				 Token(u'd.j-c@localhost.localdomain', 53, 80, 'EMAIL'), 
+				 Token(u'localhost.localdomain', 81, 102, 'HOST'), 
+				 Token(u'1.2e5', 103, 108, 'HOST'), 
+				 Token(u'978-006074045-0', 109, 124, 'NUM')])
+	
+	def testLucene1068(self):
+		""" <https://issues.apache.org/jira/browse/LUCENE-1068>
+		Fix committed to Lucene trunk in r607161 for 2.3. For now we maintain 
+		the (incorrect) 2.1 behaviour. XXX change this later"""
+		self.assertTokens(u'www.apache.org www.apache.org. w.a.o w.a.o.', 
+				[Token(u'www.apache.org', 0, 14, 'HOST'), 
+				 Token(u'www.apache.org.', 15, 30, 'ACRONYM'), # should be HOST
+				 Token(u'w.a.o', 31, 36, 'HOST'), 
+				 Token(u'w.a.o.', 37, 43, 'ACRONYM')])
+	
+	def testEmbeddedNULs(self):
+		self.assertTokens(u'Many years \x00later, as he faced t\x00he firing squad,', 
+				[Token(u'Many', 0, 4, 'ALPHANUM'), 
+				 Token(u'years', 5, 10, 'ALPHANUM'), 
+				 Token(u'later', 12, 17, 'ALPHANUM'), 
+				 Token(u'as', 19, 21, 'ALPHANUM'), 
+				 Token(u'he', 22, 24, 'ALPHANUM'), 
+				 Token(u'faced', 25, 30, 'ALPHANUM'), 
+				 Token(u't', 31, 32, 'ALPHANUM'), 
+				 Token(u'he', 33, 35, 'ALPHANUM'), 
+				 Token(u'firing', 36, 42, 'ALPHANUM'), 
+				 Token(u'squad', 43, 48, 'ALPHANUM')])
+
+
+if __name__ == '__main__':
+	unittest.main()
Index: src/lucille/analysis/__init__.py
===================================================================
--- src/lucille/analysis/__init__.py	(.../trunk)	(revision 115)
+++ src/lucille/analysis/__init__.py	(.../branches/re2c)	(revision 115)
@@ -59,11 +59,6 @@
 class Token(object):
     """ <http://svn.apache.org/viewvc/lucene/java/tags/lucene_2_1_0/src/java/org/apache/lucene/analysis/Token.java> """
     
-    @classmethod
-    def from_ply_token(cls, ply_token):
-        return cls(ply_token.value, ply_token.lexpos, ply_token.lexpos + 
-                len(ply_token.value), ply_token.type)
-
     __slots__ = ('text', 'start_offset', 'end_offset', 'type')
 
     def __init__(self, text, start_offset, end_offset, type=None):
@@ -80,7 +75,16 @@
             return (u'(%s,%d,%d)' % (self.text, self.start_offset, 
                     self.end_offset)).encode('utf8')
 
+    def __repr__(self):
+        return 'Token%s' % self
 
+    def __eq__(self, other):
+        return (self.text == other.text and 
+                self.start_offset == other.start_offset and 
+                self.end_offset == other.end_offset and
+                self.type == other.type)
+
+
 def keyword_tokenizer(input):
     """ <http://svn.apache.org/viewvc/lucene/java/tags/lucene_2_1_0/src/java/org/apache/lucene/analysis/KeywordTokenizer.java>
 
Index: src/lucille/analysis/standard.py
===================================================================
--- src/lucille/analysis/standard.py	(.../trunk)	(revision 115)
+++ src/lucille/analysis/standard.py	(.../branches/re2c)	(revision 115)
@@ -18,78 +18,10 @@
 # You should have received a copy of the GNU General Public License along with 
 # this program. If not, see <http://www.gnu.org/licenses/>.
 
-import re
-from ply.lex import lex, TOKEN# XXX PLY is abysmally slow and not very powerful; need Unicode flex :-(
-
 from lucille.analysis import Token, stop_filter, lowercase_filter, ENGLISH_STOP_WORDS
+from lucille.analysis._standard import standard_tokenizer
 
 
-class _standard_tokenizer_def:
-    """ <http://svn.apache.org/viewvc/lucene/java/tags/lucene_2_1_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.jj>
-
-    This is not a real class, and should not be instantiated. """
-
-    tokens = ('ALPHANUM', 'NUM', 'HOST', 'EMAIL', 'COMPANY', 'ACRONYM', 
-            'APOSTROPHE', 'CJ')
-
-    DIGIT = ur'[\u0030-\u0039\u0660-\u0669\u06F0-\u06F9\u0966-\u096F\u09E6-\u09EF\u0A66-\u0A6F\u0AE6-\u0AEF\u0B66-\u0B6F\u0BE7-\u0BEF\u0C66-\u0C6F\u0CE6-\u0CEF\u0D66-\u0D6F\u0E50-\u0E59\u0ED0-\u0ED9\u1040-\u1049]'
-    LETTER = ur'[\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\uFFA0-\uFFDC]'
-    CJ = ur'[\u3040-\u318F\u3100-\u312F\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3300-\u337F\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\uFF65-\uFF9F]'
-    KOREAN = ur'[\uAC00-\uD7AF\u1100-\u11FF]'
-
-    ALPHA = ur'%(LETTER)s+' % locals()
-    HAS_DIGIT = ur'(%(LETTER)s|%(DIGIT)s)*%(DIGIT)s(%(LETTER)s|%(DIGIT)s)*' % locals()
-    P = r'[-_/.,]'
-
-    t_ALPHANUM = ur'(%(LETTER)s|%(DIGIT)s|%(KOREAN)s)+' % locals()
-
-    # Defining the following tokens as functions causes PLY to recognise them 
-    # in preference to any string rules (and in the order defined here).
-    # Longest-match-first can't be relied on because substituting earlier token 
-    # definitions causes the length of these tokens to differ.
-
-    @TOKEN(ur'%(t_ALPHANUM)s([.\-_]%(t_ALPHANUM)s)*@%(t_ALPHANUM)s([.\-]%(t_ALPHANUM)s)+' % locals())
-    def t_EMAIL(self, t): return t
-
-    @TOKEN(ur'%(t_ALPHANUM)s(\.%(t_ALPHANUM)s)+' % locals())
-    def t_HOST(self, t): return t
-
-    @TOKEN(ur'(%(ALPHA)s\.){2,}' % locals())
-    def t_ACRONYM(self, t): return t
-
-    @TOKEN(ur'(%(HAS_DIGIT)s%(P)s%(t_ALPHANUM)s(%(P)s%(HAS_DIGIT)s%(P)s%(t_ALPHANUM)s)+' \
-            ur'|%(t_ALPHANUM)s%(P)s%(HAS_DIGIT)s(%(P)s%(t_ALPHANUM)s%(P)s%(HAS_DIGIT)s)+' \
-            ur'|%(t_ALPHANUM)s(%(P)s%(HAS_DIGIT)s%(P)s%(t_ALPHANUM)s)+' \
-            ur'|%(HAS_DIGIT)s(%(P)s%(t_ALPHANUM)s%(P)s%(HAS_DIGIT)s)+' \
-            ur'|%(t_ALPHANUM)s%(P)s%(HAS_DIGIT)s' \
-            ur'|%(HAS_DIGIT)s%(P)s%(t_ALPHANUM)s' \
-            ur')' % locals())
-    def t_NUM(self, t): return t
-
-    @TOKEN(ur'%(ALPHA)s[&@]%(ALPHA)s' % locals())
-    def t_COMPANY(self, t): return t
-
-    @TOKEN(ur'%(ALPHA)s(\'%(ALPHA)s)+' % locals())
-    def t_APOSTROPHE(self, t): return t
-
-    t_ignore_NOISE = ur'.'
-
-    def t_error(self, token): assert False # can never actually happen due to t_ignore
-
-_standard_tokenizer_def.lexer = lex(object=_standard_tokenizer_def(), reflags=re.DOTALL)
-
-
-def standard_tokenizer(input):
-    """ <http://svn.apache.org/viewvc/lucene/java/tags/lucene_2_1_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java> """
-    lexer = _standard_tokenizer_def.lexer.clone()
-    lexer.input(input)
-    while True:
-        ply_token = lexer.token()
-        if ply_token is None:
-            raise StopIteration()
-        yield Token.from_ply_token(ply_token)
-
-
 def standard_filter(tokens):
     """ <http://svn.apache.org/viewvc/lucene/java/tags/lucene_2_1_0/src/java/org/apache/lucene/analysis/standard/StandardFilter.java>
 
Index: src/lucille/analysis/_standardmodule.re
===================================================================
--- src/lucille/analysis/_standardmodule.re	(.../trunk)	(revision 0)
+++ src/lucille/analysis/_standardmodule.re	(.../branches/re2c)	(revision 115)
@@ -0,0 +1,287 @@
+
+/* Copyright 2007 Dan Callaghan <djc@djc.id.au> */
+
+/*
+ * This file is part of Lucille, based on Apache Lucene 
+ * <http://lucene.apache.org>.
+ *
+ * Lucille is free software; you can redistribute it and/or modify it under the 
+ * terms of the GNU General Public License as published by the Free Software 
+ * Foundation; either version 3 of the License, or (at your option) any later 
+ * version.
+ *
+ * Lucille is distributed in the hope that it will be useful, but WITHOUT ANY 
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with 
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <Python.h>
+#include "structmember.h"
+
+
+static PyObject *alphanum_type = NULL;
+static PyObject *apostrophe_type = NULL;
+static PyObject *acronym_type = NULL;
+static PyObject *company_type = NULL;
+static PyObject *email_type = NULL;	
+static PyObject *host_type = NULL;
+static PyObject *num_type = NULL;
+static PyObject *cj_type = NULL;
+static PyObject *analysis_module = NULL;
+
+
+#define StandardTokenizer_doc "<http://svn.apache.org/viewvc/lucene/java/tags/lucene_2_1_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.jj>"
+
+typedef struct {
+    PyObject_HEAD
+	PyUnicodeObject *input;
+	Py_UNICODE *cursor;
+	Py_UNICODE *limit;
+} StandardTokenizer;
+
+static PyMemberDef StandardTokenizer_members[] = {
+	/*{"input", T_OBJECT_EX, offsetof(StandardTokenizer, input), RO, 
+		"input unicode"}, */
+	{NULL} /* sentinel */
+};
+
+static PyObject *StandardTokenizer_new(PyTypeObject *type, PyObject *args, 
+		PyObject *kwargs) {
+	StandardTokenizer *self;
+
+	if ((self = (StandardTokenizer *)type->tp_alloc(type, 0)) == NULL)
+		goto error;
+	self->input = NULL;
+	self->cursor = NULL;
+	self->limit = NULL;
+	return (PyObject *)self;
+
+error:
+	return NULL;
+}
+
+static int StandardTokenizer_init(StandardTokenizer *self, PyObject *args, 
+		PyObject *kwargs) {
+	PyUnicodeObject *input;
+
+	static char *kwlist[] = {"input", NULL};
+	/* XXX should coerce to unicode? or at least accept str ... */
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "U", kwlist,
+			&input))
+		goto error;
+
+	Py_INCREF(input);
+	Py_XDECREF(self->input);
+	self->input = input;
+
+	self->cursor = PyUnicode_AS_UNICODE(self->input);
+
+	/* XXX NOT We position self->limit one *beyond* the end of the string, because 
+	 * Python always keeps a NUL terminator at the end of the buffer and the 
+	 * re2c-generated code relies on this to detect the end of the buffer. */
+	self->limit = self->cursor + PyUnicode_GET_SIZE(self->input);
+
+	return 0;
+
+error:
+	return -1;
+}
+
+static void StandardTokenizer_dealloc(StandardTokenizer *self) {
+	Py_XDECREF(self->input);
+	self->ob_type->tp_free((PyObject *)self);
+}
+
+static PyObject *StandardTokenizer_iter(StandardTokenizer *self) {
+	Py_INCREF(self);
+	return (PyObject *)self; /* per iterator proto */
+}
+
+static PyObject *StandardTokenizer_iternext(StandardTokenizer *self) {
+
+	if (self->cursor < self->limit) {
+
+#define END_TOKEN(_type) type = _type; break;
+
+		Py_UNICODE *cursor = self->cursor;
+		Py_UNICODE *marker = NULL;
+		PyObject *type = NULL;
+		for (;;) {
+/*!re2c
+	re2c:define:YYCTYPE  = Py_UNICODE;
+	re2c:define:YYCURSOR = cursor;
+	re2c:define:YYMARKER = marker;
+	re2c:yyfill:enable   = 0;
+	re2c:indent:top      = 2;
+
+	LETTER = [\u0041-\u005a\u0061-\u007a\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u00ff\u0100-\u1fff\uffa0-\uffdc];
+	DIGIT = [\u0030-\u0039\u0660-\u0669\u06f0-\u06f9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be7-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0e50-\u0e59\u0ed0-\u0ed9\u1040-\u1049];
+	KOREAN = [\uac00-\ud7af\u1100-\u11ff];
+	ALPHA = LETTER+;
+	P = [-_/.,];
+	HAS_DIGIT = (LETTER|DIGIT)*DIGIT(LETTER|DIGIT)*;
+
+	ALPHANUM = (LETTER|DIGIT|KOREAN)+;
+	APOSTROPHE = ALPHA ("'" ALPHA)+;
+	ACRONYM = ALPHA "." (ALPHA ".")+;
+	COMPANY = ALPHA [&@] ALPHA;
+	EMAIL = ALPHANUM ([-._] ALPHANUM)* "@" ALPHANUM ([-.] ALPHANUM)+;
+	HOST = ALPHANUM ("." ALPHANUM)+;
+	NUM = (ALPHANUM P HAS_DIGIT
+       | HAS_DIGIT P ALPHANUM
+       | ALPHANUM (P HAS_DIGIT P ALPHANUM)+
+       | HAS_DIGIT (P ALPHANUM P HAS_DIGIT)+
+       | ALPHANUM P HAS_DIGIT (P ALPHANUM P HAS_DIGIT)+
+       | HAS_DIGIT P ALPHANUM (P HAS_DIGIT P ALPHANUM)+
+        );
+	CJ = [\u3040-\u318f\u3100-\u312f\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3300-\u337f\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff65-\uff9f];
+
+	ALPHANUM { END_TOKEN(alphanum_type); }
+	APOSTROPHE { END_TOKEN(apostrophe_type); }
+	ACRONYM { END_TOKEN(acronym_type); }
+	COMPANY { END_TOKEN(company_type); }
+	EMAIL { END_TOKEN(email_type); }
+	HOST { END_TOKEN(host_type); }
+	NUM { END_TOKEN(num_type); }
+	CJ { END_TOKEN(cj_type); }
+	[^] {
+		// discard everything else
+		self->cursor = cursor;
+		if (self->cursor < self->limit)
+			continue;
+		else
+			return NULL; // exhausted
+	}
+*/
+		}
+
+		/* create and return a new Token */
+		PyObject *token = NULL;
+		PyObject *u = PyUnicode_FromUnicode(self->cursor,
+				cursor - self->cursor);
+		if (u) {
+			token = PyObject_CallMethod(analysis_module, "Token", "OnnO", 
+					u, 
+					(Py_ssize_t) (self->cursor - 
+						PyUnicode_AS_UNICODE(self->input)), 
+					(Py_ssize_t) (cursor - PyUnicode_AS_UNICODE(self->input)), 
+					type);
+			Py_DECREF(u); /* "O" above increfs */
+		}
+		self->cursor = cursor;
+		return token;
+	} else {
+		return NULL; /* exhausted */
+	}
+}
+
+static PyMethodDef StandardTokenizer_methods[] = {
+	{NULL} /* sentinel */
+};
+
+static PyTypeObject StandardTokenizerType = {
+	PyObject_HEAD_INIT(NULL)
+	0,							/* ob_size */
+	"lucille.analysis._standard.standard_tokenizer",/* tp_name */
+	sizeof(StandardTokenizer),	/* tp_basicsize */
+	0,							/* tp_itemsize */
+	(destructor) StandardTokenizer_dealloc, /* tp_dealloc */
+	0,							/* tp_print */
+	0,							/* tp_getattr */
+	0,							/* tp_setattr */
+	0,							/* tp_compare */
+	0,							/* tp_repr */
+	0,							/* tp_as_number */
+	0,							/* tp_as_sequence */
+	0,							/* tp_as_mapping */
+	0,							/* tp_hash  */
+	0,							/* tp_call */
+	0,							/* tp_str */
+	0,							/* tp_getattro */
+	0,							/* tp_setattro */
+	0,							/* tp_as_buffer */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_ITER, /* tp_flags */
+	StandardTokenizer_doc,		/* tp_doc */
+	0,							/* tp_traverse */
+	0,							/* tp_clear */
+	0,							/* tp_richcompare */
+	0,							/* tp_weaklistoffset */
+	(getiterfunc) StandardTokenizer_iter, /* tp_iter */
+	(iternextfunc) StandardTokenizer_iternext, /* tp_iternext */
+	StandardTokenizer_methods,	/* tp_methods */
+	StandardTokenizer_members,	/* tp_members */
+	0,							/* tp_getset */
+	0,							/* tp_base */
+	0,							/* tp_dict */
+	0,							/* tp_descr_get */
+	0,							/* tp_descr_set */
+	0,							/* tp_dictoffset */
+	(initproc) StandardTokenizer_init, /* tp_init */
+	0,							/* tp_alloc */
+	StandardTokenizer_new,		/* tp_new */
+};
+
+static PyMethodDef _standard_methods[] = {
+	{NULL}  /* sentinel */
+};
+
+PyMODINIT_FUNC init_standard(void) {
+	PyObject *analysis_module_name = NULL;
+
+	/* create strings */
+	if (!(alphanum_type = PyString_FromString("ALPHANUM")))
+		goto error;
+	if (!(apostrophe_type = PyString_FromString("APOSTROPHE")))
+		goto error;
+	if (!(acronym_type = PyString_FromString("ACRONYM")))
+		goto error;
+	if (!(company_type = PyString_FromString("COMPANY")))
+		goto error;
+	if (!(email_type = PyString_FromString("EMAIL")))
+		goto error;
+	if (!(host_type = PyString_FromString("HOST")))
+		goto error;
+	if (!(num_type = PyString_FromString("NUM")))
+		goto error;
+	if (!(cj_type = PyString_FromString("CJ")))
+		goto error;
+
+	/* import lucille.analysis */
+	if (!(analysis_module_name = PyString_FromString("lucille.analysis")))
+		goto error;
+	if (!(analysis_module = PyImport_Import(analysis_module_name)))
+		goto error;
+	Py_DECREF(analysis_module_name);
+	analysis_module_name = NULL;
+
+	/* initialise this module */
+	PyObject *m;
+
+	StandardTokenizerType.tp_new = PyType_GenericNew;
+	if (PyType_Ready(&StandardTokenizerType) < 0)
+		goto error;
+
+	m = Py_InitModule("lucille.analysis._standard", _standard_methods);
+
+	Py_INCREF(&StandardTokenizerType);
+	PyModule_AddObject(m, "standard_tokenizer", (PyObject *)&StandardTokenizerType);
+
+	return;
+
+error:
+	Py_XDECREF(alphanum_type);
+	Py_XDECREF(apostrophe_type);
+	Py_XDECREF(acronym_type);
+	Py_XDECREF(company_type);
+	Py_XDECREF(email_type);
+	Py_XDECREF(host_type);
+	Py_XDECREF(num_type);
+	Py_XDECREF(cj_type);
+	Py_XDECREF(analysis_module);
+	Py_XDECREF(analysis_module_name);
+	return;
+}
Index: src/lucille/Makefile
===================================================================
--- src/lucille/Makefile	(.../trunk)	(revision 115)
+++ src/lucille/Makefile	(.../branches/re2c)	(revision 115)
@@ -1,11 +1,32 @@
-CFLAGS += -fno-strict-aliasing -fPIC $(shell python-config --includes)
+RE2C = re2c
+RE2CFLAGS = -b
+ifeq ($(CC), gcc)
+	RE2CFLAGS += -g
+endif
+PYTHON_IS_UCS4 = $(shell python -c 'import sys; print sys.maxunicode > 65536')
+ifeq ($(PYTHON_IS_UCS4), True)
+	RE2CFLAGS += -u
+else
+	RE2CFLAGS += -w
+endif
+CFLAGS += -fPIC $(shell python-config --includes)
 
+MODULES = _util.so analysis/_standard.so
+
 .PHONY: all
-all: _util.so
+all: $(MODULES)
 
-_util.so: _utilmodule.o
+%.so: %module.o
 	$(CC) -shared -o $@ $<
 
+%.c: %.re
+	$(RE2C) $(RE2CFLAGS) -o $@ $<
+
+_utilmodule.o: _utilmodule.c
+	$(CC) $(CFLAGS) -fno-strict-aliasing -o $@ -c $<
+
+analysis/_standardmodule.c: analysis/_standardmodule.re
+
 .PHONY: clean
 clean:
-	rm -f _util.so *.o
+	rm -f $(MODULES) *.o analysis/*.o analysis/_standardmodule.c
