[Rework] Update snowball stemmer to 2.0 and remove all crap aside of UTF8

9 files changed, 2201 insertions, 2445 deletions

diff --git a/contrib/snowball/compiler/analyser.c b/contrib/snowball/compiler/analyser.c
index ed712f64f..dffa5552d 100644
--- a/contrib/snowball/compiler/analyser.c
+++ b/contrib/snowball/compiler/analyser.c
@@ -4,31 +4,53 @@
 #include <string.h>  /* memmove */
 #include "header.h"
 
+typedef enum {
+    e_token_omitted = 0,
+    e_unexpected_token = 1,
+    e_string_omitted = 2,
+    e_unexpected_token_in_among = 3,
+    /* For codes above here, report "after " t->previous_token after the error. */
+    e_unresolved_substring = 14,
+    e_not_allowed_inside_reverse = 15,
+    e_empty_grouping = 16,
+    e_already_backwards = 17,
+    e_empty_among = 18,
+    e_adjacent_bracketed_in_among = 19,
+    e_substring_preceded_by_substring = 20,
+    /* For codes below here, tokeniser->b is printed before the error. */
+    e_redeclared = 30,
+    e_undeclared = 31,
+    e_declared_as_different_mode = 32,
+    e_not_of_type_x = 33,
+    e_not_of_type_string_or_integer = 34,
+    e_misplaced = 35,
+    e_redefined = 36,
+    e_misused = 37
+} error_code;
+
 /* recursive usage: */
 
 static void read_program_(struct analyser * a, int terminator);
 static struct node * read_C(struct analyser * a);
-static struct node * C_style(struct analyser * a, char * s, int token);
-
+static struct node * C_style(struct analyser * a, const char * s, int token);
 
-static void fault(int n) { fprintf(stderr, "fault %d\n", n); exit(1); }
 
 static void print_node_(struct node * p, int n, const char * s) {
 
     int i;
     for (i = 0; i < n; i++) fputs(i == n - 1 ? s : "  ", stdout);
     printf("%s ", name_of_token(p->type));
-    unless (p->name == 0) report_b(stdout, p->name->b);
-    unless (p->literalstring == 0) {
+    if (p->name) report_b(stdout, p->name->b);
+    if (p->literalstring) {
         printf("'");
         report_b(stdout, p->literalstring);
         printf("'");
     }
     printf("\n");
-    unless (p->AE == 0) print_node_(p->AE, n+1, "# ");
-    unless (p->left == 0) print_node_(p->left, n+1, "  ");
-    unless (p->right == 0) print_node_(p->right, n, "  ");
-    if (p->aux != 0) print_node_(p->aux, n+1, "@ ");
+    if (p->AE) print_node_(p->AE, n+1, "# ");
+    if (p->left) print_node_(p->left, n+1, "  ");
+    if (p->aux) print_node_(p->aux, n+1, "@ ");
+    if (p->right) print_node_(p->right, n, "  ");
 }
 
 extern void print_program(struct analyser * a) {
@@ -52,22 +74,37 @@ static struct node * new_node(struct analyser * a, int type) {
 
 static const char * name_of_mode(int n) {
     switch (n) {
-         default: fault(0);
-         case m_backward: return "string backward";
-         case m_forward:  return "string forward";
-    /*   case m_integer:  return "integer";  */
+        case m_backward: return "string backward";
+        case m_forward:  return "string forward";
+    /*  case m_integer:  return "integer";  */
     }
+    fprintf(stderr, "Invalid mode %d in name_of_mode()\n", n);
+    exit(1);
 }
 
 static const char * name_of_type(int n) {
     switch (n) {
-         default: fault(1);
-         case 's': return "string";
-         case 'i': return "integer";
-         case 'r': return "routine";
-         case 'R': return "routine or grouping";
-         case 'g': return "grouping";
+        case 's': return "string";
+        case 'i': return "integer";
+        case 'r': return "routine";
+        case 'R': return "routine or grouping";
+        case 'g': return "grouping";
+    }
+    fprintf(stderr, "Invalid type %d in name_of_type()\n", n);
+    exit(1);
+}
+
+static const char * name_of_name_type(int code) {
+    switch (code) {
+        case t_string: return "string";
+        case t_boolean: return "boolean";
+        case t_integer: return "integer";
+        case t_routine: return "routine";
+        case t_external: return "external";
+        case t_grouping: return "grouping";
     }
+    fprintf(stderr, "Invalid type code %d in name_of_name_type()\n", code);
+    exit(1);
 }
 
 static void count_error(struct analyser * a) {
@@ -76,86 +113,78 @@ static void count_error(struct analyser * a) {
     t->error_count++;
 }
 
-static void error2(struct analyser * a, int n, int x) {
+static void error2(struct analyser * a, error_code n, int x) {
     struct tokeniser * t = a->tokeniser;
     count_error(a);
     fprintf(stderr, "%s:%d: ", t->file, t->line_number);
-    if (n >= 30) report_b(stderr, t->b);
+    if ((int)n >= (int)e_redeclared) report_b(stderr, t->b);
     switch (n) {
-        case 0:
+        case e_token_omitted:
             fprintf(stderr, "%s omitted", name_of_token(t->omission)); break;
-        case 3:
+        case e_unexpected_token_in_among:
             fprintf(stderr, "in among(...), ");
-        case 1:
+            /* fall through */
+        case e_unexpected_token:
             fprintf(stderr, "unexpected %s", name_of_token(t->token));
             if (t->token == c_number) fprintf(stderr, " %d", t->number);
             if (t->token == c_name) {
                 fprintf(stderr, " ");
                 report_b(stderr, t->b);
             } break;
-        case 2:
+        case e_string_omitted:
             fprintf(stderr, "string omitted"); break;
 
-        case 14:
+        case e_unresolved_substring:
             fprintf(stderr, "unresolved substring on line %d", x); break;
-        case 15:
+        case e_not_allowed_inside_reverse:
             fprintf(stderr, "%s not allowed inside reverse(...)", name_of_token(t->token)); break;
-        case 16:
+        case e_empty_grouping:
             fprintf(stderr, "empty grouping"); break;
-        case 17:
+        case e_already_backwards:
             fprintf(stderr, "backwards used when already in this mode"); break;
-        case 18:
+        case e_empty_among:
             fprintf(stderr, "empty among(...)"); break;
-        case 19:
+        case e_adjacent_bracketed_in_among:
             fprintf(stderr, "two adjacent bracketed expressions in among(...)"); break;
-        case 20:
+        case e_substring_preceded_by_substring:
             fprintf(stderr, "substring preceded by another substring on line %d", x); break;
 
-        case 30:
+        case e_redeclared:
             fprintf(stderr, " re-declared"); break;
-        case 31:
+        case e_undeclared:
             fprintf(stderr, " undeclared"); break;
-        case 32:
+        case e_declared_as_different_mode:
             fprintf(stderr, " declared as %s mode; used as %s mode",
                             name_of_mode(a->mode), name_of_mode(x)); break;
-        case 33:
+        case e_not_of_type_x:
             fprintf(stderr, " not of type %s", name_of_type(x)); break;
-        case 34:
+        case e_not_of_type_string_or_integer:
             fprintf(stderr, " not of type string or integer"); break;
-        case 35:
+        case e_misplaced:
             fprintf(stderr, " misplaced"); break;
-        case 36:
+        case e_redefined:
             fprintf(stderr, " redefined"); break;
-        case 37:
+        case e_misused:
             fprintf(stderr, " mis-used as %s mode",
                             name_of_mode(x)); break;
-        default:
-            fprintf(stderr, " error %d", n); break;
-
     }
-    if (n <= 13 && t->previous_token > 0)
+    if ((int)n < (int)e_unresolved_substring && t->previous_token > 0)
         fprintf(stderr, " after %s", name_of_token(t->previous_token));
     fprintf(stderr, "\n");
 }
 
-static void error(struct analyser * a, int n) { error2(a, n, 0); }
-
-static void error3(struct analyser * a, struct node * p, symbol * b) {
-    count_error(a);
-    fprintf(stderr, "%s:%d: among(...) has repeated string '", a->tokeniser->file, p->line_number);
-    report_b(stderr, b);
-    fprintf(stderr, "'\n");
-}
+static void error(struct analyser * a, error_code n) { error2(a, n, 0); }
 
 static void error4(struct analyser * a, struct name * q) {
     count_error(a);
+    fprintf(stderr, "%s:%d: ", a->tokeniser->file, q->used->line_number);
     report_b(stderr, q->b);
     fprintf(stderr, " undefined\n");
 }
 
 static void omission_error(struct analyser * a, int n) {
     a->tokeniser->omission = n;
-    error(a, 0);
+    error(a, e_token_omitted);
 }
 
 static int check_token(struct analyser * a, int code) {
@@ -169,71 +198,120 @@ static int get_token(struct analyser * a, int code) {
     read_token(t);
     {
         int x = check_token(a, code);
-        unless (x) t->token_held = true;
+        if (!x) t->token_held = true;
         return x;
     }
 }
 
 static struct name * look_for_name(struct analyser * a) {
-    struct name * p = a->names;
     symbol * q = a->tokeniser->b;
-    repeat {
-        if (p == 0) return 0;
-        {   symbol * b = p->b;
-            int n = SIZE(b);
-            if (n == SIZE(q) && memcmp(q, b, n * sizeof(symbol)) == 0) {
-                p->referenced = true;
-                return p;
-            }
+    struct name * p;
+    for (p = a->names; p; p = p->next) {
+        symbol * b = p->b;
+        int n = SIZE(b);
+        if (n == SIZE(q) && memcmp(q, b, n * sizeof(symbol)) == 0) {
+            p->referenced = true;
+            return p;
         }
-        p = p->next;
     }
+    return 0;
 }
 
 static struct name * find_name(struct analyser * a) {
     struct name * p = look_for_name(a);
-    if (p == 0) error(a, 31);
+    if (p == 0) error(a, e_undeclared);
     return p;
 }
 
 static void check_routine_mode(struct analyser * a, struct name * p, int mode) {
     if (p->mode < 0) p->mode = mode; else
-    unless (p->mode == mode) error2(a, 37, mode);
+    if (p->mode != mode) error2(a, e_misused, mode);
 }
 
 static void check_name_type(struct analyser * a, struct name * p, int type) {
     switch (type) {
-        case 's': if (p->type == t_string) return; break;
-        case 'i': if (p->type == t_integer) return; break;
-        case 'b': if (p->type == t_boolean) return; break;
-        case 'R': if (p->type == t_grouping) return;
-        case 'r': if (p->type == t_routine || p->type == t_external) return; break;
-        case 'g': if (p->type == t_grouping) return; break;
+        case 's':
+            if (p->type == t_string) return;
+            break;
+        case 'i':
+            if (p->type == t_integer) return;
+            break;
+        case 'b':
+            if (p->type == t_boolean) return;
+            break;
+        case 'R':
+            if (p->type == t_grouping) return;
+            /* FALLTHRU */
+        case 'r':
+            if (p->type == t_routine || p->type == t_external) return;
+            break;
+        case 'g':
+            if (p->type == t_grouping) return;
+            break;
     }
-    error2(a, 33, type);
+    error2(a, e_not_of_type_x, type);
 }
 
 static void read_names(struct analyser * a, int type) {
     struct tokeniser * t = a->tokeniser;
-    unless (get_token(a, c_bra)) return;
-    repeat {
-        if (read_token(t) != c_name) break;
-        if (look_for_name(a) != 0) error(a, 30); else {
-            NEW(name, p);
-            p->b = copy_b(t->b);
-            p->type = type;
-            p->mode = -1; /* routines, externals */
-            p->count = a->name_count[type];
-            p->referenced = false;
-            p->used = false;
-            p->grouping = 0;
-            p->definition = 0;
-            a->name_count[type] ++;
-            p->next = a->names;
-            a->names = p;
-        }
-    }
-    unless (check_token(a, c_ket)) t->token_held = true;
+    if (!get_token(a, c_bra)) return;
+    while (true) {
+        int token = read_token(t);
+        switch (token) {
+            case c_len: {
+                /* Context-sensitive token - once declared as a name, it loses
+                 * its special meaning, for compatibility with older versions
+                 * of snowball.
+                 */
+                static const symbol c_len_lit[] = {
+                    'l', 'e', 'n'
+                };
+                MOVE_TO_B(t->b, c_len_lit);
+                goto handle_as_name;
+            }
+            case c_lenof: {
+                /* Context-sensitive token - once declared as a name, it loses
+                 * its special meaning, for compatibility with older versions
+                 * of snowball.
+                 */
+                static const symbol c_lenof_lit[] = {
+                    'l', 'e', 'n', 'o', 'f'
+                };
+                MOVE_TO_B(t->b, c_lenof_lit);
+                goto handle_as_name;
+            }
+            case c_name:
+handle_as_name:
+                if (look_for_name(a) != 0) error(a, e_redeclared); else {
+                    NEW(name, p);
+                    p->b = copy_b(t->b);
+                    p->type = type;
+                    p->mode = -1; /* routines, externals */
+                    /* We defer assigning counts until after we've eliminated
+                     * variables whose values are never used. */
+                    p->count = -1;
+                    p->referenced = false;
+                    p->used_in_among = false;
+                    p->used = 0;
+                    p->value_used = false;
+                    p->initialised = false;
+                    p->used_in_definition = false;
+                    p->local_to = 0;
+                    p->grouping = 0;
+                    p->definition = 0;
+                    p->declaration_line_number = t->line_number;
+                    p->next = a->names;
+                    a->names = p;
+                    if (token != c_name) {
+                        disable_token(t, token);
+                    }
+                }
+                break;
+            default:
+                if (!check_token(a, c_ket)) t->token_held = true;
+                return;
+        }
+    }
 }
 
 static symbol * new_literalstring(struct analyser * a) {
@@ -259,7 +337,7 @@ static int read_AE_test(struct analyser * a) {
         case c_ge:
         case c_ls:
         case c_le: return t->token;
-        default: error(a, 1); t->token_held = true; return c_eq;
+        default: error(a, e_unexpected_token); t->token_held = true; return c_eq;
     }
 }
 
@@ -271,11 +349,23 @@ static int binding(int t) {
     }
 }
 
+static void mark_used_in(struct analyser * a, struct name * q, struct node * p) {
+    if (!q->used) {
+        q->used = p;
+        q->local_to = a->program_end->name;
+    } else if (q->local_to) {
+        if (q->local_to != a->program_end->name) {
+            /* Used in more than one routine/external. */
+            q->local_to = NULL;
+        }
+    }
+}
+
 static void name_to_node(struct analyser * a, struct node * p, int type) {
     struct name * q = find_name(a);
-    unless (q == 0) {
+    if (q) {
         check_name_type(a, q, type);
-        q->used = true;
+        mark_used_in(a, q, p);
     }
     p->name = q;
 }
@@ -286,8 +376,18 @@ static struct node * read_AE(struct analyser * a, int B) {
     struct node * q;
     switch (read_token(t)) {
         case c_minus: /* monadic */
+            q = read_AE(a, 100);
+            if (q->type == c_neg) {
+                /* Optimise away double negation, which avoids generators
+                 * having to worry about generating "--" (decrement operator
+                 * in many languages).
+                 */
+                p = q->right;
+                /* Don't free q, it's in the linked list a->nodes. */
+                break;
+            }
             p = new_node(a, c_neg);
-            p->right = read_AE(a, 100);
+            p->right = q;
             break;
         case c_bra:
             p = read_AE(a, 0);
@@ -296,11 +396,15 @@ static struct node * read_AE(struct analyser * a, int B) {
         case c_name:
             p = new_node(a, c_name);
             name_to_node(a, p, 'i');
+            if (p->name) p->name->value_used = true;
             break;
         case c_maxint:
         case c_minint:
+            a->int_limits_used = true;
+            /* fall through */
         case c_cursor:
         case c_limit:
+        case c_len:
         case c_size:
             p = new_node(a, t->token);
             break;
@@ -308,18 +412,19 @@ static struct node * read_AE(struct analyser * a, int B) {
             p = new_node(a, c_number);
             p->number = t->number;
             break;
+        case c_lenof:
         case c_sizeof:
-            p = C_style(a, "s", c_sizeof);
+            p = C_style(a, "s", t->token);
             break;
         default:
-            error(a, 1);
+            error(a, e_unexpected_token);
             t->token_held = true;
             return 0;
     }
-    repeat {
+    while (true) {
         int token = read_token(t);
         int b = binding(token);
-        unless (binding(token) > B) {
+        if (binding(token) <= B) {
             t->token_held = true;
             return p;
         }
@@ -335,14 +440,11 @@ static struct node * read_C_connection(struct analyser * a, struct node * q, int
     struct node * p = new_node(a, op);
     struct node * p_end = q;
     p->left = q;
-    repeat {
+    do {
         q = read_C(a);
         p_end->right = q; p_end = q;
-        if (read_token(t) != op) {
-            t->token_held = true;
-            break;
-        }
-    }
+    } while (read_token(t) == op);
+    t->token_held = true;
     return p;
 }
 
@@ -350,14 +452,14 @@ static struct node * read_C_list(struct analyser * a) {
     struct tokeniser * t = a->tokeniser;
     struct node * p = new_node(a, c_bra);
     struct node * p_end = 0;
-    repeat {
+    while (true) {
         int token = read_token(t);
         if (token == c_ket) return p;
         if (token < 0) { omission_error(a, c_ket); return p; }
         t->token_held = true;
         {
             struct node * q = read_C(a);
-            repeat {
+            while (true) {
                 token = read_token(t);
                 if (token != c_and && token != c_or) {
                     t->token_held = true;
@@ -371,10 +473,10 @@ static struct node * read_C_list(struct analyser * a) {
     }
 }
 
-static struct node * C_style(struct analyser * a, char * s, int token) {
+static struct node * C_style(struct analyser * a, const char * s, int token) {
     int i;
     struct node * p = new_node(a, token);
-    for (i = 0; s[i] != 0; i++) switch(s[i]) {
+    for (i = 0; s[i] != 0; i++) switch (s[i]) {
         case 'C':
             p->left = read_C(a); continue;
         case 'D':
@@ -388,7 +490,7 @@ static struct node * C_style(struct analyser * a, char * s, int token) {
                 int str_token = read_token(a->tokeniser);
                 if (str_token == c_name) name_to_node(a, p, 's'); else
                 if (str_token == c_literalstring) p->literalstring = new_literalstring(a);
-                else error(a, 2);
+                else error(a, e_string_omitted);
             }
             continue;
         case 'b':
@@ -408,7 +510,7 @@ static struct node * read_literalstring(struct analyser * a) {
 
 static void reverse_b(symbol * b) {
     int i = 0; int j = SIZE(b) - 1;
-    until (i >= j) {
+    while (i < j) {
         int ch1 = b[i]; int ch2 = b[j];
         b[i++] = ch2; b[j--] = ch1;
     }
@@ -423,7 +525,74 @@ static int compare_amongvec(const void *pv, const void *qv) {
     int i;
     for (i = 0; i < smaller_size; i++)
         if (b_p[i] != b_q[i]) return b_p[i] - b_q[i];
-    return p_size - q_size;
+    if (p_size - q_size)
+        return p_size - q_size;
+    return p->line_number - q->line_number;
+}
+
+#define PTR_NULL_CHECK(P, Q) do {\
+        if ((Q) == NULL) {\
+            if ((P) != NULL) return 1;\
+        } else {\
+            if ((P) == NULL) return -1;\
+        }\
+    } while (0)
+
+static int compare_node(const struct node *p, const struct node *q) {
+    PTR_NULL_CHECK(p, q);
+    if (q == NULL) {
+        /* p must be NULL too. */
+        return 0;
+    }
+
+    if (p->type != q->type) return p->type > q->type ? 1 : -1;
+    if (p->mode != q->mode) return p->mode > q->mode ? 1 : -1;
+    if (p->type == c_number) {
+        if (p->number != q->number)
+            return p->number > q->number ? 1 : -1;
+    }
+
+    PTR_NULL_CHECK(p->left, q->left);
+    if (p->left) {
+        int r = compare_node(p->left, q->left);
+        if (r != 0) return r;
+    }
+
+    PTR_NULL_CHECK(p->AE, q->AE);
+    if (p->AE) {
+        int r = compare_node(p->AE, q->AE);
+        if (r != 0) return r;
+    }
+
+    PTR_NULL_CHECK(p->aux, q->aux);
+    if (p->aux) {
+        int r = compare_node(p->aux, q->aux);
+        if (r != 0) return r;
+    }
+
+    PTR_NULL_CHECK(p->name, q->name);
+    if (p->name) {
+        int r;
+        if (SIZE(p->name->b) != SIZE(q->name->b)) {
+            return SIZE(p->name->b) - SIZE(q->name->b);
+        }
+        r = memcmp(p->name->b, q->name->b,
+                   SIZE(p->name->b) * sizeof(symbol));
+        if (r != 0) return r;
+    }
+
+    PTR_NULL_CHECK(p->literalstring, q->literalstring);
+    if (p->literalstring) {
+        int r;
+        if (SIZE(p->literalstring) != SIZE(q->literalstring)) {
+            return SIZE(p->literalstring) - SIZE(q->literalstring);
+        }
+        r = memcmp(p->literalstring, q->literalstring,
+                   SIZE(p->literalstring) * sizeof(symbol));
+        if (r != 0) return r;
+    }
+
+    return compare_node(p->right, q->right);
 }
 
 static void make_among(struct analyser * a, struct node * p, struct node * substring) {
@@ -443,38 +612,84 @@ static void make_among(struct analyser * a, struct node * p, struct node * subst
     x->next = 0;
     x->b = v;
     x->number = a->among_count++;
+    x->function_count = 0;
     x->starter = 0;
+    x->nocommand_count = 0;
+    x->amongvar_needed = false;
 
     if (q->type == c_bra) { x->starter = q; q = q->right; }
 
-    until (q == 0) {
+    while (q) {
         if (q->type == c_literalstring) {
             symbol * b = q->literalstring;
             w1->b = b;           /* pointer to case string */
-            w1->p = 0;           /* pointer to corresponding case expression */
+            w1->action = NULL;   /* action gets filled in below */
+            w1->line_number = q->line_number;
             w1->size = SIZE(b);  /* number of characters in string */
             w1->i = -1;          /* index of longest substring */
             w1->result = -1;     /* number of corresponding case expression */
-            w1->function = q->left == 0 ? 0 : q->left->name;
-            unless (w1->function == 0)
-                check_routine_mode(a, w1->function, direction);
+            if (q->left) {
+                struct name * function = q->left->name;
+                w1->function = function;
+                function->used_in_among = true;
+                check_routine_mode(a, function, direction);
+                x->function_count++;
+            } else {
+                w1->function = 0;
+            }
             w1++;
-        }
-        else
-        if (q->left == 0)  /* empty command: () */
+        } else if (q->left == 0) {
+            /* empty command: () */
             w0 = w1;
-        else {
-            until (w0 == w1) {
-                w0->p = q;
-                w0->result = result;
+        } else {
+            /* Check for previous action which is the same as this one and use
+             * the same action code if we find one.
+             */
+            int among_result = -1;
+            struct amongvec * w;
+            for (w = v; w < w0; ++w) {
+                if (w->action && compare_node(w->action->left, q->left) == 0) {
+                    if (w->result <= 0) {
+                        printf("Among code %d isn't positive\n", w->result);
+                        exit(1);
+                    }
+                    among_result = w->result;
+                    break;
+                }
+            }
+            if (among_result < 0) {
+                among_result = result++;
+            }
+
+            while (w0 != w1) {
+                w0->action = q;
+                w0->result = among_result;
                 w0++;
             }
-            result++;
         }
         q = q->right;
     }
-    unless (w1-v == p->number) { fprintf(stderr, "oh! %d %d\n", (int)(w1-v), p->number); exit(1); }
-    if (backward) for (w0 = v; w0 < w1; w0++) reverse_b(w0->b);
+    if (w1-v != p->number) { fprintf(stderr, "oh! %d %d\n", (int)(w1-v), p->number); exit(1); }
+    x->command_count = result - 1;
+    {
+        NEWVEC(node*, commands, x->command_count);
+        memset(commands, 0, x->command_count * sizeof(struct node*));
+        for (w0 = v; w0 < w1; w0++) {
+            if (w0->result > 0) {
+                /* result == -1 when there's no command. */
+                if (w0->result > x->command_count) {
+                    fprintf(stderr, "More among codes than expected\n");
+                    exit(1);
+                }
+                if (!commands[w0->result - 1])
+                    commands[w0->result - 1] = w0->action;
+            } else {
+                ++x->nocommand_count;
+            }
+            if (backward) reverse_b(w0->b);
+        }
+        x->commands = commands;
+    }
     qsort(v, w1 - v, sizeof(struct amongvec), compare_amongvec);
 
     /* the following loop is O(n squared) */
@@ -494,15 +709,30 @@ static void make_among(struct analyser * a, struct node * p, struct node * subst
 
     for (w0 = v; w0 < w1 - 1; w0++)
         if (w0->size == (w0 + 1)->size &&
-            memcmp(w0->b, (w0 + 1)->b, w0->size * sizeof(symbol)) == 0) error3(a, p, w0->b);
+            memcmp(w0->b, (w0 + 1)->b, w0->size * sizeof(symbol)) == 0) {
+            count_error(a);
+            fprintf(stderr, "%s:%d: among(...) has repeated string '",
+                    a->tokeniser->file, (w0 + 1)->line_number);
+            report_b(stderr, (w0 + 1)->b);
+            fprintf(stderr, "'\n");
+            count_error(a);
+            fprintf(stderr, "%s:%d: previously seen here\n",
+                    a->tokeniser->file, w0->line_number);
+        }
 
     x->literalstring_count = p->number;
-    x->command_count = result - 1;
     p->among = x;
 
     x->substring = substring;
     if (substring != 0) substring->among = x;
-    unless (x->command_count == 0 && x->starter == 0) a->amongvar_needed = true;
+    if (x->command_count > 1 ||
+        (x->command_count == 1 && x->nocommand_count > 0) ||
+        x->starter != 0) {
+        /* We need to set among_var rather than just checking if find_among*()
+         * returns zero or not.
+         */
+        x->amongvar_needed = a->amongvar_needed = true;
+    }
 }
 
 static struct node * read_among(struct analyser * a) {
@@ -514,8 +744,8 @@ static struct node * read_among(struct analyser * a) {
 
     a->substring = 0;
     p->number = 0; /* counts the number of literals */
-    unless (get_token(a, c_bra)) return p;
-    repeat {
+    if (!get_token(a, c_bra)) return p;
+    while (true) {
         struct node * q;
         int token = read_token(t);
         switch (token) {
@@ -529,12 +759,14 @@ static struct node * read_among(struct analyser * a) {
                 else t->token_held = true;
                 p->number++; break;
             case c_bra:
-                if (previous_token == c_bra) error(a, 19);
+                if (previous_token == c_bra) error(a, e_adjacent_bracketed_in_among);
                 q = read_C_list(a); break;
             default:
-                error(a, 3);
+                error(a, e_unexpected_token_in_among);
+                previous_token = token;
+                continue;
             case c_ket:
-                if (p->number == 0) error(a, 18);
+                if (p->number == 0) error(a, e_empty_among);
                 if (t->error_count == 0) make_among(a, p, substring);
                 return p;
         }
@@ -547,13 +779,13 @@ static struct node * read_among(struct analyser * a) {
 static struct node * read_substring(struct analyser * a) {
 
     struct node * p = new_node(a, c_substring);
-    if (a->substring != 0) error2(a, 20, a->substring->line_number);
+    if (a->substring != 0) error2(a, e_substring_preceded_by_substring, a->substring->line_number);
     a->substring = p;
     return p;
 }
 
 static void check_modifyable(struct analyser * a) {
-    unless (a->modifyable) error(a, 15);
+    if (!a->modifyable) error(a, e_not_allowed_inside_reverse);
 }
 
 static struct node * read_C(struct analyser * a) {
@@ -565,7 +797,7 @@ static struct node * read_C(struct analyser * a) {
         case c_backwards:
             {
                 int mode = a->mode;
-                if (a->mode == m_backward) error(a, 17); else a->mode = m_backward;
+                if (a->mode == m_backward) error(a, e_already_backwards); else a->mode = m_backward;
                 {   struct node * p = C_style(a, "C", token);
                     a->mode = mode;
                     return p;
@@ -596,14 +828,18 @@ static struct node * read_C(struct analyser * a) {
         case c_loop:
         case c_atleast:
             return C_style(a, "AC", token);
-        case c_setmark:
-            return C_style(a, "i", token);
+        case c_setmark: {
+            struct node * n = C_style(a, "i", token);
+            if (n->name) n->name->initialised = true;
+            return n;
+        }
         case c_tomark:
         case c_atmark:
         case c_hop:
             return C_style(a, "A", token);
         case c_delete:
             check_modifyable(a);
+            /* fall through */
         case c_next:
         case c_tolimit:
         case c_atlimit:
@@ -612,60 +848,147 @@ static struct node * read_C(struct analyser * a) {
         case c_true:
         case c_false:
         case c_debug:
-            return C_style(a, "", token);
+            return new_node(a, token);
         case c_assignto:
-        case c_sliceto:
+        case c_sliceto: {
+            struct node *n;
             check_modifyable(a);
-            return C_style(a, "s", token);
+            n = C_style(a, "s", token);
+            if (n->name) n->name->initialised = true;
+            return n;
+        }
         case c_assign:
         case c_insert:
         case c_attach:
-        case c_slicefrom:
+        case c_slicefrom: {
+            struct node *n;
             check_modifyable(a);
-            return C_style(a, "S", token);
+            n = C_style(a, "S", token);
+            if (n->name) n->name->value_used = true;
+            return n;
+        }
         case c_setlimit:
             return C_style(a, "CfD", token);
         case c_set:
-        case c_unset:
-            return C_style(a, "b", token);
-        case c_dollar:
-            get_token(a, c_name);
-            {
+        case c_unset: {
+            struct node * n = C_style(a, "b", token);
+            if (n->name) n->name->initialised = true;
+            return n;
+        }
+        case c_dollar: {
+            struct tokeniser * t = a->tokeniser;
+            read_token(t);
+            if (t->token == c_bra) {
+                /* Handle newer $(AE REL_OP AE) syntax. */
+                struct node * n = read_AE(a, 0);
+                read_token(t);
+                switch (t->token) {
+                    case c_eq:
+                    case c_ne:
+                    case c_gr:
+                    case c_ge:
+                    case c_ls:
+                    case c_le: {
+                        struct node * lhs = n;
+                        n = new_node(a, t->token);
+                        n->left = lhs;
+                        n->AE = read_AE(a, 0);
+                        get_token(a, c_ket);
+                        break;
+                    }
+                    default:
+                        error(a, e_unexpected_token);
+                        t->token_held = true;
+                        break;
+                }
+                return n;
+            }
+
+            if (t->token == c_name) {
                 struct node * p;
                 struct name * q = find_name(a);
                 int mode = a->mode;
                 int modifyable = a->modifyable;
-                switch (q ? q->type : t_string)
-                    /* above line was: switch (q->type) - bug #1 fix 7/2/2003 */
-                {
-                    default: error(a, 34);
-                    case t_string:
-                        a->mode = m_forward;
-                        a->modifyable = true;
-                        p = new_node(a, c_dollar);
-                        p->left = read_C(a); break;
-                    case t_integer:
-                    /*  a->mode = m_integer;  */
-                        p = new_node(a, read_AE_test(a));
-                        p->AE = read_AE(a, 0); break;
+                if (q && q->type == t_string) {
+                    /* Assume for now that $ on string both initialises and
+                     * uses the string variable.  FIXME: Can we do better?
+                     */
+                    q->initialised = true;
+                    q->value_used = true;
+                    a->mode = m_forward;
+                    a->modifyable = true;
+                    p = new_node(a, c_dollar);
+                    p->left = read_C(a);
+                    p->name = q;
+                } else {
+                    if (q && q->type != t_integer) {
+                        /* If $ is used on an unknown name or a name which
+                         * isn't a string or an integer then we assume the
+                         * unknown name is an integer as $ is used more often
+                         * on integers than strings, so hopefully this it less
+                         * likely to cause an error avalanche.
+                         *
+                         * For an unknown name, we'll already have reported an
+                         * error.
+                         */
+                        error(a, e_not_of_type_string_or_integer);
+                        q = NULL;
+                    }
+                    p = new_node(a, read_AE_test(a));
+                    p->AE = read_AE(a, 0);
+
+                    if (q) {
+                        switch (p->type) {
+                            case c_mathassign:
+                                q->initialised = true;
+                                p->name = q;
+                                break;
+                            default:
+                                /* +=, etc don't "initialise" as they only
+                                 * amend an existing value.  Similarly, they
+                                 * don't count as using the value.
+                                 */
+                                p->name = q;
+                                break;
+                            case c_eq:
+                            case c_ne:
+                            case c_gr:
+                            case c_ge:
+                            case c_ls:
+                            case c_le:
+                                p->left = new_node(a, c_name);
+                                p->left->name = q;
+                                q->value_used = true;
+                                break;
+                        }
+                    }
                 }
-                p->name = q;
+                if (q) mark_used_in(a, q, p);
                 a->mode = mode;
                 a->modifyable = modifyable;
                 return p;
             }
+
+            error(a, e_unexpected_token);
+            t->token_held = true;
+            return new_node(a, c_dollar);
+        }
         case c_name:
             {
                 struct name * q = find_name(a);
                 struct node * p = new_node(a, c_name);
-                unless (q == 0) {
-                    q->used = true;
+                if (q) {
+                    mark_used_in(a, q, p);
                     switch (q->type) {
                         case t_boolean:
-                            p->type = c_booltest; break;
+                            p->type = c_booltest;
+                            q->value_used = true;
+                            break;
                         case t_integer:
-                            error(a, 35); /* integer name misplaced */
+                            error(a, e_misplaced); /* integer name misplaced */
+                            break;
                         case t_string:
+                            q->value_used = true;
                             break;
                         case t_routine:
                         case t_external:
@@ -684,7 +1007,7 @@ static struct node * read_C(struct analyser * a) {
                 struct node * p = new_node(a, token);
                 read_token(t);
                 if (t->token == c_minus) read_token(t);
-                unless (check_token(a, c_name)) { omission_error(a, c_name); return p; }
+                if (!check_token(a, c_name)) { omission_error(a, c_name); return p; }
                 name_to_node(a, p, 'g');
                 return p;
             }
@@ -692,7 +1015,7 @@ static struct node * read_C(struct analyser * a) {
             return read_literalstring(a);
         case c_among: return read_among(a);
         case c_substring: return read_substring(a);
-        default: error(a, 1); return 0;
+        default: error(a, e_unexpected_token); return 0;
     }
 }
 
@@ -739,26 +1062,27 @@ static void read_define_grouping(struct analyser * a, struct name * q) {
         NEW(grouping, p);
         if (a->groupings == 0) a->groupings = p; else a->groupings_end->next = p;
         a->groupings_end = p;
-        q->grouping = p;
+        if (q) q->grouping = p;
         p->next = 0;
         p->name = q;
-        p->number = q->count;
+        p->line_number = a->tokeniser->line_number;
         p->b = create_b(0);
-        repeat {
+        while (true) {
             switch (read_token(t)) {
                 case c_name:
                     {
                         struct name * r = find_name(a);
-                        unless (r == 0) {
+                        if (r) {
                             check_name_type(a, r, 'g');
                             p->b = alter_grouping(p->b, r->grouping->b, style, false);
+                            r->used_in_definition = true;
                         }
                     }
                     break;
                 case c_literalstring:
-                    p->b = alter_grouping(p->b, t->b, style, a->utf8);
+                    p->b = alter_grouping(p->b, t->b, style, (a->encoding == ENC_UTF8));
                     break;
-                default: error(a, 1); return;
+                default: error(a, e_unexpected_token); return;
             }
             switch (read_token(t)) {
                 case c_plus:
@@ -777,7 +1101,7 @@ static void read_define_grouping(struct analyser * a, struct name * q) {
             }
             p->largest_ch = max;
             p->smallest_ch = min;
-            if (min == 1<<16) error(a, 16);
+            if (min == 1<<16) error(a, e_empty_grouping);
         }
         t->token_held = true; return;
     }
@@ -786,32 +1110,49 @@ static void read_define_grouping(struct analyser * a, struct name * q) {
 static void read_define_routine(struct analyser * a, struct name * q) {
     struct node * p = new_node(a, c_define);
     a->amongvar_needed = false;
-    unless (q == 0) {
+    if (q) {
         check_name_type(a, q, 'R');
-        if (q->definition != 0) error(a, 36);
+        if (q->definition != 0) error(a, e_redefined);
         if (q->mode < 0) q->mode = a->mode; else
-        if (q->mode != a->mode) error2(a, 32, q->mode);
+        if (q->mode != a->mode) error2(a, e_declared_as_different_mode, q->mode);
     }
     p->name = q;
     if (a->program == 0) a->program = p; else a->program_end->right = p;
     a->program_end = p;
     get_token(a, c_as);
     p->left = read_C(a);
-    unless (q == 0) q->definition = p->left;
+    if (q) q->definition = p->left;
 
     if (a->substring != 0) {
-         error2(a, 14, a->substring->line_number);
-         a->substring = 0;
+        error2(a, e_unresolved_substring, a->substring->line_number);
+        a->substring = 0;
     }
     p->amongvar_needed = a->amongvar_needed;
 }
 
 static void read_define(struct analyser * a) {
-    unless (get_token(a, c_name)) return;
-    {
+    if (get_token(a, c_name)) {
         struct name * q = find_name(a);
-        if (q != 0 && q->type == t_grouping) read_define_grouping(a, q);
-            else read_define_routine(a, q);
+        int type;
+        if (q) {
+            type = q->type;
+        } else {
+            /* No declaration, so sniff next token - if it is 'as' then parse
+             * as a routine, otherwise as a grouping.
+             */
+            if (read_token(a->tokeniser) == c_as) {
+                type = t_routine;
+            } else {
+                type = t_grouping;
+            }
+            a->tokeniser->token_held = true;
+        }
+
+        if (type == t_grouping) {
+            read_define_grouping(a, q);
+        } else {
+            read_define_routine(a, q);
+        }
     }
 }
 
@@ -827,7 +1168,7 @@ static void read_backwardmode(struct analyser * a) {
 
 static void read_program_(struct analyser * a, int terminator) {
     struct tokeniser * t = a->tokeniser;
-    repeat {
+    while (true) {
         switch (read_token(t)) {
             case c_strings:     read_names(a, t_string); break;
             case c_booleans:    read_names(a, t_boolean); break;
@@ -839,25 +1180,60 @@ static void read_program_(struct analyser * a, int terminator) {
             case c_backwardmode:read_backwardmode(a); break;
             case c_ket:
                 if (terminator == c_ket) return;
+                /* fall through */
             default:
-                error(a, 1); break;
+                error(a, e_unexpected_token); break;
             case -1:
-                unless (terminator < 0) omission_error(a, c_ket);
+                if (terminator >= 0) omission_error(a, c_ket);
                 return;
         }
     }
 }
 
+static void remove_dead_assignments(struct node * p, struct name * q) {
+    if (p->name == q) {
+        switch (p->type) {
+            case c_assignto:
+            case c_sliceto:
+            case c_mathassign:
+            case c_plusassign:
+            case c_minusassign:
+            case c_multiplyassign:
+            case c_divideassign:
+            case c_setmark:
+            case c_set:
+            case c_unset:
+            case c_dollar:
+                /* c_true is a no-op. */
+                p->type = c_true;
+                break;
+            default:
+                /* There are no read accesses to this variable, so any
+                 * references must be assignments.
+                 */
+                fprintf(stderr, "Unhandled type of dead assignment via %s\n",
+                        name_of_token(p->type));
+                exit(1);
+        }
+    }
+    if (p->AE) remove_dead_assignments(p->AE, q);
+    if (p->left) remove_dead_assignments(p->left, q);
+    if (p->aux) remove_dead_assignments(p->aux, q);
+    if (p->right) remove_dead_assignments(p->right, q);
+}
+
 extern void read_program(struct analyser * a) {
     read_program_(a, -1);
     {
         struct name * q = a->names;
-        until (q == 0) {
-            switch(q->type) {
+        while (q) {
+            switch (q->type) {
                 case t_external: case t_routine:
-                    if (q->used && q->definition == 0) { error4(a, q); } break;
+                    if (q->used && q->definition == 0) error4(a, q);
+                    break;
                 case t_grouping:
-                    if (q->used && q->grouping == 0) { error4(a, q); } break;
+                    if (q->used && q->grouping == 0) error4(a, q);
+                    break;
             }
             q = q->next;
         }
@@ -865,33 +1241,77 @@ extern void read_program(struct analyser * a) {
 
     if (a->tokeniser->error_count == 0) {
         struct name * q = a->names;
-        int warned = false;
-        until (q == 0) {
-            unless (q->referenced) {
-                unless (warned) {
-                    fprintf(stderr, "Declared but not used:");
-                    warned = true;
+        struct name ** ptr = &(a->names);
+        while (q) {
+            if (!q->referenced) {
+                fprintf(stderr, "%s:%d: warning: %s '",
+                        a->tokeniser->file,
+                        q->declaration_line_number,
+                        name_of_name_type(q->type));
+                report_b(stderr, q->b);
+                if (q->type == t_routine ||
+                    q->type == t_external ||
+                    q->type == t_grouping) {
+                    fprintf(stderr, "' declared but not defined\n");
+                } else {
+                    fprintf(stderr, "' defined but not used\n");
+                    q = q->next;
+                    *ptr = q;
+                    continue;
                 }
-                fprintf(stderr, " "); report_b(stderr, q->b);
+            } else if (q->type == t_routine || q->type == t_grouping) {
+                /* It's OK to define a grouping but only use it to define other
+                 * groupings.
+                 */
+                if (!q->used && !q->used_in_definition) {
+                    int line_num;
+                    if (q->type == t_routine) {
+                        line_num = q->definition->line_number;
+                    } else {
+                        line_num = q->grouping->line_number;
+                    }
+                    fprintf(stderr, "%s:%d: warning: %s '",
+                            a->tokeniser->file,
+                            line_num,
+                            name_of_name_type(q->type));
+                    report_b(stderr, q->b);
+                    fprintf(stderr, "' defined but not used\n");
+                }
+            } else if (q->type == t_external) {
+                /* Unused is OK. */
+            } else if (!q->initialised) {
+                fprintf(stderr, "%s:%d: warning: %s '",
+                        a->tokeniser->file,
+                        q->declaration_line_number,
+                        name_of_name_type(q->type));
+                report_b(stderr, q->b);
+                fprintf(stderr, "' is never initialised\n");
+            } else if (!q->value_used) {
+                fprintf(stderr, "%s:%d: warning: %s '",
+                        a->tokeniser->file,
+                        q->declaration_line_number,
+                        name_of_name_type(q->type));
+                report_b(stderr, q->b);
+                fprintf(stderr, "' is set but never used\n");
+                remove_dead_assignments(a->program, q);
+                q = q->next;
+                *ptr = q;
+                continue;
             }
+            ptr = &(q->next);
             q = q->next;
         }
-        if (warned) fprintf(stderr, "\n");
 
-        q = a->names;
-        warned = false;
-        until (q == 0) {
-            if (! q->used && (q->type == t_routine ||
-                              q->type == t_grouping)) {
-                unless (warned) {
-                    fprintf(stderr, "Declared and defined but not used:");
-                    warned = true;
-                }
-                fprintf(stderr, " "); report_b(stderr, q->b);
+        {
+            /* Now we've eliminated variables whose values are never used we
+             * can number the variables, which is used by some generators.
+             */
+            int * name_count = a->name_count;
+            struct name * n;
+            for (n = a->names; n; n = n->next) {
+                n->count = name_count[n->type]++;
             }
-            q = q->next;
         }
-        if (warned) fprintf(stderr, "\n");
     }
 }
 
@@ -909,13 +1329,14 @@ extern struct analyser * create_analyser(struct tokeniser * t) {
     a->modifyable = true;
     { int i; for (i = 0; i < t_size; i++) a->name_count[i] = 0; }
     a->substring = 0;
+    a->int_limits_used = false;
     return a;
 }
 
 extern void close_analyser(struct analyser * a) {
     {
         struct node * q = a->nodes;
-        until (q == 0) {
+        while (q) {
             struct node * q_next = q->next;
             FREE(q);
             q = q_next;
@@ -923,7 +1344,7 @@ extern void close_analyser(struct analyser * a) {
     }
     {
         struct name * q = a->names;
-        until (q == 0) {
+        while (q) {
             struct name * q_next = q->next;
             lose_b(q->b); FREE(q);
             q = q_next;
@@ -931,7 +1352,7 @@ extern void close_analyser(struct analyser * a) {
     }
     {
         struct literalstring * q = a->literalstrings;
-        until (q == 0) {
+        while (q) {
             struct literalstring * q_next = q->next;
             lose_b(q->b); FREE(q);
             q = q_next;
@@ -939,15 +1360,17 @@ extern void close_analyser(struct analyser * a) {
     }
     {
         struct among * q = a->amongs;
-        until (q == 0) {
+        while (q) {
             struct among * q_next = q->next;
-            FREE(q->b); FREE(q);
+            FREE(q->b);
+            FREE(q->commands);
+            FREE(q);
             q = q_next;
         }
     }
     {
         struct grouping * q = a->groupings;
-        until (q == 0) {
+        while (q) {
             struct grouping * q_next = q->next;
             lose_b(q->b); FREE(q);
             q = q_next;
@@ -955,4 +1378,3 @@ extern void close_analyser(struct analyser * a) {
     }
     FREE(a);
 }
-
diff --git a/contrib/snowball/compiler/driver.c b/contrib/snowball/compiler/driver.c
index fbb1e9cae..9cbe447a5 100644
--- a/contrib/snowball/compiler/driver.c
+++ b/contrib/snowball/compiler/driver.c
@@ -1,48 +1,86 @@
+#include <ctype.h>   /* for toupper etc */
 #include <stdio.h>   /* for fprintf etc */
 #include <stdlib.h>  /* for free etc */
-#include <string.h>  /* for strlen */
+#include <string.h>  /* for strcmp */
 #include "header.h"
 
-#define DEFAULT_PACKAGE "org.tartarus.snowball.ext"
-#define DEFAULT_BASE_CLASS "org.tartarus.snowball.SnowballProgram"
-#define DEFAULT_AMONG_CLASS "org.tartarus.snowball.Among"
-#define DEFAULT_STRING_CLASS "java.lang.StringBuilder"
+#define DEFAULT_JAVA_PACKAGE "org.tartarus.snowball.ext"
+#define DEFAULT_JAVA_BASE_CLASS "org.tartarus.snowball.SnowballProgram"
+#define DEFAULT_JAVA_AMONG_CLASS "org.tartarus.snowball.Among"
+#define DEFAULT_JAVA_STRING_CLASS "java.lang.StringBuilder"
+
+#define DEFAULT_GO_PACKAGE "snowball"
+#define DEFAULT_GO_SNOWBALL_RUNTIME "github.com/snowballstem/snowball/go"
+
+#define DEFAULT_CS_NAMESPACE "Snowball"
+#define DEFAULT_CS_BASE_CLASS "Stemmer"
+#define DEFAULT_CS_AMONG_CLASS "Among"
+#define DEFAULT_CS_STRING_CLASS "StringBuilder"
+
+#define DEFAULT_JS_BASE_CLASS "BaseStemmer"
+
+#define DEFAULT_PYTHON_BASE_CLASS "BaseStemmer"
 
 static int eq(const char * s1, const char * s2) {
-    int s1_len = strlen(s1);
-    int s2_len = strlen(s2);
-    return s1_len == s2_len && memcmp(s1, s2, s1_len) == 0;
+    return strcmp(s1, s2) == 0;
 }
 
-static void print_arglist(void) {
-    fprintf(stderr, "Usage: snowball <file> [options]\n\n"
-                    "options are: [-o[utput] file]\n"
-                    "             [-s[yntax]]\n"
+static void print_arglist(int exit_code) {
+    FILE * f = exit_code ? stderr : stdout;
+    fprintf(f, "Usage: snowball SOURCE_FILE... [OPTIONS]\n\n"
+               "Supported options:\n"
+               "  -o[utput] file\n"
+               "  -s[yntax]\n"
+               "  -comments\n"
 #ifndef DISABLE_JAVA
-                    "             [-j[ava]]\n"
+               "  -j[ava]\n"
 #endif
-                    "             [-c++]\n"
-                    "             [-w[idechars]]\n"
-                    "             [-u[tf8]]\n"
-                    "             [-n[ame] class name]\n"
-                    "             [-ep[refix] string]\n"
-                    "             [-vp[refix] string]\n"
-                    "             [-i[nclude] directory]\n"
-                    "             [-r[untime] path to runtime headers]\n"
-#ifndef DISABLE_JAVA
-                    "             [-p[arentclassname] fully qualified parent class name]\n"
-                    "             [-P[ackage] package name for stemmers]\n"
-                    "             [-S[tringclass] StringBuffer-compatible class]\n"
-                    "             [-a[mongclass] fully qualified name of the Among class]\n"
+#ifndef DISABLE_CSHARP
+               "  -cs[harp]\n"
+#endif
+               "  -c++\n"
+#ifndef DISABLE_PASCAL
+               "  -pascal\n"
+#endif
+#ifndef DISABLE_PYTHON
+               "  -py[thon]\n"
 #endif
+#ifndef DISABLE_JS
+               "  -js\n"
+#endif
+#ifndef DISABLE_RUST
+               "  -rust\n"
+#endif
+#ifndef DISABLE_GO
+               "  -go\n"
+#endif
+               "  -w[idechars]\n"
+               "  -u[tf8]\n"
+               "  -n[ame] class name\n"
+               "  -ep[refix] string\n"
+               "  -vp[refix] string\n"
+               "  -i[nclude] directory\n"
+               "  -r[untime] path to runtime headers\n"
+               "  -p[arentclassname] fully qualified parent class name\n"
+#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP)
+               "  -P[ackage] package name for stemmers\n"
+               "  -S[tringclass] StringBuffer-compatible class\n"
+               "  -a[mongclass] fully qualified name of the Among class\n"
+#endif
+#ifndef DISABLE_GO
+               "  -gop[ackage] Go package name for stemmers\n"
+               "  -gor[untime] Go snowball runtime package\n"
+#endif
+               "  --help        display this help and exit\n"
+               "  --version     output version information and exit\n"
            );
-    exit(1);
+    exit(exit_code);
 }
 
 static void check_lim(int i, int argc) {
     if (i >= argc) {
         fprintf(stderr, "argument list is one short\n");
-        print_arglist();
+        print_arglist(1);
     }
 }
 
@@ -57,35 +95,47 @@ static FILE * get_output(symbol * b) {
     return output;
 }
 
-static void read_options(struct options * o, int argc, char * argv[]) {
+static int read_options(struct options * o, int argc, char * argv[]) {
     char * s;
-    int i = 2;
+    int i = 1;
+    int new_argc = 1;
+    /* Note down the last option used to specify an explicit encoding so
+     * we can warn we ignored it for languages with a fixed encoding.
+     */
+    const char * encoding_opt = NULL;
 
     /* set defaults: */
 
     o->output_file = 0;
     o->syntax_tree = false;
-    o->externals_prefix = "";
+    o->comments = false;
+    o->externals_prefix = NULL;
     o->variables_prefix = 0;
     o->runtime_path = 0;
-    o->parent_class_name = DEFAULT_BASE_CLASS;
-    o->string_class = DEFAULT_STRING_CLASS;
-    o->among_class = DEFAULT_AMONG_CLASS;
-    o->package = DEFAULT_PACKAGE;
-    o->name = "";
+    o->parent_class_name = NULL;
+    o->string_class = NULL;
+    o->among_class = NULL;
+    o->package = NULL;
+    o->go_snowball_runtime = DEFAULT_GO_SNOWBALL_RUNTIME;
+    o->name = NULL;
     o->make_lang = LANG_C;
-    o->widechars = false;
     o->includes = 0;
     o->includes_end = 0;
-    o->utf8 = false;
+    o->encoding = ENC_SINGLEBYTE;
 
     /* read options: */
 
-    repeat {
-        if (i >= argc) break;
+    while (i < argc) {
         s = argv[i++];
-        {   if (eq(s, "-o") || eq(s, "-output")) {
-                check_lim(i, argc);
+        if (s[0] != '-') {
+            /* Non-option argument - shuffle down. */
+            argv[new_argc++] = s;
+            continue;
+        }
+
+        {
+            if (eq(s, "-o") || eq(s, "-output")) {
+               check_lim(i, argc);
                 o->output_file = argv[i++];
                 continue;
             }
@@ -94,10 +144,33 @@ static void read_options(struct options * o, int argc, char * argv[]) {
                 o->name = argv[i++];
                 continue;
             }
+#ifndef DISABLE_JS
+            if (eq(s, "-js")) {
+                o->make_lang = LANG_JAVASCRIPT;
+                continue;
+            }
+#endif
+#ifndef DISABLE_RUST
+            if (eq(s, "-rust")) {
+                o->make_lang = LANG_RUST;
+                continue;
+            }
+#endif
+#ifndef DISABLE_GO
+            if (eq(s, "-go")) {
+                o->make_lang = LANG_GO;
+                continue;
+            }
+#endif
 #ifndef DISABLE_JAVA
             if (eq(s, "-j") || eq(s, "-java")) {
                 o->make_lang = LANG_JAVA;
-                o->widechars = true;
+                continue;
+            }
+#endif
+#ifndef DISABLE_CSHARP
+            if (eq(s, "-cs") || eq(s, "-csharp")) {
+                o->make_lang = LANG_CSHARP;
                 continue;
             }
 #endif
@@ -105,15 +178,31 @@ static void read_options(struct options * o, int argc, char * argv[]) {
                 o->make_lang = LANG_CPLUSPLUS;
                 continue;
             }
+#ifndef DISABLE_PASCAL
+            if (eq(s, "-pascal")) {
+                o->make_lang = LANG_PASCAL;
+                continue;
+            }
+#endif
+#ifndef DISABLE_PYTHON
+            if (eq(s, "-py") || eq(s, "-python")) {
+                o->make_lang = LANG_PYTHON;
+                continue;
+            }
+#endif
             if (eq(s, "-w") || eq(s, "-widechars")) {
-                o->widechars = true;
-                o->utf8 = false;
+                encoding_opt = s;
+                o->encoding = ENC_WIDECHARS;
                 continue;
             }
             if (eq(s, "-s") || eq(s, "-syntax")) {
                 o->syntax_tree = true;
                 continue;
             }
+            if (eq(s, "-comments")) {
+                o->comments = true;
+                continue;
+            }
             if (eq(s, "-ep") || eq(s, "-eprefix")) {
                 check_lim(i, argc);
                 o->externals_prefix = argv[i++];
@@ -145,16 +234,16 @@ static void read_options(struct options * o, int argc, char * argv[]) {
                 continue;
             }
             if (eq(s, "-u") || eq(s, "-utf8")) {
-                o->utf8 = true;
-                o->widechars = false;
+                encoding_opt = s;
+                o->encoding = ENC_UTF8;
                 continue;
             }
-#ifndef DISABLE_JAVA
             if (eq(s, "-p") || eq(s, "-parentclassname")) {
                 check_lim(i, argc);
                 o->parent_class_name = argv[i++];
                 continue;
             }
+#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP)
             if (eq(s, "-P") || eq(s, "-Package")) {
                 check_lim(i, argc);
                 o->package = argv[i++];
@@ -171,44 +260,216 @@ static void read_options(struct options * o, int argc, char * argv[]) {
                 continue;
             }
 #endif
+#ifndef DISABLE_GO
+            if (eq(s, "-gop") || eq(s, "-gopackage")) {
+                check_lim(i, argc);
+                o->package = argv[i++];
+                continue;
+            }
+            if (eq(s, "-gor") || eq(s, "-goruntime")) {
+                check_lim(i, argc);
+                o->go_snowball_runtime = argv[i++];
+                continue;
+            }
+#endif
+            if (eq(s, "--help")) {
+                print_arglist(0);
+            }
+
+            if (eq(s, "--version")) {
+                printf("Snowball compiler version " SNOWBALL_VERSION "\n");
+                exit(0);
+            }
+
             fprintf(stderr, "'%s' misplaced\n", s);
-            print_arglist();
+            print_arglist(1);
         }
     }
+    if (new_argc == 1) {
+        fprintf(stderr, "no source files specified\n");
+        print_arglist(1);
+    }
+    argv[new_argc] = NULL;
+
+    /* Set language-dependent defaults. */
+    switch (o->make_lang) {
+        case LANG_C:
+        case LANG_CPLUSPLUS:
+            encoding_opt = NULL;
+            break;
+        case LANG_CSHARP:
+            o->encoding = ENC_WIDECHARS;
+            if (!o->parent_class_name)
+                o->parent_class_name = DEFAULT_CS_BASE_CLASS;
+            if (!o->string_class)
+                o->string_class = DEFAULT_CS_STRING_CLASS;
+            if (!o->among_class)
+                o->among_class = DEFAULT_CS_AMONG_CLASS;
+            if (!o->package)
+                o->package = DEFAULT_CS_NAMESPACE;
+            break;
+        case LANG_GO:
+            o->encoding = ENC_UTF8;
+            if (!o->package)
+                o->package = DEFAULT_GO_PACKAGE;
+            break;
+        case LANG_JAVA:
+            o->encoding = ENC_WIDECHARS;
+            if (!o->parent_class_name)
+                o->parent_class_name = DEFAULT_JAVA_BASE_CLASS;
+            if (!o->string_class)
+                o->string_class = DEFAULT_JAVA_STRING_CLASS;
+            if (!o->among_class)
+                o->among_class = DEFAULT_JAVA_AMONG_CLASS;
+            if (!o->package)
+                o->package = DEFAULT_JAVA_PACKAGE;
+            break;
+        case LANG_JAVASCRIPT:
+            o->encoding = ENC_WIDECHARS;
+            if (!o->parent_class_name)
+                o->parent_class_name = DEFAULT_JS_BASE_CLASS;
+            break;
+        case LANG_PYTHON:
+            o->encoding = ENC_WIDECHARS;
+            if (!o->parent_class_name)
+                o->parent_class_name = DEFAULT_PYTHON_BASE_CLASS;
+            break;
+        case LANG_RUST:
+            o->encoding = ENC_UTF8;
+            break;
+        default:
+            break;
+    }
+
+    if (encoding_opt) {
+        fprintf(stderr, "warning: %s only meaningful for C and C++\n",
+                encoding_opt);
+    }
+
+    if (o->make_lang != LANG_C && o->make_lang != LANG_CPLUSPLUS) {
+        if (o->runtime_path) {
+            fprintf(stderr, "warning: -r/-runtime only meaningful for C and C++\n");
+        }
+        if (o->externals_prefix) {
+            fprintf(stderr, "warning: -ep/-eprefix only meaningful for C and C++\n");
+        }
+    }
+    if (!o->externals_prefix) o->externals_prefix = "";
+
+    if (!o->name && o->output_file) {
+        /* Default class name to basename of output_file - this is the standard
+         * convention for at least Java and C#.
+         */
+        const char * slash = strrchr(o->output_file, '/');
+        size_t len;
+        const char * leaf = (slash == NULL) ? o->output_file : slash + 1;
+
+        slash = strrchr(leaf, '\\');
+        if (slash != NULL) leaf = slash + 1;
+
+        {
+            const char * dot = strchr(leaf, '.');
+            len = (dot == NULL) ? strlen(leaf) : (size_t)(dot - leaf);
+        }
+
+        {
+            char * new_name = malloc(len + 1);
+            switch (o->make_lang) {
+                case LANG_CSHARP:
+                case LANG_PASCAL:
+                    /* Upper case initial letter. */
+                    memcpy(new_name, leaf, len);
+                    new_name[0] = toupper(new_name[0]);
+                    break;
+                case LANG_JAVASCRIPT:
+                case LANG_PYTHON: {
+                    /* Upper case initial letter and change each
+                     * underscore+letter or hyphen+letter to an upper case
+                     * letter.
+                     */
+                    size_t i, j = 0;
+                    int uc_next = true;
+                    for (i = 0; i != len; ++i) {
+                        unsigned char ch = leaf[i];
+                        if (ch == '_' || ch == '-') {
+                            uc_next = true;
+                        } else {
+                            if (uc_next) {
+                                new_name[j] = toupper(ch);
+                                uc_next = false;
+                            } else {
+                                new_name[j] = ch;
+                            }
+                            ++j;
+                        }
+                    }
+                    len = j;
+                    break;
+                }
+                default:
+                    /* Just copy. */
+                    memcpy(new_name, leaf, len);
+                    break;
+            }
+            new_name[len] = '\0';
+            o->name = new_name;
+        }
+    }
+
+    return new_argc;
 }
 
 extern int main(int argc, char * argv[]) {
 
+    int i;
     NEW(options, o);
-    if (argc == 1) print_arglist();
-    read_options(o, argc, argv);
+    argc = read_options(o, argc, argv);
     {
-        symbol * filename = add_s_to_b(0, argv[1]);
-        char * file;
-        symbol * u = get_input(filename, &file);
+        char * file = argv[1];
+        symbol * u = get_input(file);
         if (u == 0) {
-            fprintf(stderr, "Can't open input %s\n", argv[1]);
+            fprintf(stderr, "Can't open input %s\n", file);
             exit(1);
         }
         {
             struct tokeniser * t = create_tokeniser(u, file);
             struct analyser * a = create_analyser(t);
-            t->widechars = o->widechars;
+            struct input ** next_input_ptr = &(t->next);
+            a->encoding = t->encoding = o->encoding;
             t->includes = o->includes;
-            a->utf8 = t->utf8 = o->utf8;
+            /* If multiple source files are specified, set up the others to be
+             * read after the first in order, using the same mechanism as
+             * 'get' uses. */
+            for (i = 2; i != argc; ++i) {
+                NEW(input, q);
+                file = argv[i];
+                u = get_input(file);
+                if (u == 0) {
+                    fprintf(stderr, "Can't open input %s\n", file);
+                    exit(1);
+                }
+                q->p = u;
+                q->c = 0;
+                q->file = file;
+                q->file_needs_freeing = false;
+                q->line_number = 1;
+                *next_input_ptr = q;
+                next_input_ptr = &(q->next);
+            }
+            *next_input_ptr = NULL;
             read_program(a);
             if (t->error_count > 0) exit(1);
             if (o->syntax_tree) print_program(a);
             close_tokeniser(t);
-            unless (o->syntax_tree) {
+            if (!o->syntax_tree) {
                 struct generator * g;
 
-                char * s = o->output_file;
-                unless (s) {
+                const char * s = o->output_file;
+                if (!s) {
                     fprintf(stderr, "Please include the -o option\n");
-                    print_arglist();
-                    exit(1);
+                    print_arglist(1);
                 }
+                g = create_generator(a, o);
                 if (o->make_lang == LANG_C || o->make_lang == LANG_CPLUSPLUS) {
                     symbol * b = add_s_to_b(0, s);
                     b = add_s_to_b(b, ".h");
@@ -217,41 +478,96 @@ extern int main(int argc, char * argv[]) {
                     if (o->make_lang == LANG_CPLUSPLUS) {
                         b = add_s_to_b(b, "c");
                     }
-                    o->output_c = get_output(b);
+                    o->output_src = get_output(b);
                     lose_b(b);
 
-                    g = create_generator_c(a, o);
                     generate_program_c(g);
-                    close_generator_c(g);
-                    fclose(o->output_c);
+                    fclose(o->output_src);
                     fclose(o->output_h);
                 }
 #ifndef DISABLE_JAVA
                 if (o->make_lang == LANG_JAVA) {
                     symbol * b = add_s_to_b(0, s);
                     b = add_s_to_b(b, ".java");
-                    o->output_java = get_output(b);
+                    o->output_src = get_output(b);
                     lose_b(b);
-                    g = create_generator_java(a, o);
                     generate_program_java(g);
-                    close_generator_java(g);
-                    fclose(o->output_java);
+                    fclose(o->output_src);
                 }
 #endif
+#ifndef DISABLE_PASCAL
+                if (o->make_lang == LANG_PASCAL) {
+                    symbol *b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".pas");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_pascal(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_PYTHON
+                if (o->make_lang == LANG_PYTHON) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".py");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_python(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_JS
+                if (o->make_lang == LANG_JAVASCRIPT) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".js");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_js(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_CSHARP
+                if (o->make_lang == LANG_CSHARP) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".cs");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_csharp(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_RUST
+                if (o->make_lang == LANG_RUST) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".rs");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_rust(g);
+                    fclose(o->output_src);
+                }
+#endif
+#ifndef DISABLE_GO
+                if (o->make_lang == LANG_GO) {
+                    symbol * b = add_s_to_b(0, s);
+                    b = add_s_to_b(b, ".go");
+                    o->output_src = get_output(b);
+                    lose_b(b);
+                    generate_program_go(g);
+                    fclose(o->output_src);
+                }
+#endif
+                close_generator(g);
             }
             close_analyser(a);
         }
         lose_b(u);
-        lose_b(filename);
     }
     {   struct include * p = o->includes;
-        until (p == 0)
-        {   struct include * q = p->next;
+        while (p) {
+            struct include * q = p->next;
             lose_b(p->b); FREE(p); p = q;
         }
     }
     FREE(o);
-    unless (space_count == 0) fprintf(stderr, "%d blocks unfreed\n", space_count);
+    if (space_count) fprintf(stderr, "%d blocks unfreed\n", space_count);
     return 0;
 }
-
diff --git a/contrib/snowball/compiler/generator.c b/contrib/snowball/compiler/generator.c
index 7294b0471..eed86c117 100644
--- a/contrib/snowball/compiler/generator.c
+++ b/contrib/snowball/compiler/generator.c
@@ -12,286 +12,349 @@
 
 static void generate(struct generator * g, struct node * p);
 
-enum special_labels {
-
-    x_return = -1
-
-};
-
 static int new_label(struct generator * g) {
     return g->next_label++;
 }
 
-/* Output routines */
-static void output_str(FILE * outfile, struct str * str) {
+/* Write routines for simple entities */
 
-    char * s = b_to_s(str_data(str));
-    fprintf(outfile, "%s", s);
-    free(s);
+/* Write a space if the preceding character was not whitespace */
+static void ws_opt_space(struct generator * g, const char * s) {
+    int ch = str_back(g->outbuf);
+    if (ch != ' ' && ch != '\n' && ch != '\t' && ch != -1)
+        write_char(g, ' ');
+    write_string(g, s);
 }
 
-static void wch(struct generator * g, int ch) {
-    str_append_ch(g->outbuf, ch); /* character */
+static void wi3(struct generator * g, int i) {
+    if (i < 100) write_char(g, ' ');
+    if (i < 10)  write_char(g, ' ');
+    write_int(g, i); /* integer (width 3) */
 }
 
-static void wnl(struct generator * g) {
-    str_append_ch(g->outbuf, '\n'); /* newline */
-    g->line_count++;
-}
 
-static void ws(struct generator * g, const char * s) {
-    str_append_string(g->outbuf, s); /* string */
-}
+/* Write routines for items from the syntax tree */
 
-static void wi(struct generator * g, int i) {
-    str_append_int(g->outbuf, i); /* integer */
-}
+static void write_varname(struct generator * g, struct name * p) {
 
-static void wh_ch(struct generator * g, int i) {
-    str_append_ch(g->outbuf, "0123456789ABCDEF"[i & 0xF]); /* hexchar */
+    int ch = "SIIrxg"[p->type];
+    switch (p->type) {
+        case t_external:
+            write_string(g, g->options->externals_prefix); break;
+        case t_string:
+        case t_boolean:
+        case t_integer: {
+            int count = p->count;
+            if (count < 0) {
+                fprintf(stderr, "Reference to optimised out variable ");
+                report_b(stderr, p->b);
+                fprintf(stderr, " attempted\n");
+                exit(1);
+            }
+            if (p->type == t_boolean) {
+                /* We use a single array for booleans and integers, with the
+                 * integers first.
+                 */
+                count += g->analyser->name_count[t_integer];
+            }
+            write_char(g, ch);
+            write_char(g, '[');
+            write_int(g, count);
+            write_char(g, ']');
+            return;
+        }
+        default:
+            write_char(g, ch); write_char(g, '_');
+    }
+    write_b(g, p->b);
 }
 
-static void wh(struct generator * g, int i) {
-    if (i >> 4) wh(g, i >> 4);
-    wh_ch(g, i); /* hex integer */
+static void write_varref(struct generator * g, struct name * p) {  /* reference to variable */
+    if (p->type < t_routine) write_string(g, "z->");
+    write_varname(g, p);
 }
 
-static void wi3(struct generator * g, int i) {
-    if (i < 100) wch(g, ' ');
-    if (i < 10)  wch(g, ' ');
-    wi(g, i); /* integer (width 3) */
+static void write_hexdigit(struct generator * g, int i) {
+    str_append_ch(g->outbuf, "0123456789ABCDEF"[i & 0xF]); /* hexchar */
 }
 
-static void wvn(struct generator * g, struct name * p) {  /* variable name */
-
-    int ch = "SBIrxg"[p->type];
-    switch (p->type) {
-        case t_string:
-        case t_boolean:
-        case t_integer:
-            wch(g, ch); wch(g, '['); wi(g, p->count); wch(g, ']'); return;
-        case t_external:
-            ws(g, g->options->externals_prefix); break;
-        default:
-            wch(g, ch); wch(g, '_');
-    }
-    str_append_b(g->outbuf, p->b);
+static void write_hex(struct generator * g, int i) {
+    if (i >> 4) write_hex(g, i >> 4);
+    write_hexdigit(g, i); /* hex integer */
 }
 
-static void wv(struct generator * g, struct name * p) {  /* reference to variable */
-    if (p->type < t_routine) ws(g, "z->");
-    wvn(g, p);
+/* write character literal */
+static void wlitch(struct generator * g, int ch) {
+    if (32 <= ch && ch < 127) {
+        write_char(g, '\'');
+        if (ch == '\'' || ch == '\\') {
+            write_char(g, '\\');
+        }
+        write_char(g, ch);
+        write_char(g, '\'');
+    } else {
+        write_string(g, "0x"); write_hex(g, ch);
+    }
 }
 
 static void wlitarray(struct generator * g, symbol * p) {  /* write literal array */
 
-    ws(g, "{ ");
+    write_string(g, "{ ");
     {
         int i;
         for (i = 0; i < SIZE(p); i++) {
-            int ch = p[i];
-            if (32 <= ch && ch < 127) {
-                wch(g, '\'');
-                switch (ch) {
-                    case '\'':
-                    case '\\': wch(g, '\\');
-                    default:   wch(g, ch);
-                }
-                wch(g, '\'');
-            }  else {
-                wch(g, '0'); wch(g, 'x'); wh(g, ch);
-            }
-            if (i < SIZE(p) - 1) ws(g, ", ");
+            wlitch(g, p[i]);
+            if (i < SIZE(p) - 1) write_string(g, ", ");
         }
     }
-    ws(g, " }");
+    write_string(g, " }");
 }
 
 static void wlitref(struct generator * g, symbol * p) {  /* write ref to literal array */
 
-    if (SIZE(p) == 0) ws(g, "0"); else {
+    if (SIZE(p) == 0) {
+        write_char(g, '0');
+    } else {
         struct str * s = g->outbuf;
         g->outbuf = g->declarations;
-        ws(g, "static const symbol s_"); wi(g, g->literalstring_count); ws(g, "[] = ");
+        write_string(g, "static const symbol s_"); write_int(g, g->literalstring_count); write_string(g, "[] = ");
         wlitarray(g, p);
-        ws(g, ";\n");
+        write_string(g, ";\n");
         g->outbuf = s;
-        ws(g, "s_"); wi(g, g->literalstring_count);
+        write_string(g, "s_"); write_int(g, g->literalstring_count);
         g->literalstring_count++;
     }
 }
 
-
-static void wm(struct generator * g) {       /* margin */
+static void write_margin(struct generator * g) {
     int i;
-    for (i = 0; i < g->margin; i++) ws(g, "    ");
+    for (i = 0; i < g->margin; i++) write_string(g, "    ");
 }
 
-static void wc(struct generator * g, struct node * p) { /* comment */
-
-    ws(g, " /* ");
+void write_comment_content(struct generator * g, struct node * p) {
     switch (p->type) {
         case c_mathassign:
         case c_plusassign:
         case c_minusassign:
         case c_multiplyassign:
         case c_divideassign:
+            if (p->name) {
+                write_char(g, '$');
+                write_b(g, p->name->b);
+                write_char(g, ' ');
+            }
+            write_string(g, name_of_token(p->type));
+            write_string(g, " <integer expression>");
+            break;
         case c_eq:
         case c_ne:
         case c_gr:
         case c_ge:
         case c_ls:
         case c_le:
-            if (p->name) {
-                wch(g, '$');
-                str_append_b(g->outbuf, p->name->b);
-                wch(g, ' ');
-            }
-            ws(g, name_of_token(p->type));
-            ws(g, " <integer expression>");
+            write_string(g, "$(<integer expression> ");
+            write_string(g, name_of_token(p->type));
+            write_string(g, " <integer expression>)");
             break;
         default:
-            ws(g, name_of_token(p->type));
+            write_string(g, name_of_token(p->type));
             if (p->name) {
-                wch(g, ' ');
-                str_append_b(g->outbuf, p->name->b);
+                write_char(g, ' ');
+                write_b(g, p->name->b);
             }
     }
-    ws(g, ", line "); wi(g, p->line_number); ws(g, " */");
-    wnl(g);
+    write_string(g, ", line ");
+    write_int(g, p->line_number);
+}
+
+static void write_comment(struct generator * g, struct node * p) {
+    if (g->options->comments) {
+        ws_opt_space(g, "/* ");
+        write_comment_content(g, p);
+        write_string(g, " */");
+    }
+    write_newline(g);
 }
 
 static void wms(struct generator * g, const char * s) {
-    wm(g); ws(g, s);   } /* margin + string */
+    write_margin(g); write_string(g, s);   } /* margin + string */
 
-static void wbs(struct generator * g) { /* block start */
+static void write_block_start(struct generator * g) { /* block start */
     wms(g, "{   ");
     g->margin++;
 }
 
-static void wbe(struct generator * g) {    /* block end */
+static void write_block_end(struct generator * g) {    /* block end */
 
-    if (g->line_labelled == g->line_count) { wms(g, ";"); wnl(g); }
+    if (g->line_labelled == g->line_count) { wms(g, ";"); write_newline(g); }
     g->margin--;
-    wms(g, "}"); wnl(g);
+    wms(g, "}"); write_newline(g);
 }
 
-static void wk(struct generator * g, struct node * p) {     /* keep c */
+static void w(struct generator * g, const char * s);
+
+/* keep c */
+static void wk(struct generator * g, struct node * p, int keep_limit) {
     ++g->keep_count;
     if (p->mode == m_forward) {
-        ws(g, "int c"); wi(g, g->keep_count); ws(g, " = z->c;");
+        write_string(g, "int c");
+        write_int(g, g->keep_count);
+        write_string(g, " = z->c");
+        if (keep_limit) {
+            write_string(g, ", mlimit");
+            write_int(g, g->keep_count);
+        }
+        write_char(g, ';');
     } else {
-        ws(g, "int m"); wi(g, g->keep_count); ws(g, " = z->l - z->c; (void)m");
-        wi(g, g->keep_count); ws(g, ";");
+        write_string(g, "int m");
+        write_int(g, g->keep_count);
+        write_string(g, " = z->l - z->c");
+        if (keep_limit) {
+            write_string(g, ", mlimit");
+            write_int(g, g->keep_count);
+        }
+        write_string(g, "; (void)m");
+        write_int(g, g->keep_count);
+        write_char(g, ';');
     }
 }
 
 static void wrestore(struct generator * g, struct node * p, int keep_token) {     /* restore c */
     if (p->mode == m_forward) {
-        ws(g, "z->c = c");
+        write_string(g, "z->c = c");
     } else {
-        ws(g, "z->c = z->l - m");
+        write_string(g, "z->c = z->l - m");
     }
-    wi(g, keep_token); ws(g, ";");
+    write_int(g, keep_token); write_char(g, ';');
+}
+
+static void wrestorelimit(struct generator * g, struct node * p, int keep_token) {     /* restore limit */
+    if (p->mode == m_forward) {
+        w(g, "z->l += mlimit");
+    } else {
+        w(g, "z->lb = mlimit");
+    }
+    write_int(g, keep_token); write_string(g, ";");
 }
 
 static void winc(struct generator * g, struct node * p) {     /* increment c */
-    ws(g, p->mode == m_forward ? "z->c++;" :
+    write_string(g, p->mode == m_forward ? "z->c++;" :
                                  "z->c--;");
 }
 
 static void wsetl(struct generator * g, int n) {
 
     g->margin--;
-    wms(g, "lab"); wi(g, n); wch(g, ':'); wnl(g);
+    wms(g, "lab"); write_int(g, n); write_char(g, ':'); write_newline(g);
     g->line_labelled = g->line_count;
     g->margin++;
 }
 
 static void wgotol(struct generator * g, int n) {
-    wms(g, "goto lab"); wi(g, n); wch(g, ';'); wnl(g);
+    wms(g, "goto lab"); write_int(g, n); write_char(g, ';'); write_newline(g);
 }
 
-static void wf(struct generator * g) {          /* fail */
-    if (g->failure_string != 0) { ws(g, "{ "); ws(g, g->failure_string); wch(g, ' '); }
-    switch (g->failure_label)
-    {
+static void write_failure(struct generator * g, struct node * p) {          /* fail */
+    if (g->failure_keep_count != 0) {
+        write_string(g, "{ ");
+        if (g->failure_keep_count > 0) {
+            wrestore(g, p, g->failure_keep_count);
+        } else {
+            wrestorelimit(g, p, -g->failure_keep_count);
+        }
+        write_char(g, ' ');
+    }
+    switch (g->failure_label) {
         case x_return:
-           ws(g, "return 0;");
-           break;
+            write_string(g, "return 0;");
+            break;
         default:
-           ws(g, "goto lab");
-           wi(g, g->failure_label);
-           wch(g, ';');
-           g->label_used = 1;
+            write_string(g, "goto lab");
+            write_int(g, g->failure_label);
+            write_char(g, ';');
+            g->label_used = 1;
     }
-    if (g->failure_string != 0) ws(g, " }");
+    if (g->failure_keep_count != 0) write_string(g, " }");
 }
 
-static void wlim(struct generator * g, struct node * p) {     /* if at limit fail */
 
-    ws(g, p->mode == m_forward ? "if (z->c >= z->l) " :
+/* if at limit fail */
+static void write_check_limit(struct generator * g, struct node * p) {
+
+    write_string(g, p->mode == m_forward ? "if (z->c >= z->l) " :
                                  "if (z->c <= z->lb) ");
-    wf(g);
+    write_failure(g, p);
 }
 
-static void wp(struct generator * g, const char * s, struct node * p) { /* formatted write */
+static void write_data_address(struct generator * g, struct node * p) {
+    symbol * b = p->literalstring;
+    if (b != 0) {
+        write_int(g, SIZE(b)); w(g, ", ");
+        wlitref(g, b);
+    } else {
+        write_varref(g, p->name);
+    }
+}
+
+/* Formatted write. */
+static void writef(struct generator * g, const char * input, struct node * p) {
     int i = 0;
-    int l = strlen(s);
-    until (i >= l) {
-        int ch = s[i++];
-        if (ch != '~') wch(g, ch); else
-        switch(s[i++]) {
-            default:  wch(g, s[i - 1]); continue;
-            case 'C': wc(g, p); continue;
-            case 'k': wk(g, p); continue;
-            case 'K': /* keep for c_test */
-                ws(g, p->mode == m_forward ? "int c_test = z->c;" :
-                                             "int m_test = z->l - z->c;");
-                continue;
-            case 'R': /* restore for c_test */
-                ws(g, p->mode == m_forward ? "z->c = c_test;" :
-                                             "z->c = z->l - m_test;");
-                continue;
+    int l = strlen(input);
+
+    while (i < l) {
+        int ch = input[i++];
+        if (ch != '~') {
+            write_char(g, ch);
+            continue;
+        }
+        switch (input[i++]) {
+            default: write_char(g, input[i - 1]); continue;
+            case 'C': write_comment(g, p); continue;
+            case 'k': wk(g, p, false); continue;
+            case 'K': wk(g, p, true); continue;
             case 'i': winc(g, p); continue;
-            case 'l': wlim(g, p); continue;
-            case 'f': wf(g); continue;
-            case 'M': wm(g); continue;
-            case 'N': wnl(g); continue;
-            case '{': wbs(g); continue;
-            case '}': wbe(g); continue;
-            case 'S': ws(g, g->S[s[i++] - '0']); continue;
-            case 'I': wi(g, g->I[s[i++] - '0']); continue;
-            case 'J': wi3(g, g->I[s[i++] - '0']); continue;
-            case 'V': wv(g, g->V[s[i++] - '0']); continue;
-            case 'W': wvn(g, g->V[s[i++] - '0']); continue;
-            case 'L': wlitref(g, g->L[s[i++] - '0']); continue;
-            case 'A': wlitarray(g, g->L[s[i++] - '0']); continue;
+            case 'l': write_check_limit(g, p); continue;
+            case 'f': write_failure(g, p); continue;
+            case 'M': write_margin(g); continue;
+            case 'N': write_newline(g); continue;
+            case '{': write_block_start(g); continue;
+            case '}': write_block_end(g); continue;
+            case 'S': write_string(g, g->S[input[i++] - '0']); continue;
+            case 'I': write_int(g, g->I[input[i++] - '0']); continue;
+            case 'J': wi3(g, g->I[input[i++] - '0']); continue;
+            case 'V': write_varref(g, g->V[input[i++] - '0']); continue;
+            case 'W': write_varname(g, g->V[input[i++] - '0']); continue;
+            case 'L': wlitref(g, g->L[input[i++] - '0']); continue;
+            case 'A': wlitarray(g, g->L[input[i++] - '0']); continue;
+            case 'c': wlitch(g, g->I[input[i++] - '0']); continue;
+            case 'a': write_data_address(g, p); continue;
             case '+': g->margin++; continue;
             case '-': g->margin--; continue;
             case '$': /* insert_s, insert_v etc */
-                wch(g, p->literalstring == 0 ? 'v' : 's');
+                write_char(g, p->literalstring == 0 ? 'v' : 's');
                 continue;
-            case 'p': ws(g, g->options->externals_prefix); continue;
+            case 'p': write_string(g, g->options->externals_prefix); continue;
         }
     }
 }
 
-static void w(struct generator * g, const char * s) { wp(g, s, 0); }
+static void w(struct generator * g, const char * s) {
+    writef(g, s, 0);
+}
 
 static void generate_AE(struct generator * g, struct node * p) {
-    char * s;
+    const char * s;
     switch (p->type) {
         case c_name:
-            wv(g, p->name); break;
+            write_varref(g, p->name); break;
         case c_number:
-            wi(g, p->number); break;
+            write_int(g, p->number); break;
         case c_maxint:
-            ws(g, "MAXINT"); break;
+            write_string(g, "MAXINT"); break;
         case c_minint:
-            ws(g, "MININT"); break;
+            write_string(g, "MININT"); break;
         case c_neg:
-            wch(g, '-'); generate_AE(g, p->right); break;
+            write_char(g, '-'); generate_AE(g, p->right); break;
         case c_multiply:
             s = " * "; goto label0;
         case c_plus:
@@ -301,28 +364,45 @@ static void generate_AE(struct generator * g, struct node * p) {
         case c_divide:
             s = " / ";
         label0:
-            wch(g, '('); generate_AE(g, p->left);
-            ws(g, s); generate_AE(g, p->right); wch(g, ')'); break;
-        case c_sizeof:
-            g->V[0] = p->name;
-            w(g, "SIZE(~V0)"); break;
+            write_char(g, '('); generate_AE(g, p->left);
+            write_string(g, s); generate_AE(g, p->right); write_char(g, ')'); break;
         case c_cursor:
             w(g, "z->c"); break;
         case c_limit:
             w(g, p->mode == m_forward ? "z->l" : "z->lb"); break;
+        case c_len:
+            if (g->options->encoding == ENC_UTF8) {
+                w(g, "len_utf8(z->p)");
+                break;
+            }
+            /* FALLTHRU */
         case c_size:
-            w(g, "SIZE(z->p)"); break;
+            w(g, "SIZE(z->p)");
+            break;
+        case c_lenof:
+            if (g->options->encoding == ENC_UTF8) {
+                g->V[0] = p->name;
+                w(g, "len_utf8(~V0)");
+                break;
+            }
+            /* FALLTHRU */
+        case c_sizeof:
+            g->V[0] = p->name;
+            w(g, "SIZE(~V0)");
+            break;
     }
 }
 
 /* K_needed() tests to see if we really need to keep c. Not true when the
-   the command does not touch the cursor. This and repeat_score() could be
+   command does not touch the cursor. This and repeat_score() could be
    elaborated almost indefinitely.
 */
 
-static int K_needed(struct generator * g, struct node * p) {
-    until (p == 0) {
+static int K_needed_(struct generator * g, struct node * p, int call_depth) {
+    while (p) {
         switch (p->type) {
+            case c_atlimit:
+            case c_do:
             case c_dollar:
             case c_leftslice:
             case c_rightslice:
@@ -338,17 +418,28 @@ static int K_needed(struct generator * g, struct node * p) {
             case c_ls:
             case c_le:
             case c_sliceto:
+            case c_booltest:
+            case c_set:
+            case c_unset:
             case c_true:
             case c_false:
             case c_debug:
                 break;
 
             case c_call:
-                if (K_needed(g, p->name->definition)) return true;
+                /* Recursive functions aren't typical in snowball programs, so
+                 * make the pessimistic assumption that keep is needed if we
+                 * hit a generous limit on recursion.  It's not likely to make
+                 * a difference to any real world program, but means we won't
+                 * recurse until we run out of stack for pathological cases.
+                 */
+                if (call_depth >= 100) return true;
+                if (K_needed_(g, p->name->definition, call_depth + 1))
+                    return true;
                 break;
 
             case c_bra:
-                if (K_needed(g, p->left)) return true;
+                if (K_needed_(g, p->left, call_depth)) return true;
                 break;
 
             default: return true;
@@ -358,10 +449,13 @@ static int K_needed(struct generator * g, struct node * p) {
     return false;
 }
 
-static int repeat_score(struct generator * g, struct node * p) {
+extern int K_needed(struct generator * g, struct node * p) {
+    return K_needed_(g, p, 0);
+}
+
+static int repeat_score(struct generator * g, struct node * p, int call_depth) {
     int score = 0;
-    until (p == 0)
-    {
+    while (p) {
         switch (p->type) {
             case c_dollar:
             case c_leftslice:
@@ -382,11 +476,25 @@ static int repeat_score(struct generator * g, struct node * p) {
                 break;
 
             case c_call:
-                score += repeat_score(g, p->name->definition);
+                /* Recursive functions aren't typical in snowball programs, so
+                 * make the pessimistic assumption that repeat requires cursor
+                 * reinstatement if we hit a generous limit on recursion.  It's
+                 * not likely to make a difference to any real world program,
+                 * but means we won't recurse until we run out of stack for
+                 * pathological cases.
+                 */
+                if (call_depth >= 100) {
+                    return 2;
+                }
+                score += repeat_score(g, p->name->definition, call_depth + 1);
+                if (score >= 2)
+                    return score;
                 break;
 
             case c_bra:
-                score += repeat_score(g, p->left);
+                score += repeat_score(g, p->left, call_depth);
+                if (score >= 2)
+                    return score;
                 break;
 
             case c_name:
@@ -395,9 +503,12 @@ static int repeat_score(struct generator * g, struct node * p) {
             case c_grouping:
             case c_non:
             case c_hop:
-                score = score + 1; break;
+                if (++score >= 2)
+                    return score;
+                break;
 
-            default: score = 2; break;
+            default:
+                return 2;
         }
         p = p->right;
     }
@@ -406,25 +517,28 @@ static int repeat_score(struct generator * g, struct node * p) {
 
 /* tests if an expression requires cursor reinstatement in a repeat */
 
-static int repeat_restore(struct generator * g, struct node * p) {
-    return repeat_score(g, p) >= 2;
+extern int repeat_restore(struct generator * g, struct node * p) {
+    return repeat_score(g, p, 0) >= 2;
 }
 
 static void generate_bra(struct generator * g, struct node * p) {
     p = p->left;
-    until (p == 0) { generate(g, p); p = p->right; }
+    while (p) {
+        generate(g, p);
+        p = p->right;
+    }
 }
 
 static void generate_and(struct generator * g, struct node * p) {
     int keep_c = 0;
     if (K_needed(g, p->left)) {
-        wp(g, "~{~k~C", p);
+        writef(g, "~{~k~C", p);
         keep_c = g->keep_count;
     } else {
-        wp(g, "~M~C", p);
+        writef(g, "~M~C", p);
     }
     p = p->left;
-    until (p == 0) {
+    while (p) {
         generate(g, p);
         if (keep_c && p->right != 0) {
             w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
@@ -439,19 +553,19 @@ static void generate_or(struct generator * g, struct node * p) {
 
     int used = g->label_used;
     int a0 = g->failure_label;
-    const char * a1 = g->failure_string;
+    int a1 = g->failure_keep_count;
 
     int out_lab = new_label(g);
 
     if (K_needed(g, p->left)) {
-        wp(g, "~{~k~C", p);
+        writef(g, "~{~k~C", p);
         keep_c = g->keep_count;
     } else {
-        wp(g, "~M~C", p);
+        writef(g, "~M~C", p);
     }
     p = p->left;
-    g->failure_string = 0;
-    until (p->right == 0) {
+    g->failure_keep_count = 0;
+    while (p->right) {
         g->failure_label = new_label(g);
         g->label_used = 0;
         generate(g, p);
@@ -465,7 +579,7 @@ static void generate_or(struct generator * g, struct node * p) {
     }
     g->label_used = used;
     g->failure_label = a0;
-    g->failure_string = a1;
+    g->failure_keep_count = a1;
 
     generate(g, p);
     if (keep_c) w(g, "~}");
@@ -474,7 +588,7 @@ static void generate_or(struct generator * g, struct node * p) {
 
 static void generate_backwards(struct generator * g, struct node * p) {
 
-    wp(g,"~Mz->lb = z->c; z->c = z->l;~C~N", p);
+    writef(g, "~Mz->lb = z->c; z->c = z->l;~C~N", p);
     generate(g, p->left);
     w(g, "~Mz->c = z->lb;~N");
 }
@@ -485,18 +599,18 @@ static void generate_not(struct generator * g, struct node * p) {
 
     int used = g->label_used;
     int a0 = g->failure_label;
-    const char * a1 = g->failure_string;
+    int a1 = g->failure_keep_count;
 
     if (K_needed(g, p->left)) {
-        wp(g, "~{~k~C", p);
+        writef(g, "~{~k~C", p);
         keep_c = g->keep_count;
     } else {
-        wp(g, "~M~C", p);
+        writef(g, "~M~C", p);
     }
 
     g->failure_label = new_label(g);
     g->label_used = 0;
-    g->failure_string = 0;
+    g->failure_keep_count = 0;
     generate(g, p->left);
 
     {
@@ -505,9 +619,9 @@ static void generate_not(struct generator * g, struct node * p) {
 
         g->label_used = used;
         g->failure_label = a0;
-        g->failure_string = a1;
+        g->failure_keep_count = a1;
 
-        w(g, "~M~f~N");
+        writef(g, "~M~f~N", p);
         if (u)
             wsetl(g, l);
     }
@@ -518,20 +632,14 @@ static void generate_not(struct generator * g, struct node * p) {
 
 
 static void generate_try(struct generator * g, struct node * p) {
-    int keep_c = K_needed(g, p->left);
-
-    if (keep_c) {
-        if (p->mode == m_forward) {
-            wp(g, "~{int c_keep = z->c;~C", p);
-            g->failure_string = "z->c = c_keep;";
-        } else {
-            wp(g, "~{int m_keep = z->l - z->c;/* (void) m_keep;*/~C", p);
-            g->failure_string = "z->c = z->l - m_keep;";
-        }
+    int keep_c = 0;
+    if (K_needed(g, p->left)) {
+        writef(g, "~{~k~C", p);
+        keep_c = g->keep_count;
     } else {
-        wp(g, "~M~C", p);
-        g->failure_string = 0;
+        writef(g, "~M~C", p);
     }
+    g->failure_keep_count = keep_c;
 
     g->failure_label = new_label(g);
     g->label_used = 0;
@@ -544,47 +652,65 @@ static void generate_try(struct generator * g, struct node * p) {
 }
 
 static void generate_set(struct generator * g, struct node * p) {
-    g->V[0] = p->name; wp(g, "~M~V0 = 1;~C", p);
+    g->V[0] = p->name; writef(g, "~M~V0 = 1;~C", p);
 }
 
 static void generate_unset(struct generator * g, struct node * p) {
-    g->V[0] = p->name; wp(g, "~M~V0 = 0;~C", p);
+    g->V[0] = p->name; writef(g, "~M~V0 = 0;~C", p);
 }
 
 static void generate_fail(struct generator * g, struct node * p) {
     generate(g, p->left);
-    wp(g, "~M~f~C", p);
+    writef(g, "~M~f~C", p);
 }
 
 /* generate_test() also implements 'reverse' */
 
 static void generate_test(struct generator * g, struct node * p) {
-    int keep_c = K_needed(g, p->left);
-    if (keep_c) wp(g, "~{~K~C", p);
-           else wp(g, "~M~C", p);
+    int keep_c = 0;
+    if (K_needed(g, p->left)) {
+        keep_c = ++g->keep_count;
+        w(g, p->mode == m_forward ? "~{int c_test" :
+                                    "~{int m_test");
+        write_int(g, keep_c);
+        w(g, p->mode == m_forward ? " = z->c;" :
+                                    " = z->l - z->c;");
+        writef(g, "~C", p);
+    } else writef(g, "~M~C", p);
 
     generate(g, p->left);
 
-    if (keep_c) wp(g, "~M~R~N"
-                   "~}", p);
+    if (keep_c) {
+        w(g, p->mode == m_forward ? "~Mz->c = c_test" :
+                                    "~Mz->c = z->l - m_test");
+        write_int(g, keep_c);
+        writef(g, ";~N~}", p);
+    }
 }
 
 static void generate_do(struct generator * g, struct node * p) {
     int keep_c = 0;
     if (K_needed(g, p->left)) {
-        wp(g, "~{~k~C", p);
+        writef(g, "~{~k~C", p);
         keep_c = g->keep_count;
     } else {
-        wp(g, "~M~C", p);
+        writef(g, "~M~C", p);
     }
 
-    g->failure_label = new_label(g);
-    g->label_used = 0;
-    g->failure_string = 0;
-    generate(g, p->left);
+    if (p->left->type == c_call) {
+        /* Optimise do <call> */
+        g->V[0] = p->left->name;
+        writef(g, "~{int ret = ~V0(z);~C", p->left);
+        w(g, "~Mif (ret < 0) return ret;~N~}");
+    } else {
+        g->failure_label = new_label(g);
+        g->label_used = 0;
+        g->failure_keep_count = 0;
+        generate(g, p->left);
 
-    if (g->label_used)
-        wsetl(g, g->failure_label);
+        if (g->label_used)
+            wsetl(g, g->failure_label);
+    }
     if (keep_c) {
         w(g, "~M"); wrestore(g, p, keep_c);
         w(g, "~N~}");
@@ -592,17 +718,17 @@ static void generate_do(struct generator * g, struct node * p) {
 }
 
 static void generate_next(struct generator * g, struct node * p) {
-    if (g->options->utf8) {
+    if (g->options->encoding == ENC_UTF8) {
         if (p->mode == m_forward)
             w(g, "~{int ret = skip_utf8(z->p, z->c, 0, z->l, 1");
         else
             w(g, "~{int ret = skip_utf8(z->p, z->c, z->lb, 0, -1");
-        wp(g, ");~N"
+        writef(g, ");~N"
               "~Mif (ret < 0) ~f~N"
               "~Mz->c = ret;~C"
               "~}", p);
     } else
-        wp(g, "~M~l~N"
+        writef(g, "~M~l~N"
               "~M~i~C", p);
 }
 
@@ -611,21 +737,21 @@ static void generate_GO_grouping(struct generator * g, struct node * p, int is_g
     struct grouping * q = p->name->grouping;
     g->S[0] = p->mode == m_forward ? "" : "_b";
     g->S[1] = complement ? "in" : "out";
-    g->S[2] = g->options->utf8 ? "_U" : "";
+    g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : "";
     g->V[0] = p->name;
     g->I[0] = q->smallest_ch;
     g->I[1] = q->largest_ch;
     if (is_goto) {
-	wp(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1) < 0) ~f /* goto */~C", p);
+        writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1) < 0) ~f~C", p);
     } else {
-	wp(g, "~{ /* gopast */~C"
-	      "~Mint ret = ~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1);~N"
-	      "~Mif (ret < 0) ~f~N", p);
-	if (p->mode == m_forward)
-	    w(g, "~Mz->c += ret;~N");
-	else
-	    w(g, "~Mz->c -= ret;~N");
-	w(g, "~}");
+        writef(g, "~{~C"
+              "~Mint ret = ~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1);~N"
+              "~Mif (ret < 0) ~f~N", p);
+        if (p->mode == m_forward)
+            w(g, "~Mz->c += ret;~N");
+        else
+            w(g, "~Mz->c -= ret;~N");
+        w(g, "~}");
     }
 }
 
@@ -634,23 +760,26 @@ static void generate_GO(struct generator * g, struct node * p, int style) {
 
     int used = g->label_used;
     int a0 = g->failure_label;
-    const char * a1 = g->failure_string;
+    int a1 = g->failure_keep_count;
 
     if (p->left->type == c_grouping || p->left->type == c_non) {
-	/* Special case for "goto" or "gopast" when used on a grouping or an
-	 * inverted grouping - the movement of c by the matching action is
-	 * exactly what we want! */
+        /* Special case for "goto" or "gopast" when used on a grouping or an
+         * inverted grouping - the movement of c by the matching action is
+         * exactly what we want! */
 #ifdef OPTIMISATION_WARNINGS
-	printf("Optimising %s %s\n", style ? "goto" : "gopast", p->left->type == c_non ? "non" : "grouping");
+        printf("Optimising %s %s\n", style ? "goto" : "gopast", p->left->type == c_non ? "non" : "grouping");
 #endif
-	generate_GO_grouping(g, p->left, style, p->left->type == c_non);
-	return;
+        if (g->options->comments) {
+            writef(g, "~M~C", p);
+        }
+        generate_GO_grouping(g, p->left, style, p->left->type == c_non);
+        return;
     }
 
-    w(g, "~Mwhile(1) {"); wp(g, "~C~+", p);
+    w(g, "~Mwhile(1) {"); writef(g, "~C~+", p);
 
     if (style == 1 || repeat_restore(g, p->left)) {
-        wp(g, "~M~k~N", p);
+        writef(g, "~M~k~N", p);
         keep_c = g->keep_count;
     }
 
@@ -671,16 +800,16 @@ static void generate_GO(struct generator * g, struct node * p, int style) {
 
     g->label_used = used;
     g->failure_label = a0;
-    g->failure_string = a1;
+    g->failure_keep_count = a1;
 
-/*  wp(g, "~M~l~N"
+/*  writef(g, "~M~l~N"
           "~M~i~N", p);  */
     generate_next(g, p);
     w(g, "~}");
 }
 
 static void generate_loop(struct generator * g, struct node * p) {
-    w(g, "~{int i; for (i = "); generate_AE(g, p->AE); wp(g, "; i > 0; i--)~C"
+    w(g, "~{int i; for (i = "); generate_AE(g, p->AE); writef(g, "; i > 0; i--)~C"
             "~{", p);
 
     generate(g, p->left);
@@ -689,18 +818,22 @@ static void generate_loop(struct generator * g, struct node * p) {
          "~}");
 }
 
-static void generate_repeat(struct generator * g, struct node * p, int atleast_case) {
+static void generate_repeat_or_atleast(struct generator * g, struct node * p, int atleast_case) {
     int keep_c = 0;
-    wp(g, "~Mwhile(1) {~C~+", p);
+    if (atleast_case) {
+        writef(g, "~Mwhile(1) {~+~N", p);
+    } else {
+        writef(g, "~Mwhile(1) {~+~C", p);
+    }
 
     if (repeat_restore(g, p->left)) {
-        wp(g, "~M~k~N", p);
+        writef(g, "~M~k~N", p);
         keep_c = g->keep_count;
     }
 
     g->failure_label = new_label(g);
     g->label_used = 0;
-    g->failure_string = 0;
+    g->failure_keep_count = 0;
     generate(g, p->left);
 
     if (atleast_case) w(g, "~Mi--;~N");
@@ -717,130 +850,120 @@ static void generate_repeat(struct generator * g, struct node * p, int atleast_c
       "~}");
 }
 
+static void generate_repeat(struct generator * g, struct node * p) {
+    generate_repeat_or_atleast(g, p, false);
+}
+
 static void generate_atleast(struct generator * g, struct node * p) {
-    w(g, "~{int i = "); generate_AE(g, p->AE); w(g, ";~N");
+    w(g, "~{int i = "); generate_AE(g, p->AE); w(g, ";~C");
     {
         int used = g->label_used;
         int a0 = g->failure_label;
-        const char * a1 = g->failure_string;
+        int a1 = g->failure_keep_count;
 
-        generate_repeat(g, p, true);
+        generate_repeat_or_atleast(g, p, true);
 
         g->label_used = used;
         g->failure_label = a0;
-        g->failure_string = a1;
+        g->failure_keep_count = a1;
     }
-    w(g, "~Mif (i > 0) ~f~N"
-      "~}");
+    writef(g, "~Mif (i > 0) ~f~N"
+       "~}", p);
 }
 
 static void generate_setmark(struct generator * g, struct node * p) {
     g->V[0] = p->name;
-    wp(g, "~M~V0 = z->c;~C", p);
+    writef(g, "~M~V0 = z->c;~C", p);
 }
 
 static void generate_tomark(struct generator * g, struct node * p) {
     g->S[0] = p->mode == m_forward ? ">" : "<";
 
-    w(g, "~Mif (z->c ~S0 "); generate_AE(g, p->AE); w(g, ") ~f~N");
-    w(g, "~Mz->c = "); generate_AE(g, p->AE); wp(g, ";~C", p);
+    w(g, "~Mif (z->c ~S0 "); generate_AE(g, p->AE); writef(g, ") ~f~N", p);
+    w(g, "~Mz->c = "); generate_AE(g, p->AE); writef(g, ";~C", p);
 }
 
 static void generate_atmark(struct generator * g, struct node * p) {
 
-    w(g, "~Mif (z->c != "); generate_AE(g, p->AE); wp(g, ") ~f~C", p);
+    w(g, "~Mif (z->c != "); generate_AE(g, p->AE); writef(g, ") ~f~C", p);
 }
 
 static void generate_hop(struct generator * g, struct node * p) {
     g->S[0] = p->mode == m_forward ? "+" : "-";
     g->S[1] = p->mode == m_forward ? "0" : "z->lb";
-    if (g->options->utf8) {
+    if (g->options->encoding == ENC_UTF8) {
         w(g, "~{int ret = skip_utf8(z->p, z->c, ~S1, z->l, ~S0 ");
-        generate_AE(g, p->AE); wp(g, ");~C", p);
-        w(g, "~Mif (ret < 0) ~f~N");
+        generate_AE(g, p->AE); writef(g, ");~C", p);
+        writef(g, "~Mif (ret < 0) ~f~N", p);
     } else {
         w(g, "~{int ret = z->c ~S0 ");
-        generate_AE(g, p->AE); wp(g, ";~C", p);
-        w(g, "~Mif (~S1 > ret || ret > z->l) ~f~N");
+        generate_AE(g, p->AE); writef(g, ";~C", p);
+        writef(g, "~Mif (~S1 > ret || ret > z->l) ~f~N", p);
     }
-    wp(g, "~Mz->c = ret;~C"
+    writef(g, "~Mz->c = ret;~N"
           "~}", p);
 }
 
 static void generate_delete(struct generator * g, struct node * p) {
-    wp(g, "~{int ret = slice_del(z);~C", p);
-    wp(g, "~Mif (ret < 0) return ret;~N"
+    writef(g, "~{int ret = slice_del(z);~C", p);
+    writef(g, "~Mif (ret < 0) return ret;~N"
           "~}", p);
 }
 
 static void generate_tolimit(struct generator * g, struct node * p) {
     g->S[0] = p->mode == m_forward ? "" : "b";
-    wp(g, "~Mz->c = z->l~S0;~C", p);
+    writef(g, "~Mz->c = z->l~S0;~C", p);
 }
 
 static void generate_atlimit(struct generator * g, struct node * p) {
     g->S[0] = p->mode == m_forward ? "" : "b";
     g->S[1] = p->mode == m_forward ? "<" : ">";
-    wp(g, "~Mif (z->c ~S1 z->l~S0) ~f~C", p);
+    writef(g, "~Mif (z->c ~S1 z->l~S0) ~f~C", p);
 }
 
 static void generate_leftslice(struct generator * g, struct node * p) {
     g->S[0] = p->mode == m_forward ? "bra" : "ket";
-    wp(g, "~Mz->~S0 = z->c;~C", p);
+    writef(g, "~Mz->~S0 = z->c;~C", p);
 }
 
 static void generate_rightslice(struct generator * g, struct node * p) {
     g->S[0] = p->mode == m_forward ? "ket" : "bra";
-    wp(g, "~Mz->~S0 = z->c;~C", p);
+    writef(g, "~Mz->~S0 = z->c;~C", p);
 }
 
 static void generate_assignto(struct generator * g, struct node * p) {
     g->V[0] = p->name;
-    wp(g, "~M~V0 = assign_to(z, ~V0);~C"
+    writef(g, "~M~V0 = assign_to(z, ~V0);~C"
           "~Mif (~V0 == 0) return -1;~C", p);
 }
 
 static void generate_sliceto(struct generator * g, struct node * p) {
     g->V[0] = p->name;
-    wp(g, "~M~V0 = slice_to(z, ~V0);~C"
-          "~Mif (~V0 == 0) return -1;~C", p);
-}
-
-static void generate_data_address(struct generator * g, struct node * p) {
-
-    symbol * b = p->literalstring;
-    if (b != 0) {
-        wi(g, SIZE(b)); w(g, ", ");
-        wlitref(g, b);
-    } else
-        wv(g, p->name);
+    writef(g, "~M~V0 = slice_to(z, ~V0);~C"
+          "~Mif (~V0 == 0) return -1;~N", p);
 }
 
 static void generate_insert(struct generator * g, struct node * p, int style) {
 
     int keep_c = style == c_attach;
     if (p->mode == m_backward) keep_c = !keep_c;
-    wp(g, "~{", p);
-    if (keep_c) w(g, "int c_keep = z->c;~N~M");
-    wp(g, "int ret = insert_~$(z, z->c, z->c, ", p);
-    generate_data_address(g, p);
-    wp(g, ");~C", p);
-    if (keep_c) w(g, "~Mz->c = c_keep;~N");
-    wp(g, "~Mif (ret < 0) return ret;~N"
+    writef(g, "~{int ret;~N", p);
+    if (keep_c) w(g, "~{int saved_c = z->c;~N");
+    writef(g, "~Mret = insert_~$(z, z->c, z->c, ~a);~C", p);
+    if (keep_c) w(g, "~Mz->c = saved_c;~N~}");
+    writef(g, "~Mif (ret < 0) return ret;~N"
           "~}", p);
 }
 
 static void generate_assignfrom(struct generator * g, struct node * p) {
 
     int keep_c = p->mode == m_forward; /* like 'attach' */
-    wp(g, "~{", p);
-    if (keep_c) wp(g, "int c_keep = z->c;~N"
-                   "~Mret = insert_~$(z, z->c, z->l, ", p);
-                else wp(g, "ret = insert_~$(z, z->lb, z->c, ", p);
-    generate_data_address(g, p);
-    wp(g, ");~C", p);
-    if (keep_c) w(g, "~Mz->c = c_keep;~N");
-    wp(g, "~Mif (ret < 0) return ret;~N"
+    writef(g, "~{int ret;~N", p);
+    if (keep_c) writef(g, "~{int saved_c = z->c;~N", p);
+    w(g, "~Mret = ");
+    writef(g, keep_c ? "insert_~$(z, z->c, z->l, ~a);~C" : "insert_~$(z, z->lb, z->c, ~a);~C", p);
+    if (keep_c) w(g, "~Mz->c = saved_c;~N~}");
+    writef(g, "~Mif (ret < 0) return ret;~N"
           "~}", p);
 }
 
@@ -849,81 +972,132 @@ static void generate_assignfrom(struct generator * g, struct node * p) {
 static void generate_slicefrom(struct generator * g, struct node * p) {
 
 /*  w(g, "~Mslice_from_s(z, ");   <============= bug! should be: */
-    wp(g, "~{int ret = slice_from_~$(z, ", p);
-    generate_data_address(g, p);
-    wp(g, ");~C", p);
-    wp(g, "~Mif (ret < 0) return ret;~N"
+    writef(g, "~{int ret = slice_from_~$(z, ~a);~C", p);
+    writef(g, "~Mif (ret < 0) return ret;~N"
           "~}", p);
 }
 
 static void generate_setlimit(struct generator * g, struct node * p) {
     int keep_c;
-    wp(g, "~{int mlimit;~C"
-          "~M~k~N"
-          , p);
-    keep_c = g->keep_count;
-    generate(g, p->left);
-    if (p->mode == m_forward) w(g, "~Mmlimit = z->l - z->c; z->l = z->c;~N");
-                         else w(g, "~Mmlimit = z->lb; z->lb = z->c;~N");
-    w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
-    g->failure_string = p->mode == m_forward ? "z->l += mlimit;" :
-                                               "z->lb = mlimit;";
+    if (p->left && p->left->type == c_tomark) {
+        /* Special case for:
+         *
+         *   setlimit tomark AE for C
+         *
+         * All uses of setlimit in the current stemmers we ship follow this
+         * pattern, and by special-casing we can avoid having to save and
+         * restore c.
+         */
+        struct node * q = p->left;
+
+        ++g->keep_count;
+        writef(g, "~N~{int mlimit", p);
+        write_int(g, g->keep_count);
+        writef(g, ";~C", p);
+        keep_c = g->keep_count;
+
+        g->S[0] = q->mode == m_forward ? ">" : "<";
+
+        w(g, "~Mif (z->c ~S0 "); generate_AE(g, q->AE); writef(g, ") ~f~N", q);
+        w(g, "~Mmlimit");
+        write_int(g, keep_c);
+        if (p->mode == m_forward) {
+            w(g, " = z->l - z->c; z->l = ");
+        } else {
+            w(g, " = z->lb; z->lb = ");
+        }
+        generate_AE(g, q->AE);
+        w(g, ";~N");
+    } else {
+        writef(g, "~{~K~C", p);
+        keep_c = g->keep_count;
+        generate(g, p->left);
+
+        w(g, "~Mmlimit");
+        write_int(g, keep_c);
+        if (p->mode == m_forward)
+            w(g, " = z->l - z->c; z->l = z->c;~N");
+        else
+            w(g, " = z->lb; z->lb = z->c;~N");
+        w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+    }
+
+    g->failure_keep_count = -keep_c;
     generate(g, p->aux);
-    wms(g, g->failure_string);
+    w(g, "~M");
+    wrestorelimit(g, p, -g->failure_keep_count);
     w(g, "~N"
       "~}");
 }
 
+/* dollar sets snowball up to operate on a string variable as if it were the
+ * current string */
 static void generate_dollar(struct generator * g, struct node * p) {
 
     int used = g->label_used;
     int a0 = g->failure_label;
-    const char * a1 = g->failure_string;
+    int a1 = g->failure_keep_count;
+    int keep_token;
     g->failure_label = new_label(g);
     g->label_used = 0;
-    g->failure_string = 0;
+    g->failure_keep_count = 0;
 
+    keep_token = ++g->keep_count;
+    g->I[0] = keep_token;
+    writef(g, "~{struct SN_env env~I0 = * z;~C", p);
     g->V[0] = p->name;
-    wp(g, "~{struct SN_env env = * z;~C"
-             "~Mint failure = 1; /* assume failure */~N"
-             "~Mz->p = ~V0;~N"
-             "~Mz->lb = z->c = 0;~N"
-             "~Mz->l = SIZE(z->p);~N", p);
+    /* Assume failure. */
+    writef(g, "~Mint failure = 1;~N"
+          "~Mz->p = ~V0;~N"
+          "~Mz->lb = z->c = 0;~N"
+          "~Mz->l = SIZE(z->p);~N", p);
     generate(g, p->left);
-    w(g, "~Mfailure = 0; /* mark success */~N");
+    /* Mark success. */
+    w(g, "~Mfailure = 0;~N");
     if (g->label_used)
         wsetl(g, g->failure_label);
     g->V[0] = p->name; /* necessary */
 
     g->label_used = used;
     g->failure_label = a0;
-    g->failure_string = a1;
+    g->failure_keep_count = a1;
 
-    w(g, "~M~V0 = z->p;~N"
-         "~M* z = env;~N"
-         "~Mif (failure) ~f~N~}");
+    g->I[0] = keep_token;
+    writef(g, "~M~V0 = z->p;~N"
+          "~M* z = env~I0;~N"
+          "~Mif (failure) ~f~N~}", p);
 }
 
 static void generate_integer_assign(struct generator * g, struct node * p, char * s) {
 
     g->V[0] = p->name;
     g->S[0] = s;
-    w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); wp(g, ";~C", p);
+    w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); writef(g, ";~C", p);
 }
 
 static void generate_integer_test(struct generator * g, struct node * p, char * s) {
 
-    g->V[0] = p->name;
-    g->S[0] = s;
-    w(g, "~Mif (!(~V0 ~S0 "); generate_AE(g, p->AE); wp(g, ")) ~f~C", p);
+    w(g, "~Mif (!(");
+    generate_AE(g, p->left);
+    write_char(g, ' ');
+    write_string(g, s);
+    write_char(g, ' ');
+    generate_AE(g, p->AE);
+    writef(g, ")) ~f~C", p);
 }
 
 static void generate_call(struct generator * g, struct node * p) {
 
     g->V[0] = p->name;
-    wp(g, "~{int ret = ~V0(z);~C"
-          "~Mif (ret == 0) ~f~N"
-          "~Mif (ret < 0) return ret;~N~}", p);
+    writef(g, "~{int ret = ~V0(z);~C", p);
+    if (g->failure_keep_count == 0 && g->failure_label == x_return) {
+        /* Combine the two tests in this special case for better optimisation
+         * and clearer generated code. */
+        writef(g, "~Mif (ret <= 0) return ret;~N~}", p);
+    } else {
+        writef(g, "~Mif (ret == 0) ~f~N"
+              "~Mif (ret < 0) return ret;~N~}", p);
+    }
 }
 
 static void generate_grouping(struct generator * g, struct node * p, int complement) {
@@ -931,27 +1105,47 @@ static void generate_grouping(struct generator * g, struct node * p, int complem
     struct grouping * q = p->name->grouping;
     g->S[0] = p->mode == m_forward ? "" : "_b";
     g->S[1] = complement ? "out" : "in";
-    g->S[2] = g->options->utf8 ? "_U" : "";
+    g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : "";
     g->V[0] = p->name;
     g->I[0] = q->smallest_ch;
     g->I[1] = q->largest_ch;
-    wp(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 0)) ~f~C", p);
+    writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 0)) ~f~C", p);
 }
 
 static void generate_namedstring(struct generator * g, struct node * p) {
 
     g->S[0] = p->mode == m_forward ? "" : "_b";
     g->V[0] = p->name;
-    wp(g, "~Mif (!(eq_v~S0(z, ~V0))) ~f~C", p);
+    writef(g, "~Mif (!(eq_v~S0(z, ~V0))) ~f~C", p);
 }
 
 static void generate_literalstring(struct generator * g, struct node * p) {
     symbol * b = p->literalstring;
-    g->S[0] = p->mode == m_forward ? "" : "_b";
-    g->I[0] = SIZE(b);
-    g->L[0] = b;
+    if (SIZE(b) == 1) {
+        /* It's quite common to compare with a single character literal string,
+         * so just inline the simpler code for this case rather than making a
+         * function call.  In UTF-8 mode, only do this for the ASCII subset,
+         * since multi-byte characters are more complex to test against.
+         */
+        if (g->options->encoding == ENC_UTF8 && *b >= 128) {
+            printf("single byte %d\n", *b);
+            exit(1);
+        }
+        g->I[0] = *b;
+        if (p->mode == m_forward) {
+            writef(g, "~Mif (z->c == z->l || z->p[z->c] != ~c0) ~f~C"
+                  "~Mz->c++;~N", p);
+        } else {
+            writef(g, "~Mif (z->c <= z->lb || z->p[z->c - 1] != ~c0) ~f~C"
+                  "~Mz->c--;~N", p);
+        }
+    } else {
+        g->S[0] = p->mode == m_forward ? "" : "_b";
+        g->I[0] = SIZE(b);
+        g->L[0] = b;
 
-    wp(g, "~Mif (!(eq_s~S0(z, ~I0, ~L0))) ~f~C", p);
+        writef(g, "~Mif (!(eq_s~S0(z, ~I0, ~L0))) ~f~C", p);
+    }
 }
 
 static void generate_define(struct generator * g, struct node * p) {
@@ -961,9 +1155,13 @@ static void generate_define(struct generator * g, struct node * p) {
     g->S[0] = q->type == t_routine ? "static" : "extern";
     g->V[0] = q;
 
-    w(g, "~N~S0 int ~V0(struct SN_env * z) {~N~+");
+    w(g, "~N~S0 int ~V0(struct SN_env * z) {");
+    if (g->options->comments) {
+        write_string(g, p->mode == m_forward ? " /* forwardmode */" : " /* backwardmode */");
+    }
+    w(g, "~N~+");
     if (p->amongvar_needed) w(g, "~Mint among_var;~N");
-    g->failure_string = 0;
+    g->failure_keep_count = 0;
     g->failure_label = x_return;
     g->label_used = 0;
     g->keep_count = 0;
@@ -982,6 +1180,7 @@ static void generate_substring(struct generator * g, struct node * p) {
     int n_cases = 0;
     symbol cases[2];
     int shortest_size = INT_MAX;
+    int shown_comment = 0;
 
     g->S[0] = p->mode == m_forward ? "" : "_b";
     g->I[0] = x->number;
@@ -1019,19 +1218,19 @@ static void generate_substring(struct generator * g, struct node * p) {
             if (n_cases > 2) break;
         }
         if (block == -1) {
-            if (ch == cases[0]) continue;
+            if (n_cases > 0 && ch == cases[0]) continue;
             if (n_cases < 2) {
-            cases[n_cases++] = ch;
+                cases[n_cases++] = ch;
             } else if (ch != cases[1]) {
-            ++n_cases;
-            break;
+                ++n_cases;
+                break;
             }
         } else {
             if ((bitmap & (1u << (ch & 0x1f))) == 0) {
-            bitmap |= 1u << (ch & 0x1f);
-            if (n_cases < 2)
-                cases[n_cases] = ch;
-            ++n_cases;
+                bitmap |= 1u << (ch & 0x1f);
+                if (n_cases < 2)
+                    cases[n_cases] = ch;
+                ++n_cases;
             }
         }
     }
@@ -1045,16 +1244,16 @@ static void generate_substring(struct generator * g, struct node * p) {
             sprintf(buf, "z->p[z->c + %d]", shortest_size - 1);
             g->S[1] = buf;
             if (shortest_size == 1) {
-                wp(g, "~Mif (z->c >= z->l || ", p);
+                writef(g, "~Mif (z->c >= z->l", p);
             } else {
-                wp(g, "~Mif (z->c + ~I4 >= z->l || ", p);
+                writef(g, "~Mif (z->c + ~I4 >= z->l", p);
             }
         } else {
             g->S[1] = "z->p[z->c - 1]";
             if (shortest_size == 1) {
-                wp(g, "~Mif (z->c <= z->lb || ", p);
+                writef(g, "~Mif (z->c <= z->lb", p);
             } else {
-                wp(g, "~Mif (z->c - ~I4 <= z->lb || ", p);
+                writef(g, "~Mif (z->c - ~I4 <= z->lb", p);
             }
         }
         if (n_cases == 0) {
@@ -1062,81 +1261,83 @@ static void generate_substring(struct generator * g, struct node * p) {
              * This doesn't seem to be a useful construct, but it is
              * syntactically valid.
              */
-            wp(g, "0", p);
         } else if (n_cases == 1) {
             g->I[4] = cases[0];
-            wp(g, "~S1 != ~I4", p);
+            writef(g, " || ~S1 != ~I4", p);
         } else if (n_cases == 2) {
             g->I[4] = cases[0];
             g->I[5] = cases[1];
-            wp(g, "(~S1 != ~I4 && ~S1 != ~I5)", p);
+            writef(g, " || (~S1 != ~I4 && ~S1 != ~I5)", p);
         } else {
-            wp(g, "~S1 >> 5 != ~I2 || !((~I3 >> (~S1 & 0x1f)) & 1)", p);
+            writef(g, " || ~S1 >> 5 != ~I2 || !((~I3 >> (~S1 & 0x1f)) & 1)", p);
         }
-        ws(g, ") ");
+        write_string(g, ") ");
         if (empty_case != -1) {
             /* If the among includes the empty string, it can never fail
              * so not matching the bitmap means we match the empty string.
              */
             g->I[4] = among_cases[empty_case].result;
-            wp(g, "among_var = ~I4; else~C", p);
+            writef(g, "among_var = ~I4; else~C", p);
         } else {
-            wp(g, "~f~C", p);
+            writef(g, "~f~C", p);
         }
+        shown_comment = 1;
     } else {
 #ifdef OPTIMISATION_WARNINGS
         printf("Couldn't shortcut among %d\n", x->number);
 #endif
     }
 
-    if (x->command_count == 0 && x->starter == 0)
-        wp(g, "~Mif (!(find_among~S0(z, a_~I0, ~I1))) ~f~C", p);
-    else
-        wp(g, "~Mamong_var = find_among~S0(z, a_~I0, ~I1);~C"
-              "~Mif (!(among_var)) ~f~N", p);
+    if (!x->amongvar_needed) {
+        writef(g, "~Mif (!(find_among~S0(z, a_~I0, ~I1))) ~f", p);
+        writef(g, shown_comment ? "~N" : "~C", p);
+    } else {
+        writef(g, "~Mamong_var = find_among~S0(z, a_~I0, ~I1);", p);
+        writef(g, shown_comment ? "~N" : "~C", p);
+        writef(g, "~Mif (!(among_var)) ~f~N", p);
+    }
 }
 
 static void generate_among(struct generator * g, struct node * p) {
 
     struct among * x = p->among;
-    int case_number = 1;
 
     if (x->substring == 0) generate_substring(g, p);
-    if (x->command_count == 0 && x->starter == 0) return;
 
-    unless (x->starter == 0) generate(g, x->starter);
+    if (x->starter != 0) generate(g, x->starter);
 
-    p = p->left;
-    if (p != 0 && p->type != c_literalstring) p = p->right;
-    w(g, "~Mswitch(among_var) {~N~+"
-             "~Mcase 0: ~f~N");
-
-    until (p == 0) {
-         if (p->type == c_bra && p->left != 0) {
-             g->I[0] = case_number++;
-             w(g, "~Mcase ~I0:~N~+"); generate(g, p); w(g, "~Mbreak;~N~-");
-         }
-         p = p->right;
+    if (x->command_count == 1 && x->nocommand_count == 0) {
+        /* Only one outcome ("no match" already handled). */
+        generate(g, x->commands[0]);
+    } else if (x->command_count > 0) {
+        int i;
+        writef(g, "~Mswitch (among_var) {~C~+", p);
+        for (i = 1; i <= x->command_count; i++) {
+            g->I[0] = i;
+            w(g, "~Mcase ~I0:~N~+");
+            generate(g, x->commands[i - 1]);
+            w(g, "~Mbreak;~N~-");
+        }
+        w(g, "~}");
     }
-    w(g, "~}");
 }
 
 static void generate_booltest(struct generator * g, struct node * p) {
 
     g->V[0] = p->name;
-    wp(g, "~Mif (!(~V0)) ~f~C", p);
+    writef(g, "~Mif (!(~V0)) ~f~C", p);
 }
 
 static void generate_false(struct generator * g, struct node * p) {
 
-    wp(g, "~M~f~C", p);
+    writef(g, "~M~f~C", p);
 }
 
 static void generate_debug(struct generator * g, struct node * p) {
 
     g->I[0] = g->debug_count++;
     g->I[1] = p->line_number;
-    wp(g, "~Mdebug(z, ~I0, ~I1);~C", p);
+    writef(g, "~Mdebug(z, ~I0, ~I1);~C", p);
 
 }
 
@@ -1144,10 +1345,9 @@ static void generate(struct generator * g, struct node * p) {
 
     int used = g->label_used;
     int a0 = g->failure_label;
-    const char * a1 = g->failure_string;
+    int a1 = g->failure_keep_count;
 
-    switch (p->type)
-    {
+    switch (p->type) {
         case c_define:        generate_define(g, p); break;
         case c_bra:           generate_bra(g, p); break;
         case c_and:           generate_and(g, p); break;
@@ -1163,7 +1363,7 @@ static void generate(struct generator * g, struct node * p) {
         case c_do:            generate_do(g, p); break;
         case c_goto:          generate_GO(g, p, 1); break;
         case c_gopast:        generate_GO(g, p, 0); break;
-        case c_repeat:        generate_repeat(g, p, false); break;
+        case c_repeat:        generate_repeat(g, p); break;
         case c_loop:          generate_loop(g, p); break;
         case c_atleast:       generate_atleast(g, p); break;
         case c_setmark:       generate_setmark(g, p); break;
@@ -1213,30 +1413,40 @@ static void generate(struct generator * g, struct node * p) {
     if (g->failure_label != a0)
         g->label_used = used;
     g->failure_label = a0;
-    g->failure_string = a1;
+    g->failure_keep_count = a1;
 }
 
-static void generate_start_comment(struct generator * g) {
+void write_generated_comment_content(struct generator * g) {
+    w(g, "Generated by Snowball " SNOWBALL_VERSION
+         " - https://snowballstem.org/");
+}
 
-    w(g, "~N/* This file was generated automatically by the Snowball to ANSI C compiler */~N");
+void write_start_comment(struct generator * g,
+                         const char * comment_start,
+                         const char * comment_end) {
+    write_margin(g);
+    w(g, comment_start);
+    write_generated_comment_content(g);
+    if (comment_end) {
+        w(g, comment_end);
+    }
+    w(g, "~N~N");
 }
 
 static void generate_head(struct generator * g) {
 
-    if (g->options->runtime_path == 0) {
-        w(g, "~N#include \"header.h\"~N~N");
-    } else {
-        w(g, "~N#include \"");
-        ws(g, g->options->runtime_path);
+    w(g, "#include \"");
+    if (g->options->runtime_path) {
+        write_string(g, g->options->runtime_path);
         if (g->options->runtime_path[strlen(g->options->runtime_path) - 1] != '/')
-            wch(g, '/');
-        w(g, "header.h\"~N~N");
+            write_char(g, '/');
     }
+    w(g, "header.h\"~N~N");
 }
 
 static void generate_routine_headers(struct generator * g) {
-    struct name * q = g->analyser->names;
-    until (q == 0) {
+    struct name * q;
+    for (q = g->analyser->names; q; q = q->next) {
         g->V[0] = q;
         switch (q->type) {
             case t_routine:
@@ -1254,7 +1464,6 @@ static void generate_routine_headers(struct generator * g) {
                   );
                 break;
         }
-        q = q->next;
     }
 }
 
@@ -1265,12 +1474,11 @@ static void generate_among_table(struct generator * g, struct among * x) {
     g->I[0] = x->number;
     {
         int i;
-        for (i = 0; i < x->literalstring_count; i++)
-        {
+        for (i = 0; i < x->literalstring_count; i++) {
             g->I[1] = i;
             g->I[2] = v->size;
             g->L[0] = v->b;
-            unless (v->size == 0)
+            if (v->size)
                 w(g, "static const symbol s_~I0_~I1[~I2] = ~A0;~N");
             v++;
         }
@@ -1289,12 +1497,21 @@ static void generate_among_table(struct generator * g, struct among * x) {
             g->I[4] = v->result;
             g->S[0] = i < x->literalstring_count - 1 ? "," : "";
 
-            w(g, "/*~J1 */ { ~I2, ");
-            if (v->size == 0) w(g, "0,");
-                         else w(g, "s_~I0_~I1,");
+            if (g->options->comments) {
+                w(g, "/*~J1 */ ");
+            }
+            w(g, "{ ~I2, ");
+            if (v->size == 0) {
+                w(g, "0,");
+            } else {
+                w(g, "s_~I0_~I1,");
+            }
             w(g, " ~I3, ~I4, ");
-            if (v->function == 0) w(g, "0"); else
-                                  wvn(g, v->function);
+            if (v->function == 0) {
+                write_char(g, '0');
+            } else {
+                write_varname(g, v->function);
+            }
             w(g, "}~S0~N");
             v++;
         }
@@ -1303,10 +1520,9 @@ static void generate_among_table(struct generator * g, struct among * x) {
 }
 
 static void generate_amongs(struct generator * g) {
-    struct among * x = g->analyser->amongs;
-    until (x == 0) {
+    struct among * x;
+    for (x = g->analyser->amongs; x; x = x->next) {
         generate_among_table(g, x);
-        x = x->next;
     }
 }
 
@@ -1323,24 +1539,22 @@ static void generate_grouping_table(struct generator * g, struct grouping * q) {
 
     for (i = 0; i < SIZE(b); i++) set_bit(map, b[i] - q->smallest_ch);
 
-    {
-        g->V[0] = q->name;
+    g->V[0] = q->name;
 
-        w(g, "static const unsigned char ~V0[] = { ");
-        for (i = 0; i < size; i++) {
-             wi(g, map[i]);
-             if (i < size - 1) w(g, ", ");
-        }
-        w(g, " };~N~N");
+    w(g, "static const unsigned char ~V0[] = { ");
+    for (i = 0; i < size; i++) {
+        write_int(g, map[i]);
+        if (i < size - 1) w(g, ", ");
     }
+    w(g, " };~N~N");
     lose_b(map);
 }
 
 static void generate_groupings(struct generator * g) {
-    struct grouping * q = g->analyser->groupings;
-    until (q == 0) {
-        generate_grouping_table(g, q);
-        q = q->next;
+    struct grouping * q;
+    for (q = g->analyser->groupings; q; q = q->next) {
+        if (q->name->used)
+            generate_grouping_table(g, q);
     }
 }
 
@@ -1348,10 +1562,9 @@ static void generate_create(struct generator * g) {
 
     int * p = g->analyser->name_count;
     g->I[0] = p[t_string];
-    g->I[1] = p[t_integer];
-    g->I[2] = p[t_boolean];
+    g->I[1] = p[t_integer] + p[t_boolean];
     w(g, "~N"
-         "extern struct SN_env * ~pcreate_env(void) { return SN_create_env(~I0, ~I1, ~I2); }"
+         "extern struct SN_env * ~pcreate_env(void) { return SN_create_env(~I0, ~I1); }"
          "~N");
 }
 
@@ -1371,36 +1584,47 @@ static void generate_create_and_close_templates(struct generator * g) {
 
 static void generate_header_file(struct generator * g) {
 
-    struct name * q = g->analyser->names;
-    char * vp = g->options->variables_prefix;
+    struct name * q;
+    const char * vp = g->options->variables_prefix;
     g->S[0] = vp;
 
-    w(g, "~N"
-         "#ifdef __cplusplus~N"
+    w(g, "#ifdef __cplusplus~N"
          "extern \"C\" {~N"
          "#endif~N");            /* for C++ */
 
     generate_create_and_close_templates(g);
-    until (q == 0) {
+    for (q = g->analyser->names; q; q = q->next) {
         g->V[0] = q;
-        switch (q->type)
-        {
+        switch (q->type) {
             case t_external:
                 w(g, "extern int ~W0(struct SN_env * z);~N");
                 break;
-            case t_string:  g->S[1] = "S"; goto label0;
-            case t_integer: g->S[1] = "I"; goto label0;
-            case t_boolean: g->S[1] = "B";
-            label0:
+            case t_string:
+            case t_integer:
+            case t_boolean:
                 if (vp) {
-                    g->I[0] = q->count;
+                    int count = q->count;
+                    if (count < 0) {
+                        /* Unused variables should get removed from `names`. */
+                        fprintf(stderr, "Optimised out variable ");
+                        report_b(stderr, q->b);
+                        fprintf(stderr, " still in names list\n");
+                        exit(1);
+                    }
+                    if (q->type == t_boolean) {
+                        /* We use a single array for booleans and integers,
+                         * with the integers first.
+                         */
+                        count += g->analyser->name_count[t_integer];
+                    }
+                    g->I[0] = count;
+                    g->I[1] = "SIIrxg"[q->type];
                     w(g, "#define ~S0");
-                    str_append_b(g->outbuf, q->b);
-                    w(g, " (~S1[~I0])~N");
+                    write_b(g, q->b);
+                    w(g, " (~c1[~I0])~N");
                 }
                 break;
         }
-        q = q->next;
     }
 
     w(g, "~N"
@@ -1414,7 +1638,7 @@ static void generate_header_file(struct generator * g) {
 extern void generate_program_c(struct generator * g) {
 
     g->outbuf = str_new();
-    generate_start_comment(g);
+    write_start_comment(g, "/* ", " */");
     generate_head(g);
     generate_routine_headers(g);
     w(g, "#ifdef __cplusplus~N"
@@ -1433,33 +1657,69 @@ extern void generate_program_c(struct generator * g) {
     g->literalstring_count = 0;
     {
         struct node * p = g->analyser->program;
-        until (p == 0) { generate(g, p); p = p->right; }
+        while (p) { generate(g, p); p = p->right; }
     }
     generate_create(g);
     generate_close(g);
-    output_str(g->options->output_c, g->declarations);
+    output_str(g->options->output_src, g->declarations);
     str_delete(g->declarations);
-    output_str(g->options->output_c, g->outbuf);
+    output_str(g->options->output_src, g->outbuf);
     str_clear(g->outbuf);
 
-    generate_start_comment(g);
+    write_start_comment(g, "/* ", " */");
     generate_header_file(g);
     output_str(g->options->output_h, g->outbuf);
     str_delete(g->outbuf);
 }
 
-extern struct generator * create_generator_c(struct analyser * a, struct options * o) {
+/* Generator functions common to multiple languages. */
+
+extern struct generator * create_generator(struct analyser * a, struct options * o) {
     NEW(generator, g);
     g->analyser = a;
     g->options = o;
     g->margin = 0;
     g->debug_count = 0;
+    g->copy_from_count = 0;
     g->line_count = 0;
+    g->line_labelled = 0;
+    g->failure_label = -1;
+    g->unreachable = false;
+#ifndef DISABLE_PYTHON
+    g->max_label = 0;
+#endif
     return g;
 }
 
-extern void close_generator_c(struct generator * g) {
-
+extern void close_generator(struct generator * g) {
     FREE(g);
 }
 
+/* Write routines for simple entities */
+
+extern void write_char(struct generator * g, int ch) {
+    str_append_ch(g->outbuf, ch); /* character */
+}
+
+extern void write_newline(struct generator * g) {
+    str_append_ch(g->outbuf, '\n'); /* newline */
+    g->line_count++;
+}
+
+extern void write_string(struct generator * g, const char * s) {
+    str_append_string(g->outbuf, s);
+}
+
+extern void write_int(struct generator * g, int i) {
+    str_append_int(g->outbuf, i);
+}
+
+extern void write_b(struct generator * g, symbol * b) {
+
+    str_append_b(g->outbuf, b);
+}
+
+extern void write_str(struct generator * g, struct str * str) {
+
+    str_append(g->outbuf, str);
+}
diff --git a/contrib/snowball/compiler/generator_java.c b/contrib/snowball/compiler/generator_java.c
deleted file mode 100644
index 07a4b8e2e..000000000
--- a/contrib/snowball/compiler/generator_java.c
+++ /dev/null
@@ -1,1452 +0,0 @@
-
-#include <stdlib.h> /* for exit */
-#include <string.h> /* for strlen */
-#include <stdio.h> /* for fprintf etc */
-#include "header.h"
-
-/* prototypes */
-
-static void generate(struct generator * g, struct node * p);
-static void w(struct generator * g, const char * s);
-static void writef(struct generator * g, const char * s, struct node * p);
-
-
-enum special_labels {
-    x_return = -1
-};
-
-static int new_label(struct generator * g) {
-
-    return g->next_label++;
-}
-
-static struct str * vars_newname(struct generator * g) {
-
-    struct str * output;
-    g->var_number ++;
-    output = str_new();
-    str_append_string(output, "v_");
-    str_append_int(output, g->var_number);
-    return output;
-}
-
-/* Output routines */
-static void output_str(FILE * outfile, struct str * str) {
-
-    char * s = b_to_s(str_data(str));
-    fprintf(outfile, "%s", s);
-    free(s);
-}
-
-/* Write routines for simple entities */
-
-static void write_char(struct generator * g, int ch) {
-
-    str_append_ch(g->outbuf, ch);
-}
-
-static void write_newline(struct generator * g) {
-
-    str_append_string(g->outbuf, "\n");
-}
-
-static void write_string(struct generator * g, const char * s) {
-
-    str_append_string(g->outbuf, s);
-}
-
-static void write_b(struct generator * g, symbol * b) {
-
-    str_append_b(g->outbuf, b);
-}
-
-static void write_str(struct generator * g, struct str * str) {
-
-    str_append(g->outbuf, str);
-}
-
-static void write_int(struct generator * g, int i) {
-
-    str_append_int(g->outbuf, i);
-}
-
-
-/* Write routines for items from the syntax tree */
-
-static void write_varname(struct generator * g, struct name * p) {
-
-    int ch = "SBIrxg"[p->type];
-    if (p->type != t_external)
-    {
-        write_char(g, ch);
-        write_char(g, '_');
-    }
-    str_append_b(g->outbuf, p->b);
-}
-
-static void write_varref(struct generator * g, struct name * p) {
-
-    /* In java, references look just the same */
-    write_varname(g, p);
-}
-
-static void write_hexdigit(struct generator * g, int n) {
-
-    write_char(g, n < 10 ? n + '0' : n - 10 + 'A');
-}
-
-static void write_hex(struct generator * g, int ch) {
-
-    write_string(g, "\\u");
-    {
-        int i;
-        for (i = 12; i >= 0; i -= 4) write_hexdigit(g, ch >> i & 0xf);
-    }
-}
-
-static void write_literal_string(struct generator * g, symbol * p) {
-
-    int i;
-    write_string(g, "\"");
-    for (i = 0; i < SIZE(p); i++) {
-        int ch = p[i];
-        if (32 <= ch && ch <= 127) {
-            if (ch == '\"' || ch == '\\') write_string(g, "\\");
-            write_char(g, ch);
-        } else {
-            write_hex(g, ch);
-        }
-    }
-    write_string(g, "\"");
-}
-
-static void write_margin(struct generator * g) {
-
-    int i;
-    for (i = 0; i < g->margin; i++) write_string(g, "    ");
-}
-
-/* Write a variable declaration. */
-static void write_declare(struct generator * g,
-                          char * declaration,
-                          struct node * p) {
-
-    struct str * temp = g->outbuf;
-    g->outbuf = g->declarations;
-    write_string(g, "            ");
-    writef(g, declaration, p);
-    write_string(g, ";");
-    write_newline(g);
-    g->outbuf = temp;
-}
-
-static void write_comment(struct generator * g, struct node * p) {
-
-    write_margin(g);
-    write_string(g, "// ");
-    write_string(g, name_of_token(p->type));
-    if (p->name != 0) {
-        write_string(g, " ");
-        str_append_b(g->outbuf, p->name->b);
-    }
-    write_string(g, ", line ");
-    write_int(g, p->line_number);
-    write_newline(g);
-}
-
-static void write_block_start(struct generator * g) {
-
-    w(g, "~M{~+~N");
-}
-
-static void write_block_end(struct generator * g)    /* block end */ {
-
-    w(g, "~-~M}~N");
-}
-
-static void write_savecursor(struct generator * g, struct node * p,
-                             struct str * savevar) {
-
-    g->B[0] = str_data(savevar);
-    g->S[1] = "";
-    if (p->mode != m_forward) g->S[1] = "limit - ";
-    write_declare(g, "int ~B0", p);
-    writef(g, "~M~B0 = ~S1cursor;~N" , p);
-}
-
-static void restore_string(struct node * p, struct str * out, struct str * savevar) {
-
-    str_clear(out);
-    str_append_string(out, "cursor = ");
-    if (p->mode != m_forward) str_append_string(out, "limit - ");
-    str_append(out, savevar);
-    str_append_string(out, ";");
-}
-
-static void write_restorecursor(struct generator * g, struct node * p,
-                                struct str * savevar) {
-
-    struct str * temp = str_new();
-    write_margin(g);
-    restore_string(p, temp, savevar);
-    write_str(g, temp);
-    write_newline(g);
-    str_delete(temp);
-}
-
-static void write_inc_cursor(struct generator * g, struct node * p) {
-
-    write_margin(g);
-    write_string(g, p->mode == m_forward ? "cursor++;" : "cursor--;");
-    write_newline(g);
-}
-
-static void wsetlab_begin(struct generator * g, int n) {
-
-    w(g, "~Mlab");
-    write_int(g, n);
-    w(g, ": do {~+~N");
-}
-
-static void wsetlab_end(struct generator * g) {
-
-    w(g, "~-~M} while (false);~N");
-}
-
-static void wgotol(struct generator * g, int n) {
-
-    write_margin(g);
-    write_string(g, "break lab");
-    write_int(g, n);
-    write_string(g, ";");
-    write_newline(g);
-}
-
-static void write_failure(struct generator * g) {
-
-    if (str_len(g->failure_str) != 0) {
-        write_margin(g);
-        write_str(g, g->failure_str);
-        write_newline(g);
-    }
-    write_margin(g);
-    switch (g->failure_label)
-    {
-        case x_return:
-            write_string(g, "return false;");
-            break;
-        default:
-            write_string(g, "break lab");
-            write_int(g, g->failure_label);
-            write_string(g, ";");
-            g->unreachable = true;
-    }
-    write_newline(g);
-}
-
-static void write_failure_if(struct generator * g, char * s, struct node * p) {
-
-    writef(g, "~Mif (", p);
-    writef(g, s, p);
-    writef(g, ")~N", p);
-    write_block_start(g);
-    write_failure(g);
-    write_block_end(g);
-    g->unreachable = false;
-}
-
-/* if at limit fail */
-static void write_check_limit(struct generator * g, struct node * p) {
-
-    if (p->mode == m_forward) {
-        write_failure_if(g, "cursor >= limit", p);
-    } else {
-        write_failure_if(g, "cursor <= limit_backward", p);
-    }
-}
-
-/* Formatted write. */
-static void writef(struct generator * g, const char * input, struct node * p) {
-
-    int i = 0;
-    int l = strlen(input);
-
-    while (i < l) {
-        int ch = input[i++];
-        if (ch == '~') {
-            switch(input[i++]) {
-                default: write_char(g, input[i - 1]); continue;
-                case 'C': write_comment(g, p); continue;
-                case 'f': write_block_start(g);
-                          write_failure(g);
-			  g->unreachable = false;
-                          write_block_end(g);
-                          continue;
-                case 'M': write_margin(g); continue;
-                case 'N': write_newline(g); continue;
-                case '{': write_block_start(g); continue;
-                case '}': write_block_end(g); continue;
-                case 'S': write_string(g, g->S[input[i++] - '0']); continue;
-                case 'B': write_b(g, g->B[input[i++] - '0']); continue;
-                case 'I': write_int(g, g->I[input[i++] - '0']); continue;
-                case 'V': write_varref(g, g->V[input[i++] - '0']); continue;
-                case 'W': write_varname(g, g->V[input[i++] - '0']); continue;
-                case 'L': write_literal_string(g, g->L[input[i++] - '0']); continue;
-                case '+': g->margin++; continue;
-                case '-': g->margin--; continue;
-                case 'n': write_string(g, g->options->name); continue;
-            }
-        } else {
-            write_char(g, ch);
-        }
-    }
-}
-
-static void w(struct generator * g, const char * s) {
-    writef(g, s, 0);
-}
-
-static void generate_AE(struct generator * g, struct node * p) {
-    char * s;
-    switch (p->type) {
-        case c_name:
-            write_varref(g, p->name); break;
-        case c_number:
-            write_int(g, p->number); break;
-        case c_maxint:
-            write_string(g, "MAXINT"); break;
-        case c_minint:
-            write_string(g, "MININT"); break;
-        case c_neg:
-            write_string(g, "-"); generate_AE(g, p->right); break;
-        case c_multiply:
-            s = " * "; goto label0;
-        case c_plus:
-            s = " + "; goto label0;
-        case c_minus:
-            s = " - "; goto label0;
-        case c_divide:
-            s = " / ";
-        label0:
-            write_string(g, "("); generate_AE(g, p->left);
-            write_string(g, s); generate_AE(g, p->right); write_string(g, ")"); break;
-        case c_sizeof:
-            g->V[0] = p->name;
-            w(g, "(~V0.length())"); break;
-        case c_cursor:
-            w(g, "cursor"); break;
-        case c_limit:
-            w(g, p->mode == m_forward ? "limit" : "limit_backward"); break;
-        case c_size:
-            w(g, "(current.length())"); break;
-    }
-}
-
-/* K_needed() tests to see if we really need to keep c. Not true when the
-   the command does not touch the cursor. This and repeat_score() could be
-   elaborated almost indefinitely.
-*/
-
-static int K_needed(struct generator * g, struct node * p) {
-
-    while (p != 0) {
-        switch (p->type) {
-            case c_dollar:
-            case c_leftslice:
-            case c_rightslice:
-            case c_mathassign:
-            case c_plusassign:
-            case c_minusassign:
-            case c_multiplyassign:
-            case c_divideassign:
-            case c_eq:
-            case c_ne:
-            case c_gr:
-            case c_ge:
-            case c_ls:
-            case c_le:
-            case c_sliceto:
-            case c_booltest:
-            case c_true:
-            case c_false:
-            case c_debug:
-                break;
-
-            case c_call:
-                if (K_needed(g, p->name->definition)) return true;
-                break;
-
-            case c_bra:
-                if (K_needed(g, p->left)) return true;
-                break;
-
-            default: return true;
-        }
-        p = p->right;
-    }
-    return false;
-}
-
-static int repeat_score(struct generator * g, struct node * p) {
-
-    int score = 0;
-    while (p != 0) {
-        switch (p->type) {
-            case c_dollar:
-            case c_leftslice:
-            case c_rightslice:
-            case c_mathassign:
-            case c_plusassign:
-            case c_minusassign:
-            case c_multiplyassign:
-            case c_divideassign:
-            case c_eq:
-            case c_ne:
-            case c_gr:
-            case c_ge:
-            case c_ls:
-            case c_le:
-            case c_sliceto:   /* case c_not: must not be included here! */
-            case c_debug:
-                break;
-
-            case c_call:
-                score += repeat_score(g, p->name->definition);
-                break;
-
-            case c_bra:
-                score += repeat_score(g, p->left);
-                break;
-
-            case c_name:
-            case c_literalstring:
-            case c_next:
-            case c_grouping:
-            case c_non:
-            case c_hop:
-                score = score + 1;
-                break;
-
-            default:
-                score = 2;
-                break;
-        }
-        p = p->right;
-    }
-    return score;
-}
-
-/* tests if an expression requires cursor reinstatement in a repeat */
-
-static int repeat_restore(struct generator * g, struct node * p) {
-
-    return repeat_score(g, p) >= 2;
-}
-
-static void generate_bra(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    p = p->left;
-    while (p != 0) {
-        generate(g, p);
-        p = p->right;
-    }
-}
-
-static void generate_and(struct generator * g, struct node * p) {
-
-    struct str * savevar = vars_newname(g);
-    int keep_c = K_needed(g, p->left);
-
-    write_comment(g, p);
-
-    if (keep_c) write_savecursor(g, p, savevar);
-
-    p = p->left;
-    while (p != 0) {
-        generate(g, p);
-        if (g->unreachable) break;
-        if (keep_c && p->right != 0) write_restorecursor(g, p, savevar);
-        p = p->right;
-    }
-    str_delete(savevar);
-}
-
-static void generate_or(struct generator * g, struct node * p) {
-
-    struct str * savevar = vars_newname(g);
-    int keep_c = K_needed(g, p->left);
-
-    int a0 = g->failure_label;
-    struct str * a1 = str_copy(g->failure_str);
-
-    int out_lab = new_label(g);
-    write_comment(g, p);
-    wsetlab_begin(g, out_lab);
-
-    if (keep_c) write_savecursor(g, p, savevar);
-
-    p = p->left;
-    str_clear(g->failure_str);
-
-    if (p == 0) {
-        /* p should never be 0 after an or: there should be at least two
-         * sub nodes. */
-        fprintf(stderr, "Error: \"or\" node without children nodes.");
-        exit (1);
-    }
-    while (p->right != 0) {
-        g->failure_label = new_label(g);
-        wsetlab_begin(g, g->failure_label);
-        generate(g, p);
-        if (!g->unreachable) wgotol(g, out_lab);
-        wsetlab_end(g);
-        g->unreachable = false;
-        if (keep_c) write_restorecursor(g, p, savevar);
-        p = p->right;
-    }
-
-    g->failure_label = a0;
-    str_delete(g->failure_str);
-    g->failure_str = a1;
-
-    generate(g, p);
-    wsetlab_end(g);
-    str_delete(savevar);
-}
-
-static void generate_backwards(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    writef(g,"~Mlimit_backward = cursor; cursor = limit;~N", p);
-    generate(g, p->left);
-    w(g, "~Mcursor = limit_backward;");
-}
-
-
-static void generate_not(struct generator * g, struct node * p) {
-
-    struct str * savevar = vars_newname(g);
-    int keep_c = K_needed(g, p->left);
-
-    int a0 = g->failure_label;
-    struct str * a1 = str_copy(g->failure_str);
-
-    write_comment(g, p);
-    if (keep_c) {
-        write_block_start(g);
-        write_savecursor(g, p, savevar);
-    }
-
-    g->failure_label = new_label(g);
-    str_clear(g->failure_str);
-
-    wsetlab_begin(g, g->failure_label);
-
-    generate(g, p->left);
-
-    g->failure_label = a0;
-    str_delete(g->failure_str);
-    g->failure_str = a1;
-
-    if (!g->unreachable) write_failure(g);
-
-    wsetlab_end(g);
-    g->unreachable = false;
-
-    if (keep_c) write_restorecursor(g, p, savevar);
-    if (keep_c) write_block_end(g);
-    str_delete(savevar);
-}
-
-
-static void generate_try(struct generator * g, struct node * p) {
-
-    struct str * savevar = vars_newname(g);
-    int keep_c = K_needed(g, p->left);
-
-    write_comment(g, p);
-    if (keep_c) write_savecursor(g, p, savevar);
-
-    g->failure_label = new_label(g);
-    if (keep_c) restore_string(p, g->failure_str, savevar);
-
-    wsetlab_begin(g, g->failure_label);
-    generate(g, p->left);
-    wsetlab_end(g);
-    g->unreachable = false;
-
-    str_delete(savevar);
-}
-
-static void generate_set(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->V[0] = p->name;
-    writef(g, "~M~V0 = true;~N", p);
-}
-
-static void generate_unset(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->V[0] = p->name;
-    writef(g, "~M~V0 = false;~N", p);
-}
-
-static void generate_fail(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    generate(g, p->left);
-    if (!g->unreachable) write_failure(g);
-}
-
-/* generate_test() also implements 'reverse' */
-
-static void generate_test(struct generator * g, struct node * p) {
-
-    struct str * savevar = vars_newname(g);
-    int keep_c = K_needed(g, p->left);
-
-    write_comment(g, p);
-
-    if (keep_c) {
-        write_savecursor(g, p, savevar);
-    }
-
-    generate(g, p->left);
-
-    if (!g->unreachable) {
-        if (keep_c) {
-            write_restorecursor(g, p, savevar);
-        }
-    }
-    str_delete(savevar);
-}
-
-static void generate_do(struct generator * g, struct node * p) {
-
-    struct str * savevar = vars_newname(g);
-    int keep_c = K_needed(g, p->left);
-    write_comment(g, p);
-    if (keep_c) write_savecursor(g, p, savevar);
-
-    g->failure_label = new_label(g);
-    str_clear(g->failure_str);
-
-    wsetlab_begin(g, g->failure_label);
-    generate(g, p->left);
-    wsetlab_end(g);
-    g->unreachable = false;
-
-    if (keep_c) write_restorecursor(g, p, savevar);
-    str_delete(savevar);
-}
-
-static void generate_GO(struct generator * g, struct node * p, int style) {
-
-    int end_unreachable = false;
-    struct str * savevar = vars_newname(g);
-    int keep_c = style == 1 || repeat_restore(g, p->left);
-
-    int a0 = g->failure_label;
-    struct str * a1 = str_copy(g->failure_str);
-
-    int golab = new_label(g);
-    g->I[0] = golab;
-    write_comment(g, p);
-    w(g, "~Mgolab~I0: while(true)~N");
-    w(g, "~{");
-
-    if (keep_c) write_savecursor(g, p, savevar);
-
-    g->failure_label = new_label(g);
-    wsetlab_begin(g, g->failure_label);
-    generate(g, p->left);
-
-    if (g->unreachable) {
-        /* Cannot break out of this loop: therefore the code after the
-         * end of the loop is unreachable.*/
-        end_unreachable = true;
-    } else {
-        /* include for goto; omit for gopast */
-        if (style == 1) write_restorecursor(g, p, savevar);
-        g->I[0] = golab;
-        w(g, "~Mbreak golab~I0;~N");
-    }
-    g->unreachable = false;
-    wsetlab_end(g);
-    if (keep_c) write_restorecursor(g, p, savevar);
-
-    g->failure_label = a0;
-    str_delete(g->failure_str);
-    g->failure_str = a1;
-
-    write_check_limit(g, p);
-    write_inc_cursor(g, p);
-    write_block_end(g);
-    str_delete(savevar);
-    g->unreachable = end_unreachable;
-}
-
-static void generate_loop(struct generator * g, struct node * p) {
-
-    struct str * loopvar = vars_newname(g);
-    write_comment(g, p);
-    g->B[0] = str_data(loopvar);
-    write_declare(g, "int ~B0", p);
-    w(g, "~Mfor (~B0 = ");
-    generate_AE(g, p->AE);
-    g->B[0] = str_data(loopvar);
-    writef(g, "; ~B0 > 0; ~B0--)~N", p);
-    writef(g, "~{", p);
-
-    generate(g, p->left);
-
-    w(g, "~}");
-    str_delete(loopvar);
-    g->unreachable = false;
-}
-
-static void generate_repeat(struct generator * g, struct node * p, struct str * loopvar) {
-
-    struct str * savevar = vars_newname(g);
-    int keep_c = repeat_restore(g, p->left);
-    int replab = new_label(g);
-    g->I[0] = replab;
-    write_comment(g, p);
-    writef(g, "~Mreplab~I0: while(true)~N~{", p);
-
-    if (keep_c) write_savecursor(g, p, savevar);
-
-    g->failure_label = new_label(g);
-    str_clear(g->failure_str);
-    wsetlab_begin(g, g->failure_label);
-    generate(g, p->left);
-
-    if (!g->unreachable) {
-        if (loopvar != 0) {
-            g->B[0] = str_data(loopvar);
-            w(g, "~M~B0--;~N");
-        }
-
-        g->I[0] = replab;
-        w(g, "~Mcontinue replab~I0;~N");
-    }
-
-    wsetlab_end(g);
-    g->unreachable = false;
-
-    if (keep_c) write_restorecursor(g, p, savevar);
-
-    g->I[0] = replab;
-    w(g, "~Mbreak replab~I0;~N~}");
-    str_delete(savevar);
-}
-
-static void generate_atleast(struct generator * g, struct node * p) {
-
-    struct str * loopvar = vars_newname(g);
-    write_comment(g, p);
-    w(g, "~{");
-    g->B[0] = str_data(loopvar);
-    w(g, "~Mint ~B0 = ");
-    generate_AE(g, p->AE);
-    w(g, ";~N");
-    {
-        int a0 = g->failure_label;
-        struct str * a1 = str_copy(g->failure_str);
-
-        generate_repeat(g, p, loopvar);
-
-        g->failure_label = a0;
-        str_delete(g->failure_str);
-        g->failure_str = a1;
-    }
-    g->B[0] = str_data(loopvar);
-    write_failure_if(g, "~B0 > 0", p);
-    w(g, "~}");
-    str_delete(loopvar);
-}
-
-static void generate_setmark(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->V[0] = p->name;
-    writef(g, "~M~V0 = cursor;~N", p);
-}
-
-static void generate_tomark(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->S[0] = p->mode == m_forward ? ">" : "<";
-
-    w(g, "~Mif (cursor ~S0 "); generate_AE(g, p->AE); w(g, ")~N");
-    write_block_start(g);
-    write_failure(g);
-    write_block_end(g);
-    g->unreachable = false;
-    w(g, "~Mcursor = "); generate_AE(g, p->AE); writef(g, ";~N", p);
-}
-
-static void generate_atmark(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    w(g, "~Mif (cursor != "); generate_AE(g, p->AE); writef(g, ")~N", p);
-    write_block_start(g);
-    write_failure(g);
-    write_block_end(g);
-    g->unreachable = false;
-}
-
-
-static void generate_hop(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->S[0] = p->mode == m_forward ? "+" : "-";
-
-    w(g, "~{~Mint c = cursor ~S0 ");
-    generate_AE(g, p->AE);
-    w(g, ";~N");
-
-    g->S[0] = p->mode == m_forward ? "0" : "limit_backward";
-
-    write_failure_if(g, "~S0 > c || c > limit", p);
-    writef(g, "~Mcursor = c;~N", p);
-    writef(g, "~}", p);
-}
-
-static void generate_delete(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    writef(g, "~Mslice_del();~N", p);
-}
-
-
-static void generate_next(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    write_check_limit(g, p);
-    write_inc_cursor(g, p);
-}
-
-static void generate_tolimit(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->S[0] = p->mode == m_forward ? "limit" : "limit_backward";
-    writef(g, "~Mcursor = ~S0;~N", p);
-}
-
-static void generate_atlimit(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->S[0] = p->mode == m_forward ? "limit" : "limit_backward";
-    g->S[1] = p->mode == m_forward ? "<" : ">";
-    write_failure_if(g, "cursor ~S1 ~S0", p);
-}
-
-static void generate_leftslice(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->S[0] = p->mode == m_forward ? "bra" : "ket";
-    writef(g, "~M~S0 = cursor;~N", p);
-}
-
-static void generate_rightslice(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->S[0] = p->mode == m_forward ? "ket" : "bra";
-    writef(g, "~M~S0 = cursor;~N", p);
-}
-
-static void generate_assignto(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->V[0] = p->name;
-    writef(g, "~M~V0 = assign_to(~V0);~N", p);
-}
-
-static void generate_sliceto(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->V[0] = p->name;
-    writef(g, "~M~V0 = slice_to(~V0);~N", p);
-}
-
-static void generate_address(struct generator * g, struct node * p) {
-
-    symbol * b = p->literalstring;
-    if (b != 0) {
-        write_literal_string(g, b);
-    } else {
-        write_varref(g, p->name);
-    }
-}
-
-static void generate_insert(struct generator * g, struct node * p, int style) {
-
-    int keep_c = style == c_attach;
-    write_comment(g, p);
-    if (p->mode == m_backward) keep_c = !keep_c;
-    if (keep_c) w(g, "~{~Mint c = cursor;~N");
-    writef(g, "~Minsert(cursor, cursor, ", p);
-    generate_address(g, p);
-    writef(g, ");~N", p);
-    if (keep_c) w(g, "~Mcursor = c;~N~}");
-}
-
-static void generate_assignfrom(struct generator * g, struct node * p) {
-
-    int keep_c = p->mode == m_forward; /* like 'attach' */
-
-    write_comment(g, p);
-    if (keep_c) writef(g, "~{~Mint c = cursor;~N", p);
-    if (p->mode == m_forward) {
-        writef(g, "~Minsert(cursor, limit, ", p);
-    } else {
-        writef(g, "~Minsert(limit_backward, cursor, ", p);
-    }
-    generate_address(g, p);
-    writef(g, ");~N", p);
-    if (keep_c) w(g, "~Mcursor = c;~N~}");
-}
-
-
-static void generate_slicefrom(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    w(g, "~Mslice_from(");
-    generate_address(g, p);
-    writef(g, ");~N", p);
-}
-
-static void generate_setlimit(struct generator * g, struct node * p) {
-
-    struct str * savevar = vars_newname(g);
-    struct str * varname = vars_newname(g);
-    write_comment(g, p);
-    write_savecursor(g, p, savevar);
-    generate(g, p->left);
-
-    if (!g->unreachable) {
-        g->B[0] = str_data(varname);
-        write_declare(g, "int ~B0", p);
-        if (p->mode == m_forward) {
-            w(g, "~M~B0 = limit - cursor;~N");
-            w(g, "~Mlimit = cursor;~N");
-        } else {
-            w(g, "~M~B0 = limit_backward;~N");
-            w(g, "~Mlimit_backward = cursor;~N");
-        }
-        write_restorecursor(g, p, savevar);
-
-        if (p->mode == m_forward) {
-            str_assign(g->failure_str, "limit += ");
-            str_append(g->failure_str, varname);
-            str_append_ch(g->failure_str, ';');
-        } else {
-            str_assign(g->failure_str, "limit_backward = ");
-            str_append(g->failure_str, varname);
-            str_append_ch(g->failure_str, ';');
-        }
-        generate(g, p->aux);
-
-        if (!g->unreachable) {
-            write_margin(g);
-            write_str(g, g->failure_str);
-            write_newline(g);
-        }
-    }
-    str_delete(varname);
-    str_delete(savevar);
-}
-
-/* dollar sets snowball up to operate on a string variable as if it were the
- * current string */
-static void generate_dollar(struct generator * g, struct node * p) {
-
-    struct str * savevar = vars_newname(g);
-    write_comment(g, p);
-    g->V[0] = p->name;
-
-    str_assign(g->failure_str, "copy_from(");
-    str_append(g->failure_str, savevar);
-    str_append_string(g->failure_str, ");");
-    g->B[0] = str_data(savevar);
-    writef(g, "~{~M~n ~B0 = this;~N"
-             "~Mcurrent = new StringBuffer(~V0.toString());~N"
-             "~Mcursor = 0;~N"
-             "~Mlimit = (current.length());~N", p);
-    generate(g, p->left);
-    if (!g->unreachable) {
-        write_margin(g);
-        write_str(g, g->failure_str);
-        write_newline(g);
-    }
-    w(g, "~}");
-    str_delete(savevar);
-}
-
-static void generate_integer_assign(struct generator * g, struct node * p, char * s) {
-
-    g->V[0] = p->name;
-    g->S[0] = s;
-    w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); w(g, ";~N");
-}
-
-static void generate_integer_test(struct generator * g, struct node * p, char * s) {
-
-    g->V[0] = p->name;
-    g->S[0] = s;
-    w(g, "~Mif (!(~V0 ~S0 "); generate_AE(g, p->AE); w(g, "))~N");
-    write_block_start(g);
-    write_failure(g);
-    write_block_end(g);
-    g->unreachable = false;
-}
-
-static void generate_call(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->V[0] = p->name;
-    write_failure_if(g, "!~V0()", p);
-}
-
-static void generate_grouping(struct generator * g, struct node * p, int complement) {
-
-    struct grouping * q = p->name->grouping;
-    g->S[0] = p->mode == m_forward ? "" : "_b";
-    g->S[1] = complement ? "out" : "in";
-    g->V[0] = p->name;
-    g->I[0] = q->smallest_ch;
-    g->I[1] = q->largest_ch;
-    if (q->no_gaps)
-        write_failure_if(g, "!(~S1_range~S0(~I0, ~I1))", p);
-    else
-        write_failure_if(g, "!(~S1_grouping~S0(~V0, ~I0, ~I1))", p);
-}
-
-static void generate_namedstring(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->S[0] = p->mode == m_forward ? "" : "_b";
-    g->V[0] = p->name;
-    write_failure_if(g, "!(eq_v~S0(~V0))", p);
-}
-
-static void generate_literalstring(struct generator * g, struct node * p) {
-
-    symbol * b = p->literalstring;
-    write_comment(g, p);
-    g->S[0] = p->mode == m_forward ? "" : "_b";
-    g->I[0] = SIZE(b);
-    g->L[0] = b;
-    write_failure_if(g, "!(eq_s~S0(~I0, ~L0))", p);
-}
-
-static void generate_define(struct generator * g, struct node * p) {
-
-    struct name * q = p->name;
-
-    struct str * saved_output = g->outbuf;
-    struct str * saved_declarations = g->declarations;
-
-    g->S[0] = q->type == t_routine ? "private" : "public";
-    g->V[0] = q;
-    w(g, "~+~+~N~M~S0 boolean ~V0() {~+~N");
-
-    g->outbuf = str_new();
-    g->declarations = str_new();
-
-    g->next_label = 0;
-    g->var_number = 0;
-
-    if (p->amongvar_needed) write_declare(g, "int among_var", p);
-    str_clear(g->failure_str);
-    g->failure_label = x_return;
-    g->unreachable = false;
-    generate(g, p->left);
-    if (!g->unreachable) w(g, "~Mreturn true;~N");
-    w(g, "~}~-~-");
-
-    str_append(saved_output, g->declarations);
-    str_append(saved_output, g->outbuf);
-    str_delete(g->declarations);
-    str_delete(g->outbuf);
-    g->declarations = saved_declarations;
-    g->outbuf = saved_output;
-}
-
-static void generate_substring(struct generator * g, struct node * p) {
-
-    struct among * x = p->among;
-
-    write_comment(g, p);
-
-    g->S[0] = p->mode == m_forward ? "" : "_b";
-    g->I[0] = x->number;
-    g->I[1] = x->literalstring_count;
-
-    if (x->command_count == 0 && x->starter == 0) {
-        write_failure_if(g, "find_among~S0(a_~I0, ~I1) == 0", p);
-    } else {
-        writef(g, "~Mamong_var = find_among~S0(a_~I0, ~I1);~N", p);
-        write_failure_if(g, "among_var == 0", p);
-    }
-}
-
-static void generate_among(struct generator * g, struct node * p) {
-
-    struct among * x = p->among;
-    int case_number = 1;
-
-    if (x->substring == 0) generate_substring(g, p);
-    if (x->command_count == 0 && x->starter == 0) return;
-
-    if (x->starter != 0) generate(g, x->starter);
-
-    p = p->left;
-    if (p != 0 && p->type != c_literalstring) p = p->right;
-    w(g, "~Mswitch(among_var) {~N~+");
-    w(g, "~Mcase 0:~N~+");
-    write_failure(g);
-    g->unreachable = false;
-    w(g, "~-");
-
-    while (p != 0) {
-        if (p->type == c_bra && p->left != 0) {
-            g->I[0] = case_number++;
-            w(g, "~Mcase ~I0:~N~+");
-            generate(g, p);
-            if (!g->unreachable) w(g, "~Mbreak;~N");
-            w(g, "~-");
-            g->unreachable = false;
-        }
-        p = p->right;
-    }
-    write_block_end(g);
-}
-
-static void generate_booltest(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->V[0] = p->name;
-    write_failure_if(g, "!(~V0)", p);
-}
-
-static void generate_false(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    write_failure(g);
-}
-
-static void generate_debug(struct generator * g, struct node * p) {
-
-    write_comment(g, p);
-    g->I[0] = g->debug_count++;
-    g->I[1] = p->line_number;
-    writef(g, "~Mdebug(~I0, ~I1);~N", p);
-}
-
-static void generate(struct generator * g, struct node * p) {
-
-    int a0;
-    struct str * a1;
-
-    if (g->unreachable) return;
-
-    a0 = g->failure_label;
-    a1 = str_copy(g->failure_str);
-
-    switch (p->type)
-    {
-        case c_define:        generate_define(g, p); break;
-        case c_bra:           generate_bra(g, p); break;
-        case c_and:           generate_and(g, p); break;
-        case c_or:            generate_or(g, p); break;
-        case c_backwards:     generate_backwards(g, p); break;
-        case c_not:           generate_not(g, p); break;
-        case c_set:           generate_set(g, p); break;
-        case c_unset:         generate_unset(g, p); break;
-        case c_try:           generate_try(g, p); break;
-        case c_fail:          generate_fail(g, p); break;
-        case c_reverse:
-        case c_test:          generate_test(g, p); break;
-        case c_do:            generate_do(g, p); break;
-        case c_goto:          generate_GO(g, p, 1); break;
-        case c_gopast:        generate_GO(g, p, 0); break;
-        case c_repeat:        generate_repeat(g, p, 0); break;
-        case c_loop:          generate_loop(g, p); break;
-        case c_atleast:       generate_atleast(g, p); break;
-        case c_setmark:       generate_setmark(g, p); break;
-        case c_tomark:        generate_tomark(g, p); break;
-        case c_atmark:        generate_atmark(g, p); break;
-        case c_hop:           generate_hop(g, p); break;
-        case c_delete:        generate_delete(g, p); break;
-        case c_next:          generate_next(g, p); break;
-        case c_tolimit:       generate_tolimit(g, p); break;
-        case c_atlimit:       generate_atlimit(g, p); break;
-        case c_leftslice:     generate_leftslice(g, p); break;
-        case c_rightslice:    generate_rightslice(g, p); break;
-        case c_assignto:      generate_assignto(g, p); break;
-        case c_sliceto:       generate_sliceto(g, p); break;
-        case c_assign:        generate_assignfrom(g, p); break;
-        case c_insert:
-        case c_attach:        generate_insert(g, p, p->type); break;
-        case c_slicefrom:     generate_slicefrom(g, p); break;
-        case c_setlimit:      generate_setlimit(g, p); break;
-        case c_dollar:        generate_dollar(g, p); break;
-        case c_mathassign:    generate_integer_assign(g, p, "="); break;
-        case c_plusassign:    generate_integer_assign(g, p, "+="); break;
-        case c_minusassign:   generate_integer_assign(g, p, "-="); break;
-        case c_multiplyassign:generate_integer_assign(g, p, "*="); break;
-        case c_divideassign:  generate_integer_assign(g, p, "/="); break;
-        case c_eq:            generate_integer_test(g, p, "=="); break;
-        case c_ne:            generate_integer_test(g, p, "!="); break;
-        case c_gr:            generate_integer_test(g, p, ">"); break;
-        case c_ge:            generate_integer_test(g, p, ">="); break;
-        case c_ls:            generate_integer_test(g, p, "<"); break;
-        case c_le:            generate_integer_test(g, p, "<="); break;
-        case c_call:          generate_call(g, p); break;
-        case c_grouping:      generate_grouping(g, p, false); break;
-        case c_non:           generate_grouping(g, p, true); break;
-        case c_name:          generate_namedstring(g, p); break;
-        case c_literalstring: generate_literalstring(g, p); break;
-        case c_among:         generate_among(g, p); break;
-        case c_substring:     generate_substring(g, p); break;
-        case c_booltest:      generate_booltest(g, p); break;
-        case c_false:         generate_false(g, p); break;
-        case c_true:          break;
-        case c_debug:         generate_debug(g, p); break;
-        default: fprintf(stderr, "%d encountered\n", p->type);
-                 exit(1);
-    }
-
-    g->failure_label = a0;
-    str_delete(g->failure_str);
-    g->failure_str = a1;
-}
-
-static void generate_start_comment(struct generator * g) {
-
-    w(g, "// This file was generated automatically by the Snowball to Java compiler~N");
-    w(g, "~N");
-}
-
-static void generate_class_begin(struct generator * g) {
-
-    w(g, "package " );
-    w(g, g->options->package);
-    w(g, ";~N~N" );
-
-    w(g, "import ");
-    w(g, g->options->among_class );
-    w(g, ";~N"
-         "~N"
-         " /**~N"
-         "  * This class was automatically generated by a Snowball to Java compiler ~N"
-         "  * It implements the stemming algorithm defined by a snowball script.~N"
-         "  */~N"
-         "~N"
-         "public class ~n extends ");
-
-     w(g, g->options->parent_class_name);
-     w(g, " {~N"
-          "~N"
-	  "private static final long serialVersionUID = 1L;~N"
-	  "~N"
-	  "~+~+~Mprivate final static ~n methodObject = new ~n ();~N"
-         "~N");
-}
-
-static void generate_class_end(struct generator * g) {
-
-    w(g, "~N}");
-    w(g, "~N~N");
-}
-
-static void generate_equals(struct generator * g) {
-
-    w(g, "~N"
-         "~Mpublic boolean equals( Object o ) {~N"
-         "~+~Mreturn o instanceof ");
-    w(g, g->options->name);
-	 w(g, ";~N~-~M}~N"
-	      "~N"
-	      "~Mpublic int hashCode() {~N"
-	      "~+~Mreturn ");
-    w(g, g->options->name);
-	 w(g, ".class.getName().hashCode();~N"
-	      "~-~M}~N");
-    w(g, "~N~N");
-}
-
-static void generate_among_table(struct generator * g, struct among * x) {
-
-    struct amongvec * v = x->b;
-
-    g->I[0] = x->number;
-    g->I[1] = x->literalstring_count;
-
-    w(g, "~+~+~Mprivate final static Among a_~I0[] = {~N~+");
-    {
-        int i;
-        for (i = 0; i < x->literalstring_count; i++) {
-            g->I[0] = i;
-            g->I[1] = v->i;
-            g->I[2] = v->result;
-            g->L[0] = v->b;
-            g->S[0] = i < x->literalstring_count - 1 ? "," : "";
-
-            w(g, "~Mnew Among ( ~L0, ~I1, ~I2, \"");
-            if (v->function != 0) {
-                write_varname(g, v->function);
-            }
-            w(g, "\", methodObject )~S0~N");
-            v++;
-        }
-    }
-    w(g, "~-~M};~-~-~N~N");
-}
-
-static void generate_amongs(struct generator * g) {
-
-    struct among * x = g->analyser->amongs;
-    while (x != 0) {
-        generate_among_table(g, x);
-        x = x->next;
-    }
-}
-
-static void set_bit(symbol * b, int i) { b[i/8] |= 1 << i%8; }
-
-static int bit_is_set(symbol * b, int i) { return b[i/8] & 1 << i%8; }
-
-static void generate_grouping_table(struct generator * g, struct grouping * q) {
-
-    int range = q->largest_ch - q->smallest_ch + 1;
-    int size = (range + 7)/ 8;  /* assume 8 bits per symbol */
-    symbol * b = q->b;
-    symbol * map = create_b(size);
-    int i;
-    for (i = 0; i < size; i++) map[i] = 0;
-
-    /* Using unicode would require revision here */
-
-    for (i = 0; i < SIZE(b); i++) set_bit(map, b[i] - q->smallest_ch);
-
-    q->no_gaps = true;
-    for (i = 0; i < range; i++) unless (bit_is_set(map, i)) q->no_gaps = false;
-
-    unless (q->no_gaps) {
-        g->V[0] = q->name;
-
-        w(g, "~+~+~Mprivate static final char ~V0[] = {");
-        for (i = 0; i < size; i++) {
-             write_int(g, map[i]);
-             if (i < size - 1) w(g, ", ");
-        }
-        w(g, " };~N~-~-~N");
-    }
-    lose_b(map);
-}
-
-static void generate_groupings(struct generator * g) {
-    struct grouping * q = g->analyser->groupings;
-    until (q == 0) {
-        generate_grouping_table(g, q);
-        q = q->next;
-    }
-}
-
-static void generate_members(struct generator * g) {
-
-    struct name * q = g->analyser->names;
-    until (q == 0) {
-        g->V[0] = q;
-        switch (q->type) {
-            case t_string:
-                w(g, "        private ");
-                w(g, g->options->string_class );
-                w(g, " ~W0 = new ");
-                w(g, g->options->string_class);
-                w(g, "();~N");
-                break;
-            case t_integer:
-                w(g, "        private int ~W0;~N");
-                break;
-            case t_boolean:
-                w(g, "        private boolean ~W0;~N");
-                break;
-        }
-        q = q->next;
-    }
-    w(g, "~N");
-}
-
-static void generate_copyfrom(struct generator * g) {
-
-    struct name * q;
-    w(g, "~+~+~Mprivate void copy_from(~n other) {~+~N");
-    for (q = g->analyser->names; q != 0; q = q->next) {
-        g->V[0] = q;
-        switch (q->type) {
-            case t_string:
-            case t_integer:
-            case t_boolean:
-                w(g, "~M~W0 = other.~W0;~N");
-                break;
-        }
-    }
-    w(g, "~Msuper.copy_from(other);~N");
-    w(g, "~-~M}~-~-~N");
-}
-
-static void generate_methods(struct generator * g) {
-
-    struct node * p = g->analyser->program;
-    while (p != 0) {
-        generate(g, p);
-        g->unreachable = false;
-        p = p->right;
-    }
-}
-
-extern void generate_program_java(struct generator * g) {
-
-    g->outbuf = str_new();
-    g->failure_str = str_new();
-
-    generate_start_comment(g);
-    generate_class_begin(g);
-
-    generate_amongs(g);
-    generate_groupings(g);
-
-    generate_members(g);
-    generate_copyfrom(g);
-    generate_methods(g);
-    generate_equals(g);
-
-    generate_class_end(g);
-
-    output_str(g->options->output_java, g->outbuf);
-    str_delete(g->failure_str);
-    str_delete(g->outbuf);
-}
-
-extern struct generator * create_generator_java(struct analyser * a, struct options * o) {
-
-    NEW(generator, g);
-    g->analyser = a;
-    g->options = o;
-    g->margin = 0;
-    g->debug_count = 0;
-    g->unreachable = false;
-    return g;
-}
-
-extern void close_generator_java(struct generator * g) {
-
-    FREE(g);
-}
-
diff --git a/contrib/snowball/compiler/header.h b/contrib/snowball/compiler/header.h
index 9baf1d917..dadbee36c 100644
--- a/contrib/snowball/compiler/header.h
+++ b/contrib/snowball/compiler/header.h
@@ -1,49 +1,56 @@
+#include <stdio.h>
+
+#define SNOWBALL_VERSION "2.0.0"
 
 typedef unsigned char byte;
 typedef unsigned short symbol;
 
 #define true 1
 #define false 0
-#define repeat while(true)
-#define unless(C) if(!(C))
-#define until(C) while(!(C))
 
 #define MALLOC check_malloc
 #define FREE check_free
 
 #define NEW(type, p) struct type * p = (struct type *) MALLOC(sizeof(struct type))
-#define NEWVEC(type, p, n) struct type * p = (struct type *) MALLOC(sizeof(struct type) * n)
+#define NEWVEC(type, p, n) struct type * p = (struct type *) MALLOC(sizeof(struct type) * (n))
 
-#define STARTSIZE   10
 #define SIZE(p)     ((int *)(p))[-1]
 #define CAPACITY(p) ((int *)(p))[-2]
 
 extern symbol * create_b(int n);
-extern void report_b(FILE * out, symbol * p);
+extern void report_b(FILE * out, const symbol * p);
 extern void lose_b(symbol * p);
 extern symbol * increase_capacity(symbol * p, int n);
-extern symbol * move_to_b(symbol * p, int n, symbol * q);
-extern symbol * add_to_b(symbol * p, int n, symbol * q);
-extern symbol * copy_b(symbol * p);
-extern char * b_to_s(symbol * p);
+extern symbol * move_to_b(symbol * p, int n, const symbol * q);
+extern symbol * add_to_b(symbol * p, int n, const symbol * q);
+extern symbol * copy_b(const symbol * p);
+extern char * b_to_s(const symbol * p);
 extern symbol * add_s_to_b(symbol * p, const char * s);
 
+#define MOVE_TO_B(B, LIT) \
+    move_to_b(B, sizeof(LIT) / sizeof(LIT[0]), LIT)
+
 struct str; /* defined in space.c */
 
 extern struct str * str_new(void);
 extern void str_delete(struct str * str);
-extern void str_append(struct str * str, struct str * add);
+extern void str_append(struct str * str, const struct str * add);
 extern void str_append_ch(struct str * str, char add);
-extern void str_append_b(struct str * str, symbol * q);
+extern void str_append_b(struct str * str, const symbol * q);
+extern void str_append_b_tail(struct str * str, const symbol * q, int skip);
 extern void str_append_string(struct str * str, const char * s);
 extern void str_append_int(struct str * str, int i);
 extern void str_clear(struct str * str);
-extern void str_assign(struct str * str, char * s);
-extern struct str * str_copy(struct str * old);
-extern symbol * str_data(struct str * str);
-extern int str_len(struct str * str);
+extern void str_assign(struct str * str, const char * s);
+extern struct str * str_copy(const struct str * old);
+extern symbol * str_data(const struct str * str);
+extern int str_len(const struct str * str);
+extern int str_back(const struct str *str);
 extern int get_utf8(const symbol * p, int * slot);
 extern int put_utf8(int ch, symbol * p);
+extern void output_str(FILE * outfile, struct str * str);
+
+typedef enum { ENC_SINGLEBYTE, ENC_UTF8, ENC_WIDECHARS } enc;
 
 struct m_pair {
 
@@ -60,6 +67,7 @@ struct input {
     symbol * p;
     int c;
     char * file;
+    int file_needs_freeing;
     int line_number;
 
 };
@@ -71,6 +79,28 @@ struct include {
 
 };
 
+enum token_codes {
+
+#include "syswords2.h"
+
+    c_mathassign,
+    c_name,
+    c_number,
+    c_literalstring,
+    c_neg,
+    c_call,
+    c_grouping,
+    c_booltest,
+
+    NUM_TOKEN_CODES
+};
+
+enum uplus_modes {
+    UPLUS_NONE,
+    UPLUS_DEFINED,
+    UPLUS_UNICODE
+};
+
 /* struct input must be a prefix of struct tokeniser. */
 struct tokeniser {
 
@@ -78,6 +108,7 @@ struct tokeniser {
     symbol * p;
     int c;
     char * file;
+    int file_needs_freeing;
     int line_number;
     symbol * b;
     symbol * b2;
@@ -90,34 +121,28 @@ struct tokeniser {
     int token;
     int previous_token;
     byte token_held;
-    byte widechars;
-    byte utf8;
+    enc encoding;
 
     int omission;
     struct include * includes;
 
+    /* Mode in which U+ has been used:
+     * UPLUS_NONE - not used yet
+     * UPLUS_DEFINED - stringdef U+xxxx ....
+     * UPLUS_UNICODE - {U+xxxx} used with implicit meaning
+     */
+    int uplusmode;
+
+    char token_disabled[NUM_TOKEN_CODES];
 };
 
-extern symbol * get_input(symbol * p, char ** p_file);
+extern symbol * get_input(const char * filename);
 extern struct tokeniser * create_tokeniser(symbol * b, char * file);
 extern int read_token(struct tokeniser * t);
 extern const char * name_of_token(int code);
+extern void disable_token(struct tokeniser * t, int code);
 extern void close_tokeniser(struct tokeniser * t);
 
-enum token_codes {
-
-#include "syswords2.h"
-
-    c_mathassign,
-    c_name,
-    c_number,
-    c_literalstring,
-    c_neg,
-    c_call,
-    c_grouping,
-    c_booltest
-};
-
 extern int space_count;
 extern void * check_malloc(int n);
 extern void check_free(void * p);
@@ -134,7 +159,13 @@ struct name {
     int count;                  /* 0, 1, 2 for each type */
     struct grouping * grouping; /* for grouping names */
     byte referenced;
-    byte used;
+    byte used_in_among;         /* Function used in among? */
+    byte value_used;            /* (For variables) is its value ever used? */
+    byte initialised;           /* (For variables) is it ever initialised? */
+    byte used_in_definition;    /* (grouping) used in grouping definition? */
+    struct node * used;         /* First use, or NULL if not used */
+    struct name * local_to;     /* Local to one routine/external */
+    int declaration_line_number;/* Line number of declaration */
 
 };
 
@@ -149,9 +180,10 @@ struct amongvec {
 
     symbol * b;      /* the string giving the case */
     int size;        /* - and its size */
-    struct node * p; /* the corresponding command */
+    struct node * action; /* the corresponding action */
     int i;           /* the amongvec index of the longest substring of b */
     int result;      /* the numeric result for the case */
+    int line_number; /* for diagnostics and stable sorting */
     struct name * function;
 
 };
@@ -163,19 +195,22 @@ struct among {
     int number;               /* amongs are numbered 0, 1, 2 ... */
     int literalstring_count;  /* in this among */
     int command_count;        /* in this among */
+    int nocommand_count;      /* number of "no command" entries in this among */
+    int function_count;       /* in this among */
+    int amongvar_needed;      /* do we need to set among_var? */
     struct node * starter;    /* i.e. among( (starter) 'string' ... ) */
     struct node * substring;  /* i.e. substring ... among ( ... ) */
+    struct node ** commands;  /* array with command_count entries */
 };
 
 struct grouping {
 
     struct grouping * next;
-    int number;               /* groupings are numbered 0, 1, 2 ... */
     symbol * b;               /* the characters of this group */
     int largest_ch;           /* character with max code */
     int smallest_ch;          /* character with min code */
-    byte no_gaps;             /* not used in generator.c after 11/5/05 */
     struct name * name;       /* so g->name->grouping == g */
+    int line_number;
 };
 
 struct node {
@@ -234,7 +269,8 @@ struct analyser {
     struct grouping * groupings;
     struct grouping * groupings_end;
     struct node * substring;  /* pending 'substring' in current routine definition */
-    byte utf8;
+    enc encoding;
+    byte int_limits_used;     /* are maxint or minint used? */
 };
 
 enum analyser_modes {
@@ -259,16 +295,23 @@ struct generator {
     struct str * outbuf;       /* temporary str to store output */
     struct str * declarations; /* str storing variable declarations */
     int next_label;
+#ifndef DISABLE_PYTHON
+    int max_label;
+#endif
     int margin;
 
-    const char * failure_string;     /* String to output in case of a failure. */
-#ifndef DISABLE_JAVA
-    struct str * failure_str;  /* This is used by the java generator instead of failure_string */
+    /* if > 0, keep_count to restore in case of a failure;
+     * if < 0, the negated keep_count for the limit to restore in case of
+     * failure. */
+    int failure_keep_count;
+#if !defined(DISABLE_JAVA) && !defined(DISABLE_JS) && !defined(DISABLE_PYTHON) && !defined(DISABLE_CSHARP)
+    struct str * failure_str;  /* This is used by some generators instead of failure_keep_count */
 #endif
 
     int label_used;     /* Keep track of whether the failure label is used. */
     int failure_label;
     int debug_count;
+    int copy_from_count; /* count of calls to copy_from() */
 
     const char * S[10];  /* strings */
     symbol * B[10];      /* blocks */
@@ -277,48 +320,92 @@ struct generator {
     symbol * L[5];       /* literals, used in formatted write */
 
     int line_count;      /* counts number of lines output */
-    int line_labelled;   /* in ANSI C, will need extra ';' if it is a block end */
+    int line_labelled;   /* in ISO C, will need extra ';' if it is a block end */
     int literalstring_count;
     int keep_count;      /* used to number keep/restore pairs to avoid compiler warnings
                             about shadowed variables */
 };
 
+/* Special values for failure_label in struct generator. */
+enum special_labels {
+    x_return = -1
+};
+
 struct options {
 
     /* for the command line: */
 
-    char * output_file;
-    char * name;
-    FILE * output_c;
+    const char * output_file;
+    const char * name;
+    FILE * output_src;
     FILE * output_h;
-#ifndef DISABLE_JAVA
-    FILE * output_java;
-#endif
     byte syntax_tree;
-    byte widechars;
-    enum { LANG_JAVA, LANG_C, LANG_CPLUSPLUS } make_lang;
-    char * externals_prefix;
-    char * variables_prefix;
-    char * runtime_path;
-    char * parent_class_name;
-    char * package;
-    char * string_class;
-    char * among_class;
+    byte comments;
+    enc encoding;
+    enum { LANG_JAVA, LANG_C, LANG_CPLUSPLUS, LANG_CSHARP, LANG_PASCAL, LANG_PYTHON, LANG_JAVASCRIPT, LANG_RUST, LANG_GO } make_lang;
+    const char * externals_prefix;
+    const char * variables_prefix;
+    const char * runtime_path;
+    const char * parent_class_name;
+    const char * package;
+    const char * go_snowball_runtime;
+    const char * string_class;
+    const char * among_class;
     struct include * includes;
     struct include * includes_end;
-    byte utf8;
 };
 
-/* Generator for C code. */
-extern struct generator * create_generator_c(struct analyser * a, struct options * o);
-extern void close_generator_c(struct generator * g);
+/* Generator functions common to several backends. */
+
+extern struct generator * create_generator(struct analyser * a, struct options * o);
+extern void close_generator(struct generator * g);
+
+extern void write_char(struct generator * g, int ch);
+extern void write_newline(struct generator * g);
+extern void write_string(struct generator * g, const char * s);
+extern void write_int(struct generator * g, int i);
+extern void write_b(struct generator * g, symbol * b);
+extern void write_str(struct generator * g, struct str * str);
+
+extern void write_comment_content(struct generator * g, struct node * p);
+extern void write_generated_comment_content(struct generator * g);
+extern void write_start_comment(struct generator * g,
+                                const char * comment_start,
+                                const char * comment_end);
+
+extern int K_needed(struct generator * g, struct node * p);
+extern int repeat_restore(struct generator * g, struct node * p);
 
+/* Generator for C code. */
 extern void generate_program_c(struct generator * g);
 
 #ifndef DISABLE_JAVA
 /* Generator for Java code. */
-extern struct generator * create_generator_java(struct analyser * a, struct options * o);
-extern void close_generator_java(struct generator * g);
-
 extern void generate_program_java(struct generator * g);
 #endif
+
+#ifndef DISABLE_CSHARP
+/* Generator for C# code. */
+extern void generate_program_csharp(struct generator * g);
+#endif
+
+#ifndef DISABLE_PASCAL
+extern void generate_program_pascal(struct generator * g);
+#endif
+
+#ifndef DISABLE_PYTHON
+/* Generator for Python code. */
+extern void generate_program_python(struct generator * g);
+#endif
+
+#ifndef DISABLE_JS
+extern void generate_program_js(struct generator * g);
+#endif
+
+#ifndef DISABLE_RUST
+extern void generate_program_rust(struct generator * g);
+#endif
+
+#ifndef DISABLE_GO
+extern void generate_program_go(struct generator * g);
+#endif
diff --git a/contrib/snowball/compiler/space.c b/contrib/snowball/compiler/space.c
index 310024e76..5b058763a 100644
--- a/contrib/snowball/compiler/space.c
+++ b/contrib/snowball/compiler/space.c
@@ -57,9 +57,19 @@ extern symbol * create_b(int n) {
     return p;
 }
 
-extern void report_b(FILE * out, symbol * p) {
+extern void report_b(FILE * out, const symbol * p) {
     int i;
-    for (i = 0; i < SIZE(p); i++) fprintf(out, "%c", p[i]);
+    for (i = 0; i < SIZE(p); i++) {
+        if (p[i] > 255) {
+            printf("In report_b, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]);
+            exit(1);
+        }
+        putc(p[i], out);
+    }
+}
+
+extern void output_str(FILE * outfile, struct str * str) {
+    report_b(outfile, str_data(str));
 }
 
 extern void lose_b(symbol * p) {
@@ -74,19 +84,19 @@ extern symbol * increase_capacity(symbol * p, int n) {
     lose_b(p); return q;
 }
 
-extern symbol * move_to_b(symbol * p, int n, symbol * q) {
+extern symbol * move_to_b(symbol * p, int n, const symbol * q) {
     int x = n - CAPACITY(p);
     if (x > 0) p = increase_capacity(p, x);
     memmove(p, q, n * sizeof(symbol)); SIZE(p) = n; return p;
 }
 
-extern symbol * add_to_b(symbol * p, int n, symbol * q) {
+extern symbol * add_to_b(symbol * p, int n, const symbol * q) {
     int x = SIZE(p) + n - CAPACITY(p);
     if (x > 0) p = increase_capacity(p, x);
     memmove(p + SIZE(p), q, n * sizeof(symbol)); SIZE(p) += n; return p;
 }
 
-extern symbol * copy_b(symbol * p) {
+extern symbol * copy_b(const symbol * p) {
     int n = SIZE(p);
     symbol * q = create_b(n);
     move_to_b(q, n, p);
@@ -97,7 +107,7 @@ int space_count = 0;
 
 extern void * check_malloc(int n) {
     space_count++;
-    return calloc(1, n);
+    return malloc(n);
 }
 
 extern void check_free(void * p) {
@@ -107,18 +117,18 @@ extern void check_free(void * p) {
 
 /* To convert a block to a zero terminated string:  */
 
-extern char * b_to_s(symbol * p) {
+extern char * b_to_s(const symbol * p) {
     int n = SIZE(p);
-    char * s = (char *)calloc(1, n + 1);
+    char * s = (char *)malloc(n + 1);
     {
         int i;
         for (i = 0; i < n; i++) {
-	    if (p[i] > 255) {
-		printf("In b_to_s, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]);
-		exit(1);
-	    }
-	    s[i] = (char)p[i];
-	}
+            if (p[i] > 255) {
+                printf("In b_to_s, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]);
+                exit(1);
+            }
+            s[i] = (char)p[i];
+        }
     }
     s[n] = 0;
     return s;
@@ -153,9 +163,9 @@ struct str {
 };
 
 /* Create a new string. */
-extern struct str * str_new() {
+extern struct str * str_new(void) {
 
-    struct str * output = (struct str *) calloc(1, sizeof(struct str));
+    struct str * output = (struct str *) malloc(sizeof(struct str));
     output->data = create_b(0);
     return output;
 }
@@ -168,7 +178,7 @@ extern void str_delete(struct str * str) {
 }
 
 /* Append a str to this str. */
-extern void str_append(struct str * str, struct str * add) {
+extern void str_append(struct str * str, const struct str * add) {
 
     symbol * q = add->data;
     str->data = add_to_b(str->data, SIZE(q), q);
@@ -183,12 +193,19 @@ extern void str_append_ch(struct str * str, char add) {
 }
 
 /* Append a low level block to a str. */
-extern void str_append_b(struct str * str, symbol * q) {
+extern void str_append_b(struct str * str, const symbol * q) {
 
     str->data = add_to_b(str->data, SIZE(q), q);
 }
 
-/* Append a (char *, null teminated) string to a str. */
+/* Append the tail of a low level block to a str. */
+extern void str_append_b_tail(struct str * str, const symbol * q, int skip) {
+    if (skip < 0 || skip >= SIZE(q)) return;
+
+    str->data = add_to_b(str->data, SIZE(q) - skip, q + skip);
+}
+
+/* Append a (char *, null terminated) string to a str. */
 extern void str_append_string(struct str * str, const char * s) {
 
     str->data = add_s_to_b(str->data, s);
@@ -209,14 +226,14 @@ extern void str_clear(struct str * str) {
 }
 
 /* Set a string */
-extern void str_assign(struct str * str, char * s) {
+extern void str_assign(struct str * str, const char * s) {
 
     str_clear(str);
     str_append_string(str, s);
 }
 
 /* Copy a string. */
-extern struct str * str_copy(struct str * old) {
+extern struct str * str_copy(const struct str * old) {
 
     struct str * newstr = str_new();
     str_append(newstr, old);
@@ -224,17 +241,25 @@ extern struct str * str_copy(struct str * old) {
 }
 
 /* Get the data stored in this str. */
-extern symbol * str_data(struct str * str) {
+extern symbol * str_data(const struct str * str) {
 
     return str->data;
 }
 
 /* Get the length of the str. */
-extern int str_len(struct str * str) {
+extern int str_len(const struct str * str) {
 
     return SIZE(str->data);
 }
 
+/* Get the last character of the str.
+ *
+ * Or -1 if the string is empty.
+ */
+extern int str_back(const struct str *str) {
+    return SIZE(str->data) ? str->data[SIZE(str->data) - 1] : -1;
+}
+
 extern int get_utf8(const symbol * p, int * slot) {
     int b0, b1;
     b0 = *p++;
@@ -260,4 +285,3 @@ extern int put_utf8(int ch, symbol * p) {
     p[1] = ((ch >> 6) & 0x3F) | 0x80;
     p[2] = (ch & 0x3F) | 0x80; return 3;
 }
-
diff --git a/contrib/snowball/compiler/syswords.h b/contrib/snowball/compiler/syswords.h
index 917dd5751..c401d3e4f 100644
--- a/contrib/snowball/compiler/syswords.h
+++ b/contrib/snowball/compiler/syswords.h
@@ -1,5 +1,5 @@
-static const struct system_word vocab[80+1] = {
-  { 0, (const byte *)"", 80+1},
+static const struct system_word vocab[82+1] = {
+  { 0, (const byte *)"", 82+1},
 
   { 1, (const byte *)"$",             c_dollar },
   { 1, (const byte *)"(",             c_bra },
@@ -36,6 +36,7 @@ static const struct system_word vocab[80+1] = {
   { 3, (const byte *)"get",           c_get },
   { 3, (const byte *)"hex",           c_hex },
   { 3, (const byte *)"hop",           c_hop },
+  { 3, (const byte *)"len",           c_len },
   { 3, (const byte *)"non",           c_non },
   { 3, (const byte *)"not",           c_not },
   { 3, (const byte *)"set",           c_set },
@@ -49,6 +50,7 @@ static const struct system_word vocab[80+1] = {
   { 4, (const byte *)"true",          c_true },
   { 5, (const byte *)"among",         c_among },
   { 5, (const byte *)"false",         c_false },
+  { 5, (const byte *)"lenof",         c_lenof },
   { 5, (const byte *)"limit",         c_limit },
   { 5, (const byte *)"unset",         c_unset },
   { 6, (const byte *)"atmark",        c_atmark },
diff --git a/contrib/snowball/compiler/syswords2.h b/contrib/snowball/compiler/syswords2.h
index eb8a91241..e853e32dc 100644
--- a/contrib/snowball/compiler/syswords2.h
+++ b/contrib/snowball/compiler/syswords2.h
@@ -4,8 +4,8 @@
     c_decimal, c_define, c_delete, c_divide, c_divideassign, c_do,
     c_dollar, c_eq, c_externals, c_fail, c_false, c_for, c_ge, c_get,
     c_gopast, c_goto, c_gr, c_groupings, c_hex, c_hop, c_insert,
-    c_integers, c_ket, c_le, c_leftslice, c_limit, c_loop, c_ls,
-    c_maxint, c_minint, c_minus, c_minusassign, c_multiply,
+    c_integers, c_ket, c_le, c_leftslice, c_len, c_lenof, c_limit, c_loop,
+    c_ls, c_maxint, c_minint, c_minus, c_minusassign, c_multiply,
     c_multiplyassign, c_ne, c_next, c_non, c_not, c_or, c_plus,
     c_plusassign, c_repeat, c_reverse, c_rightslice, c_routines,
     c_set, c_setlimit, c_setmark, c_size, c_sizeof, c_slicefrom,
diff --git a/contrib/snowball/compiler/tokeniser.c b/contrib/snowball/compiler/tokeniser.c
index 3dae5f744..e6c63861a 100644
--- a/contrib/snowball/compiler/tokeniser.c
+++ b/contrib/snowball/compiler/tokeniser.c
@@ -16,57 +16,57 @@ struct system_word {
 
 #include "syswords.h"
 
-static int smaller(int a, int b) { return a < b ? a : b; }
+#define INITIAL_INPUT_BUFFER_SIZE 8192
+
+static int hex_to_num(int ch);
 
-extern symbol * get_input(symbol * p, char ** p_file) {
+static int smaller(int a, int b) { return a < b ? a : b; }
 
-    char * s = b_to_s(p);
+extern symbol * get_input(const char * filename) {
+    FILE * input = fopen(filename, "r");
+    if (input == 0) { return 0; }
     {
-        FILE * input = fopen(s, "r");
-        if (input == 0) { free(s); return 0; }
-        *p_file = s;
-        {
-            symbol * u = create_b(STARTSIZE);
-            int size = 0;
-            repeat
-            {   int ch = getc(input);
-                if (ch == EOF) break;
-                if (size >= CAPACITY(u)) u = increase_capacity(u, size/2);
-                u[size++] = ch;
-            }
-            fclose(input);
-            SIZE(u) = size; return u;
+        symbol * u = create_b(INITIAL_INPUT_BUFFER_SIZE);
+        int size = 0;
+        while (true) {
+            int ch = getc(input);
+            if (ch == EOF) break;
+            if (size >= CAPACITY(u)) u = increase_capacity(u, size);
+            u[size++] = ch;
         }
+        fclose(input);
+        SIZE(u) = size;
+        return u;
     }
 }
 
-static void error(struct tokeniser * t, char * s1, int n, symbol * p, char * s2) {
+static void error(struct tokeniser * t, const char * s1, int n, symbol * p, const char * s2) {
     if (t->error_count == 20) { fprintf(stderr, "... etc\n"); exit(1); }
     fprintf(stderr, "%s:%d: ", t->file, t->line_number);
-    unless (s1 == 0) fprintf(stderr, "%s", s1);
-    unless (p == 0) {
+    if (s1) fprintf(stderr, "%s", s1);
+    if (p) {
         int i;
         for (i = 0; i < n; i++) fprintf(stderr, "%c", p[i]);
     }
-    unless (s2 == 0) fprintf(stderr, "%s", s2);
+    if (s2) fprintf(stderr, "%s", s2);
     fprintf(stderr, "\n");
     t->error_count++;
 }
 
-static void error1(struct tokeniser * t, char * s) {
+static void error1(struct tokeniser * t, const char * s) {
     error(t, s, 0,0, 0);
 }
 
-static void error2(struct tokeniser * t, char * s) {
+static void error2(struct tokeniser * t, const char * s) {
     error(t, "unexpected end of text after ", 0,0, s);
 }
 
 static int compare_words(int m, symbol * p, int n, const byte * q) {
-    unless (m == n) return m - n;
+    if (m != n) return m - n;
     {
         int i; for (i = 0; i < n; i++) {
             int diff = p[i] - q[i];
-            unless (diff == 0) return diff;
+            if (diff) return diff;
         }
     }
     return 0;
@@ -74,14 +74,13 @@ static int compare_words(int m, symbol * p, int n, const byte * q) {
 
 static int find_word(int n, symbol * p) {
     int i = 0; int j = vocab->code;
-    repeat {
+    do {
         int k = i + (j - i)/2;
         const struct system_word * w = vocab + k;
         int diff = compare_words(n, p, w->s_size, w->s);
         if (diff == 0) return w->code;
         if (diff < 0) j = k; else i = k;
-        if (j - i == 1) break;
-    }
+    } while (j - i != 1);
     return -1;
 }
 
@@ -91,7 +90,7 @@ static int get_number(int n, symbol * p) {
     return x;
 }
 
-static int eq_s(struct tokeniser * t, char * s) {
+static int eq_s(struct tokeniser * t, const char * s) {
     int l = strlen(s);
     if (SIZE(t->p) - t->c < l) return false;
     {
@@ -103,64 +102,141 @@ static int eq_s(struct tokeniser * t, char * s) {
 
 static int white_space(struct tokeniser * t, int ch) {
     switch (ch) {
-        case '\n': t->line_number++;
+        case '\n':
+            t->line_number++;
+            /* fall through */
         case '\r':
         case '\t':
-        case ' ': return true;
+        case ' ':
+            return true;
     }
     return false;
 }
 
 static symbol * find_in_m(struct tokeniser * t, int n, symbol * p) {
-    struct m_pair * q = t->m_pairs;
-    repeat {
-        if (q == 0) return 0;
-        {
-            symbol * name = q->name;
-            if (n == SIZE(name) && memcmp(name, p, n * sizeof(symbol)) == 0) return q->value;
-        }
-        q = q->next;
+    struct m_pair * q;
+    for (q = t->m_pairs; q; q = q->next) {
+        symbol * name = q->name;
+        if (n == SIZE(name) && memcmp(name, p, n * sizeof(symbol)) == 0) return q->value;
     }
+    return 0;
 }
 
 static int read_literal_string(struct tokeniser * t, int c) {
     symbol * p = t->p;
     int ch;
     SIZE(t->b) = 0;
-    repeat {
+    while (true) {
         if (c >= SIZE(p)) { error2(t, "'"); return c; }
         ch = p[c];
         if (ch == '\n') { error1(t, "string not terminated"); return c; }
         c++;
         if (ch == t->m_start) {
+            /* Inside insert characters. */
             int c0 = c;
             int newlines = false; /* no newlines as yet */
             int black_found = false; /* no printing chars as yet */
-            repeat {
+            while (true) {
                 if (c >= SIZE(p)) { error2(t, "'"); return c; }
                 ch = p[c]; c++;
                 if (ch == t->m_end) break;
-                unless (white_space(t, ch)) black_found = true;
+                if (!white_space(t, ch)) black_found = true;
                 if (ch == '\n') newlines = true;
                 if (newlines && black_found) {
                     error1(t, "string not terminated");
                     return c;
                 }
             }
-            unless (newlines) {
+            if (!newlines) {
                 int n = c - c0 - 1;    /* macro size */
                 int firstch = p[c0];
                 symbol * q = find_in_m(t, n, p + c0);
                 if (q == 0) {
                     if (n == 1 && (firstch == '\'' || firstch == t->m_start))
                         t->b = add_to_b(t->b, 1, p + c0);
-                    else
+                    else if (n >= 3 && firstch == 'U' && p[c0 + 1] == '+') {
+                        int codepoint = 0;
+                        int x;
+                        if (t->uplusmode == UPLUS_DEFINED) {
+                            /* See if found with xxxx upper-cased. */
+                            symbol * uc = create_b(n);
+                            int i;
+                            for (i = 0; i != n; ++i) {
+                                uc[i] = toupper(p[c0 + i]);
+                            }
+                            q = find_in_m(t, n, uc);
+                            lose_b(uc);
+                            if (q != 0) {
+                                t->b = add_to_b(t->b, SIZE(q), q);
+                                continue;
+                            }
+                            error1(t, "Some U+xxxx stringdefs seen but not this one");
+                        } else {
+                            t->uplusmode = UPLUS_UNICODE;
+                        }
+                        for (x = c0 + 2; x != c - 1; ++x) {
+                            int hex = hex_to_num(p[x]);
+                            if (hex < 0) {
+                                error1(t, "Bad hex digit following U+");
+                                break;
+                            }
+                            codepoint = (codepoint << 4) | hex;
+                        }
+                        if (t->encoding == ENC_UTF8) {
+                            if (codepoint < 0 || codepoint > 0x01ffff) {
+                                error1(t, "character values exceed 0x01ffff");
+                            }
+                            /* Ensure there's enough space for a max length
+                             * UTF-8 sequence. */
+                            if (CAPACITY(t->b) < SIZE(t->b) + 3) {
+                                t->b = increase_capacity(t->b, 3);
+                            }
+                            SIZE(t->b) += put_utf8(codepoint, t->b + SIZE(t->b));
+                        } else {
+                            symbol sym;
+                            if (t->encoding == ENC_SINGLEBYTE) {
+                                /* Only ISO-8859-1 is handled this way - for
+                                 * other single-byte character sets you need
+                                 * stringdef all the U+xxxx codes you use
+                                 * like - e.g.:
+                                 *
+                                 * stringdef U+0171   hex 'FB'
+                                 */
+                                if (codepoint < 0 || codepoint > 0xff) {
+                                    error1(t, "character values exceed 256");
+                                }
+                            } else {
+                                if (codepoint < 0 || codepoint > 0xffff) {
+                                    error1(t, "character values exceed 64K");
+                                }
+                            }
+                            sym = codepoint;
+                            t->b = add_to_b(t->b, 1, &sym);
+                        }
+                    } else
                         error(t, "string macro '", n, p + c0, "' undeclared");
                 } else
                     t->b = add_to_b(t->b, SIZE(q), q);
             }
         } else {
             if (ch == '\'') return c;
+            if (ch < 0 || ch >= 0x80) {
+                if (t->encoding != ENC_WIDECHARS) {
+                    /* We don't really want people using non-ASCII literal
+                     * strings, but historically it's worked for single-byte
+                     * and UTF-8 if the source encoding matches what the
+                     * generated stemmer works in and it seems unfair to just
+                     * suddenly make this a hard error.`
+                     */
+                    fprintf(stderr,
+                            "%s:%d: warning: Non-ASCII literal strings aren't "
+                            "portable - use stringdef instead\n",
+                            t->file, t->line_number);
+                } else {
+                    error1(t, "Non-ASCII literal strings aren't "
+                              "portable - use stringdef instead");
+                }
+            }
             t->b = add_to_b(t->b, 1, p + c - 1);
         }
     }
@@ -171,7 +247,7 @@ static int next_token(struct tokeniser * t) {
     int c = t->c;
     int ch;
     int code = -1;
-    repeat {
+    while (true) {
         if (c >= SIZE(p)) { t->c = c; return -1; }
         ch = p[c];
         if (white_space(t, ch)) { c++; continue; }
@@ -179,7 +255,7 @@ static int next_token(struct tokeniser * t) {
             int c0 = c;
             while (c < SIZE(p) && (isalnum(p[c]) || p[c] == '_')) c++;
             code = find_word(c - c0, p + c0);
-            if (code < 0) {
+            if (code < 0 || t->token_disabled[code]) {
                 t->b = move_to_b(t->b, c - c0, p + c0);
                 code = c_name;
             }
@@ -218,10 +294,9 @@ static int next_char(struct tokeniser * t) {
 }
 
 static int next_real_char(struct tokeniser * t) {
-    repeat {
+    while (true) {
         int ch = next_char(t);
-        if (white_space(t, ch)) continue;
-        return ch;
+        if (!white_space(t, ch)) return ch;
     }
 }
 
@@ -230,7 +305,7 @@ static void read_chars(struct tokeniser * t) {
     if (ch < 0) { error2(t, "stringdef"); return; }
     {
         int c0 = t->c-1;
-        repeat {
+        while (true) {
             ch = next_char(t);
             if (white_space(t, ch) || ch < 0) break;
         }
@@ -246,19 +321,20 @@ static int decimal_to_num(int ch) {
 static int hex_to_num(int ch) {
     if ('0' <= ch && ch <= '9') return ch - '0';
     if ('a' <= ch && ch <= 'f') return ch - 'a' + 10;
+    if ('A' <= ch && ch <= 'F') return ch - 'A' + 10;
     return -1;
 }
 
 static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) {
     int c = 0; int d = 0;
-    repeat {
+    while (true) {
         while (c < SIZE(p) && p[c] == ' ') c++;
         if (c == SIZE(p)) break;
         {
             int number = 0;
-            repeat {
+            while (c != SIZE(p)) {
                 int ch = p[c];
-                if (c == SIZE(p) || ch == ' ') break;
+                if (ch == ' ') break;
                 if (base == 10) {
                     ch = decimal_to_num(ch);
                     if (ch < 0) {
@@ -266,7 +342,7 @@ static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) {
                         return;
                     }
                 } else {
-                    ch = hex_to_num(tolower(ch));
+                    ch = hex_to_num(ch);
                     if (ch < 0) {
                         error1(t, "hex string contains non-hex characters");
                         return;
@@ -275,18 +351,18 @@ static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) {
                 number = base * number + ch;
                 c++;
             }
-            if (t->widechars || t->utf8) {
-                unless (0 <= number && number <= 0xffff) {
-                    error1(t, "character values exceed 64K");
+            if (t->encoding == ENC_SINGLEBYTE) {
+                if (number < 0 || number > 0xff) {
+                    error1(t, "character values exceed 256");
                     return;
                 }
             } else {
-                unless (0 <= number && number <= 0xff) {
-                    error1(t, "character values exceed 256");
+                if (number < 0 || number > 0xffff) {
+                    error1(t, "character values exceed 64K");
                     return;
                 }
             }
-            if (t->utf8)
+            if (t->encoding == ENC_UTF8)
                 d += put_utf8(number, p + d);
             else
                 p[d++] = number;
@@ -300,104 +376,118 @@ extern int read_token(struct tokeniser * t) {
     int held = t->token_held;
     t->token_held = false;
     if (held) return t->token;
-    repeat {
+    while (true) {
         int code = next_token(t);
         switch (code) {
             case c_comment1: /*  slash-slash comment */
-               while (t->c < SIZE(p) && p[t->c] != '\n') t->c++;
-               continue;
+                while (t->c < SIZE(p) && p[t->c] != '\n') t->c++;
+                continue;
             case c_comment2: /* slash-star comment */
-               repeat {
-                   if (t->c >= SIZE(p)) {
-                       error1(t, "/* comment not terminated");
-                       t->token = -1;
-                       return -1;
-                   }
-                   if (p[t->c] == '\n') t->line_number++;
-                   if (eq_s(t, "*/")) break;
-                   t->c++;
-               }
-               continue;
-            case c_stringescapes:
-               {
-                   int ch1 = next_real_char(t);
-                   int ch2 = next_real_char(t);
-                   if (ch2 < 0)
-                       { error2(t, "stringescapes"); continue; }
-                   if (ch1 == '\'')
-                       { error1(t, "first stringescape cannot be '"); continue; }
-                   t->m_start = ch1;
-                   t->m_end = ch2;
-               }
-               continue;
-            case c_stringdef:
-               {
-                   int base = 0;
-                   read_chars(t);
-                   code = read_token(t);
-                   if (code == c_hex) { base = 16; code = read_token(t); } else
-                   if (code == c_decimal) { base = 10; code = read_token(t); }
-                   unless (code == c_literalstring)
-                       { error1(t, "string omitted after stringdef"); continue; }
-                   if (base > 0) convert_numeric_string(t, t->b, base);
-                   {   NEW(m_pair, q);
-                       q->next = t->m_pairs;
-                       q->name = copy_b(t->b2);
-                       q->value = copy_b(t->b);
-                       t->m_pairs = q;
-                   }
-               }
-               continue;
+                while (true) {
+                    if (t->c >= SIZE(p)) {
+                        error1(t, "/* comment not terminated");
+                        t->token = -1;
+                        return -1;
+                    }
+                    if (p[t->c] == '\n') t->line_number++;
+                    if (eq_s(t, "*/")) break;
+                    t->c++;
+                }
+                continue;
+            case c_stringescapes: {
+                int ch1 = next_real_char(t);
+                int ch2 = next_real_char(t);
+                if (ch2 < 0) {
+                    error2(t, "stringescapes");
+                    continue;
+                }
+                if (ch1 == '\'') {
+                    error1(t, "first stringescape cannot be '");
+                    continue;
+                }
+                t->m_start = ch1;
+                t->m_end = ch2;
+                continue;
+            }
+            case c_stringdef: {
+                int base = 0;
+                read_chars(t);
+                code = read_token(t);
+                if (code == c_hex) { base = 16; code = read_token(t); } else
+                if (code == c_decimal) { base = 10; code = read_token(t); }
+                if (code != c_literalstring) {
+                    error1(t, "string omitted after stringdef");
+                    continue;
+                }
+                if (base > 0) convert_numeric_string(t, t->b, base);
+                {   NEW(m_pair, q);
+                    q->next = t->m_pairs;
+                    q->name = copy_b(t->b2);
+                    q->value = copy_b(t->b);
+                    t->m_pairs = q;
+                    if (t->uplusmode != UPLUS_DEFINED &&
+                        (SIZE(t->b2) >= 3 && t->b2[0] == 'U' && t->b2[1] == '+')) {
+                        if (t->uplusmode == UPLUS_UNICODE) {
+                            error1(t, "U+xxxx already used with implicit meaning");
+                        } else {
+                            t->uplusmode = UPLUS_DEFINED;
+                        }
+                    }
+                }
+                continue;
+            }
             case c_get:
-               code = read_token(t);
-               unless (code == c_literalstring) {
-                   error1(t, "string omitted after get"); continue;
-               }
-               t->get_depth++;
-               if (t->get_depth > 10) {
-                   fprintf(stderr, "get directives go 10 deep. Looping?\n");
-                   exit(1);
-               }
-               {
-                   char * file;
-                   NEW(input, q);
-                   symbol * u = get_input(t->b, &file);
-                   if (u == 0) {
-                       struct include * r = t->includes;
-                       until (r == 0) {
-                           symbol * b = copy_b(r->b);
-                           b = add_to_b(b, SIZE(t->b), t->b);
-                           u = get_input(b, &file);
-                           lose_b(b);
-                           unless (u == 0) break;
-                           r = r->next;
-                       }
-                   }
-                   if (u == 0) {
-                       error(t, "Can't get '", SIZE(t->b), t->b, "'");
-                       exit(1);
-                   }
-                   memmove(q, t, sizeof(struct input));
-                   t->next = q;
-                   t->p = u;
-                   t->c = 0;
-                   t->file = file;
-                   t->line_number = 1;
-               }
-               p = t->p;
-               continue;
+                code = read_token(t);
+                if (code != c_literalstring) {
+                    error1(t, "string omitted after get"); continue;
+                }
+                t->get_depth++;
+                if (t->get_depth > 10) {
+                    fprintf(stderr, "get directives go 10 deep. Looping?\n");
+                    exit(1);
+                }
+                {
+                    NEW(input, q);
+                    char * file = b_to_s(t->b);
+                    symbol * u = get_input(file);
+                    if (u == 0) {
+                        struct include * r;
+                        for (r = t->includes; r; r = r->next) {
+                            symbol * b = copy_b(r->b);
+                            b = add_to_b(b, SIZE(t->b), t->b);
+                            free(file);
+                            file = b_to_s(b);
+                            u = get_input(file);
+                            lose_b(b);
+                            if (u != 0) break;
+                        }
+                    }
+                    if (u == 0) {
+                        error(t, "Can't get '", SIZE(t->b), t->b, "'");
+                        exit(1);
+                    }
+                    memmove(q, t, sizeof(struct input));
+                    t->next = q;
+                    t->p = u;
+                    t->c = 0;
+                    t->file = file;
+                    t->file_needs_freeing = true;
+                    t->line_number = 1;
+                }
+                p = t->p;
+                continue;
             case -1:
-               unless (t->next == 0) {
-                   lose_b(p);
-                   {
-                       struct input * q = t->next;
-                       memmove(t, q, sizeof(struct input)); p = t->p;
-                       FREE(q);
-                   }
-                   t->get_depth--;
-                   continue;
-               }
-               /* drop through */
+                if (t->next) {
+                    lose_b(p);
+                    {
+                        struct input * q = t->next;
+                        memmove(t, q, sizeof(struct input)); p = t->p;
+                        FREE(q);
+                    }
+                    t->get_depth--;
+                    continue;
+                }
+                /* fall through */
             default:
                 t->previous_token = t->token;
                 t->token = code;
@@ -425,12 +515,17 @@ extern const char * name_of_token(int code) {
     }
 }
 
+extern void disable_token(struct tokeniser * t, int code) {
+    t->token_disabled[code] = 1;
+}
+
 extern struct tokeniser * create_tokeniser(symbol * p, char * file) {
     NEW(tokeniser, t);
     t->next = 0;
     t->p = p;
     t->c = 0;
     t->file = file;
+    t->file_needs_freeing = false;
     t->line_number = 1;
     t->b = create_b(0);
     t->b2 = create_b(0);
@@ -441,6 +536,8 @@ extern struct tokeniser * create_tokeniser(symbol * p, char * file) {
     t->token_held = false;
     t->token = -2;
     t->previous_token = -2;
+    t->uplusmode = UPLUS_NONE;
+    memset(t->token_disabled, 0, sizeof(t->token_disabled));
     return t;
 }
 
@@ -449,7 +546,7 @@ extern void close_tokeniser(struct tokeniser * t) {
     lose_b(t->b2);
     {
         struct m_pair * q = t->m_pairs;
-        until (q == 0) {
+        while (q) {
             struct m_pair * q_next = q->next;
             lose_b(q->name);
             lose_b(q->value);
@@ -459,12 +556,12 @@ extern void close_tokeniser(struct tokeniser * t) {
     }
     {
         struct input * q = t->next;
-        until (q == 0) {
+        while (q) {
             struct input * q_next = q->next;
             FREE(q);
             q = q_next;
         }
     }
-    free(t->file);
+    if (t->file_needs_freeing) free(t->file);
     FREE(t);
 }