summaryrefslogtreecommitdiffstats
path: root/contrib/snowball/compiler
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/snowball/compiler')
-rw-r--r--contrib/snowball/compiler/analyser.c959
-rw-r--r--contrib/snowball/compiler/driver.c257
-rw-r--r--contrib/snowball/compiler/generator.c1465
-rw-r--r--contrib/snowball/compiler/generator_java.c1452
-rw-r--r--contrib/snowball/compiler/header.h324
-rw-r--r--contrib/snowball/compiler/space.c263
-rw-r--r--contrib/snowball/compiler/syswords.h84
-rw-r--r--contrib/snowball/compiler/syswords2.h13
-rw-r--r--contrib/snowball/compiler/tokeniser.c470
9 files changed, 5287 insertions, 0 deletions
diff --git a/contrib/snowball/compiler/analyser.c b/contrib/snowball/compiler/analyser.c
new file mode 100644
index 000000000..68c0d2d90
--- /dev/null
+++ b/contrib/snowball/compiler/analyser.c
@@ -0,0 +1,959 @@
+
+#include <stdio.h> /* printf etc */
+#include <stdlib.h> /* exit */
+#include <string.h> /* memmove */
+#include "header.h"
+
+/* recursive usage: */
+
+static void read_program_(struct analyser * a, int terminator);
+static struct node * read_C(struct analyser * a);
+static struct node * C_style(struct analyser * a, char * s, int token);
+
+
+static void fault(int n) { fprintf(stderr, "fault %d\n", n); exit(1); }
+
+static void print_node_(struct node * p, int n, const char * s) {
+
+ int i;
+ for (i = 0; i < n; i++) fputs(i == n - 1 ? s : " ", stdout);
+ printf("%s ", name_of_token(p->type));
+ unless (p->name == 0) report_b(stdout, p->name->b);
+ unless (p->literalstring == 0) {
+ printf("'");
+ report_b(stdout, p->literalstring);
+ printf("'");
+ }
+ printf("\n");
+ unless (p->AE == 0) print_node_(p->AE, n+1, "# ");
+ unless (p->left == 0) print_node_(p->left, n+1, " ");
+ unless (p->right == 0) print_node_(p->right, n, " ");
+ if (p->aux != 0) print_node_(p->aux, n+1, "@ ");
+}
+
+extern void print_program(struct analyser * a) {
+ print_node_(a->program, 0, " ");
+}
+
+static struct node * new_node(struct analyser * a, int type) {
+ NEW(node, p);
+ p->next = a->nodes; a->nodes = p;
+ p->left = 0;
+ p->right = 0;
+ p->aux = 0;
+ p->AE = 0;
+ p->name = 0;
+ p->literalstring = 0;
+ p->mode = a->mode;
+ p->line_number = a->tokeniser->line_number;
+ p->type = type;
+ return p;
+}
+
+static const char * name_of_mode(int n) {
+ switch (n) {
+ default: fault(0);
+ case m_backward: return "string backward";
+ case m_forward: return "string forward";
+ /* case m_integer: return "integer"; */
+ }
+}
+
+static const char * name_of_type(int n) {
+ switch (n) {
+ default: fault(1);
+ case 's': return "string";
+ case 'i': return "integer";
+ case 'r': return "routine";
+ case 'R': return "routine or grouping";
+ case 'g': return "grouping";
+ }
+}
+
+static void count_error(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ if (t->error_count >= 20) { fprintf(stderr, "... etc\n"); exit(1); }
+ t->error_count++;
+}
+
+static void error2(struct analyser * a, int n, int x) {
+ struct tokeniser * t = a->tokeniser;
+ count_error(a);
+ fprintf(stderr, "%s:%d: ", t->file, t->line_number);
+ if (n >= 30) report_b(stderr, t->b);
+ switch (n) {
+ case 0:
+ fprintf(stderr, "%s omitted", name_of_token(t->omission)); break;
+ case 3:
+ fprintf(stderr, "in among(...), ");
+ case 1:
+ fprintf(stderr, "unexpected %s", name_of_token(t->token));
+ if (t->token == c_number) fprintf(stderr, " %d", t->number);
+ if (t->token == c_name) {
+ fprintf(stderr, " ");
+ report_b(stderr, t->b);
+ } break;
+ case 2:
+ fprintf(stderr, "string omitted"); break;
+
+ case 14:
+ fprintf(stderr, "unresolved substring on line %d", x); break;
+ case 15:
+ fprintf(stderr, "%s not allowed inside reverse(...)", name_of_token(t->token)); break;
+ case 16:
+ fprintf(stderr, "empty grouping"); break;
+ case 17:
+ fprintf(stderr, "backwards used when already in this mode"); break;
+ case 18:
+ fprintf(stderr, "empty among(...)"); break;
+ case 19:
+ fprintf(stderr, "two adjacent bracketed expressions in among(...)"); break;
+ case 20:
+ fprintf(stderr, "substring preceded by another substring on line %d", x); break;
+
+ case 30:
+ fprintf(stderr, " re-declared"); break;
+ case 31:
+ fprintf(stderr, " undeclared"); break;
+ case 32:
+ fprintf(stderr, " declared as %s mode; used as %s mode",
+ name_of_mode(a->mode), name_of_mode(x)); break;
+ case 33:
+ fprintf(stderr, " not of type %s", name_of_type(x)); break;
+ case 34:
+ fprintf(stderr, " not of type string or integer"); break;
+ case 35:
+ fprintf(stderr, " misplaced"); break;
+ case 36:
+ fprintf(stderr, " redefined"); break;
+ case 37:
+ fprintf(stderr, " mis-used as %s mode",
+ name_of_mode(x)); break;
+ default:
+ fprintf(stderr, " error %d", n); break;
+
+ }
+ if (n <= 13 && t->previous_token > 0)
+ fprintf(stderr, " after %s", name_of_token(t->previous_token));
+ fprintf(stderr, "\n");
+}
+
+static void error(struct analyser * a, int n) { error2(a, n, 0); }
+
+static void error3(struct analyser * a, struct node * p, symbol * b) {
+ count_error(a);
+ fprintf(stderr, "%s:%d: among(...) has repeated string '", a->tokeniser->file, p->line_number);
+ report_b(stderr, b);
+ fprintf(stderr, "'\n");
+}
+
+static void error4(struct analyser * a, struct name * q) {
+ count_error(a);
+ report_b(stderr, q->b);
+ fprintf(stderr, " undefined\n");
+}
+
+static void omission_error(struct analyser * a, int n) {
+ a->tokeniser->omission = n;
+ error(a, 0);
+}
+
+static int check_token(struct analyser * a, int code) {
+ struct tokeniser * t = a->tokeniser;
+ if (t->token != code) { omission_error(a, code); return false; }
+ return true;
+}
+
+static int get_token(struct analyser * a, int code) {
+ struct tokeniser * t = a->tokeniser;
+ read_token(t);
+ {
+ int x = check_token(a, code);
+ unless (x) t->token_held = true;
+ return x;
+ }
+}
+
+static struct name * look_for_name(struct analyser * a) {
+ struct name * p = a->names;
+ symbol * q = a->tokeniser->b;
+ repeat {
+ if (p == 0) return 0;
+ { symbol * b = p->b;
+ int n = SIZE(b);
+ if (n == SIZE(q) && memcmp(q, b, n * sizeof(symbol)) == 0) {
+ p->referenced = true;
+ return p;
+ }
+ }
+ p = p->next;
+ }
+}
+
+static struct name * find_name(struct analyser * a) {
+ struct name * p = look_for_name(a);
+ if (p == 0) error(a, 31);
+ return p;
+}
+
+static void check_routine_mode(struct analyser * a, struct name * p, int mode) {
+ if (p->mode < 0) p->mode = mode; else
+ unless (p->mode == mode) error2(a, 37, mode);
+}
+
+static void check_name_type(struct analyser * a, struct name * p, int type) {
+ switch (type) {
+ case 's': if (p->type == t_string) return; break;
+ case 'i': if (p->type == t_integer) return; break;
+ case 'b': if (p->type == t_boolean) return; break;
+ case 'R': if (p->type == t_grouping) return;
+ case 'r': if (p->type == t_routine ||
+ p->type == t_external) return; break;
+ case 'g': if (p->type == t_grouping) return; break;
+ }
+ error2(a, 33, type);
+}
+
+static void read_names(struct analyser * a, int type) {
+ struct tokeniser * t = a->tokeniser;
+ unless (get_token(a, c_bra)) return;
+ repeat {
+ if (read_token(t) != c_name) break;
+ if (look_for_name(a) != 0) error(a, 30); else {
+ NEW(name, p);
+ p->b = copy_b(t->b);
+ p->type = type;
+ p->mode = -1; /* routines, externals */
+ p->count = a->name_count[type];
+ p->referenced = false;
+ p->used = false;
+ p->grouping = 0;
+ p->definition = 0;
+ a->name_count[type] ++;
+ p->next = a->names;
+ a->names = p;
+ }
+ }
+ unless (check_token(a, c_ket)) t->token_held = true;
+}
+
+static symbol * new_literalstring(struct analyser * a) {
+ NEW(literalstring, p);
+ p->b = copy_b(a->tokeniser->b);
+ p->next = a->literalstrings;
+ a->literalstrings = p;
+ return p->b;
+}
+
+static int read_AE_test(struct analyser * a) {
+
+ struct tokeniser * t = a->tokeniser;
+ switch (read_token(t)) {
+ case c_assign: return c_mathassign;
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le: return t->token;
+ default: error(a, 1); t->token_held = true; return c_eq;
+ }
+}
+
+static int binding(int t) {
+ switch (t) {
+ case c_plus: case c_minus: return 1;
+ case c_multiply: case c_divide: return 2;
+ default: return -2;
+ }
+}
+
+static void name_to_node(struct analyser * a, struct node * p, int type) {
+ struct name * q = find_name(a);
+ unless (q == 0) {
+ check_name_type(a, q, type);
+ q->used = true;
+ }
+ p->name = q;
+}
+
+static struct node * read_AE(struct analyser * a, int B) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p;
+ struct node * q;
+ switch (read_token(t)) {
+ case c_minus: /* monadic */
+ p = new_node(a, c_neg);
+ p->right = read_AE(a, 100);
+ break;
+ case c_bra:
+ p = read_AE(a, 0);
+ get_token(a, c_ket);
+ break;
+ case c_name:
+ p = new_node(a, c_name);
+ name_to_node(a, p, 'i');
+ break;
+ case c_maxint:
+ case c_minint:
+ case c_cursor:
+ case c_limit:
+ case c_size:
+ p = new_node(a, t->token);
+ break;
+ case c_number:
+ p = new_node(a, c_number);
+ p->number = t->number;
+ break;
+ case c_sizeof:
+ p = C_style(a, "s", c_sizeof);
+ break;
+ default:
+ error(a, 1);
+ t->token_held = true;
+ return 0;
+ }
+ repeat {
+ int token = read_token(t);
+ int b = binding(token);
+ unless (binding(token) > B) {
+ t->token_held = true;
+ return p;
+ }
+ q = new_node(a, token);
+ q->left = p;
+ q->right = read_AE(a, b);
+ p = q;
+ }
+}
+
+static struct node * read_C_connection(struct analyser * a, struct node * q, int op) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p = new_node(a, op);
+ struct node * p_end = q;
+ p->left = q;
+ repeat {
+ q = read_C(a);
+ p_end->right = q; p_end = q;
+ if (read_token(t) != op) {
+ t->token_held = true;
+ break;
+ }
+ }
+ return p;
+}
+
+static struct node * read_C_list(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p = new_node(a, c_bra);
+ struct node * p_end = 0;
+ repeat {
+ int token = read_token(t);
+ if (token == c_ket) return p;
+ if (token < 0) { omission_error(a, c_ket); return p; }
+ t->token_held = true;
+ {
+ struct node * q = read_C(a);
+ repeat {
+ token = read_token(t);
+ if (token != c_and && token != c_or) {
+ t->token_held = true;
+ break;
+ }
+ q = read_C_connection(a, q, token);
+ }
+ if (p_end == 0) p->left = q; else p_end->right = q;
+ p_end = q;
+ }
+ }
+}
+
+static struct node * C_style(struct analyser * a, char * s, int token) {
+ int i;
+ struct node * p = new_node(a, token);
+ for (i = 0; s[i] != 0; i++) switch(s[i]) {
+ case 'C':
+ p->left = read_C(a); continue;
+ case 'D':
+ p->aux = read_C(a); continue;
+ case 'A':
+ p->AE = read_AE(a, 0); continue;
+ case 'f':
+ get_token(a, c_for); continue;
+ case 'S':
+ {
+ int str_token = read_token(a->tokeniser);
+ if (str_token == c_name) name_to_node(a, p, 's'); else
+ if (str_token == c_literalstring) p->literalstring = new_literalstring(a);
+ else error(a, 2);
+ }
+ continue;
+ case 'b':
+ case 's':
+ case 'i':
+ if (get_token(a, c_name)) name_to_node(a, p, s[i]);
+ continue;
+ }
+ return p;
+}
+
+static struct node * read_literalstring(struct analyser * a) {
+ struct node * p = new_node(a, c_literalstring);
+ p->literalstring = new_literalstring(a);
+ return p;
+}
+
+static void reverse_b(symbol * b) {
+ int i = 0; int j = SIZE(b) - 1;
+ until (i >= j) {
+ int ch1 = b[i]; int ch2 = b[j];
+ b[i++] = ch2; b[j--] = ch1;
+ }
+}
+
+static int compare_amongvec(const void *pv, const void *qv) {
+ const struct amongvec * p = (const struct amongvec*)pv;
+ const struct amongvec * q = (const struct amongvec*)qv;
+ symbol * b_p = p->b; int p_size = p->size;
+ symbol * b_q = q->b; int q_size = q->size;
+ int smaller_size = p_size < q_size ? p_size : q_size;
+ int i;
+ for (i = 0; i < smaller_size; i++)
+ if (b_p[i] != b_q[i]) return b_p[i] - b_q[i];
+ return p_size - q_size;
+}
+
+static void make_among(struct analyser * a, struct node * p, struct node * substring) {
+
+ NEW(among, x);
+ NEWVEC(amongvec, v, p->number);
+ struct node * q = p->left;
+ struct amongvec * w0 = v;
+ struct amongvec * w1 = v;
+ int result = 1;
+
+ int direction = substring != 0 ? substring->mode : p->mode;
+ int backward = direction == m_backward;
+
+ if (a->amongs == 0) a->amongs = x; else a->amongs_end->next = x;
+ a->amongs_end = x;
+ x->next = 0;
+ x->b = v;
+ x->number = a->among_count++;
+ x->starter = 0;
+
+ if (q->type == c_bra) { x->starter = q; q = q->right; }
+
+ until (q == 0) {
+ if (q->type == c_literalstring) {
+ symbol * b = q->literalstring;
+ w1->b = b; /* pointer to case string */
+ w1->p = 0; /* pointer to corresponding case expression */
+ w1->size = SIZE(b); /* number of characters in string */
+ w1->i = -1; /* index of longest substring */
+ w1->result = -1; /* number of corresponding case expression */
+ w1->function = q->left == 0 ? 0 : q->left->name;
+ unless (w1->function == 0)
+ check_routine_mode(a, w1->function, direction);
+ w1++;
+ }
+ else
+ if (q->left == 0) /* empty command: () */
+ w0 = w1;
+ else {
+ until (w0 == w1) {
+ w0->p = q;
+ w0->result = result;
+ w0++;
+ }
+ result++;
+ }
+ q = q->right;
+ }
+ unless (w1-v == p->number) { fprintf(stderr, "oh! %d %d\n", (int)(w1-v), p->number); exit(1); }
+ if (backward) for (w0 = v; w0 < w1; w0++) reverse_b(w0->b);
+ qsort(v, w1 - v, sizeof(struct amongvec), compare_amongvec);
+
+ /* the following loop is O(n squared) */
+ for (w0 = w1 - 1; w0 >= v; w0--) {
+ symbol * b = w0->b;
+ int size = w0->size;
+ struct amongvec * w;
+
+ for (w = w0 - 1; w >= v; w--) {
+ if (w->size < size && memcmp(w->b, b, w->size * sizeof(symbol)) == 0) {
+ w0->i = w - v; /* fill in index of longest substring */
+ break;
+ }
+ }
+ }
+ if (backward) for (w0 = v; w0 < w1; w0++) reverse_b(w0->b);
+
+ for (w0 = v; w0 < w1 - 1; w0++)
+ if (w0->size == (w0 + 1)->size &&
+ memcmp(w0->b, (w0 + 1)->b, w0->size * sizeof(symbol)) == 0) error3(a, p, w0->b);
+
+ x->literalstring_count = p->number;
+ x->command_count = result - 1;
+ p->among = x;
+
+ x->substring = substring;
+ if (substring != 0) substring->among = x;
+ unless (x->command_count == 0 && x->starter == 0) a->amongvar_needed = true;
+}
+
+static struct node * read_among(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ struct node * p = new_node(a, c_among);
+ struct node * p_end = 0;
+ int previous_token = -1;
+ struct node * substring = a->substring;
+
+ a->substring = 0;
+ p->number = 0; /* counts the number of literals */
+ unless (get_token(a, c_bra)) return p;
+ repeat {
+ struct node * q;
+ int token = read_token(t);
+ switch (token) {
+ case c_literalstring:
+ q = read_literalstring(a);
+ if (read_token(t) == c_name) {
+ struct node * r = new_node(a, c_name);
+ name_to_node(a, r, 'r');
+ q->left = r;
+ }
+ else t->token_held = true;
+ p->number++; break;
+ case c_bra:
+ if (previous_token == c_bra) error(a, 19);
+ q = read_C_list(a); break;
+ default:
+ error(a, 3);
+ case c_ket:
+ if (p->number == 0) error(a, 18);
+ if (t->error_count == 0) make_among(a, p, substring);
+ return p;
+ }
+ previous_token = token;
+ if (p_end == 0) p->left = q; else p_end->right = q;
+ p_end = q;
+ }
+}
+
+static struct node * read_substring(struct analyser * a) {
+
+ struct node * p = new_node(a, c_substring);
+ if (a->substring != 0) error2(a, 20, a->substring->line_number);
+ a->substring = p;
+ return p;
+}
+
+static void check_modifyable(struct analyser * a) {
+ unless (a->modifyable) error(a, 15);
+}
+
+static struct node * read_C(struct analyser * a) {
+ struct tokeniser * t = a->tokeniser;
+ int token = read_token(t);
+ switch (token) {
+ case c_bra:
+ return read_C_list(a);
+ case c_backwards:
+ {
+ int mode = a->mode;
+ if (a->mode == m_backward) error(a, 17); else a->mode = m_backward;
+ { struct node * p = C_style(a, "C", token);
+ a->mode = mode;
+ return p;
+ }
+ }
+ case c_reverse:
+ {
+ int mode = a->mode;
+ int modifyable = a->modifyable;
+ a->modifyable = false;
+ a->mode = mode == m_forward ? m_backward : m_forward;
+ {
+ struct node * p = C_style(a, "C", token);
+ a->mode = mode;
+ a->modifyable = modifyable;
+ return p;
+ }
+ }
+ case c_not:
+ case c_try:
+ case c_fail:
+ case c_test:
+ case c_do:
+ case c_goto:
+ case c_gopast:
+ case c_repeat:
+ return C_style(a, "C", token);
+ case c_loop:
+ case c_atleast:
+ return C_style(a, "AC", token);
+ case c_setmark:
+ return C_style(a, "i", token);
+ case c_tomark:
+ case c_atmark:
+ case c_hop:
+ return C_style(a, "A", token);
+ case c_delete:
+ check_modifyable(a);
+ case c_next:
+ case c_tolimit:
+ case c_atlimit:
+ case c_leftslice:
+ case c_rightslice:
+ case c_true:
+ case c_false:
+ case c_debug:
+ return C_style(a, "", token);
+ case c_assignto:
+ case c_sliceto:
+ check_modifyable(a);
+ return C_style(a, "s", token);
+ case c_assign:
+ case c_insert:
+ case c_attach:
+ case c_slicefrom:
+ check_modifyable(a);
+ return C_style(a, "S", token);
+ case c_setlimit:
+ return C_style(a, "CfD", token);
+ case c_set:
+ case c_unset:
+ return C_style(a, "b", token);
+ case c_dollar:
+ get_token(a, c_name);
+ {
+ struct node * p;
+ struct name * q = find_name(a);
+ int mode = a->mode;
+ int modifyable = a->modifyable;
+ switch (q ? q->type : t_string)
+ /* above line was: switch (q->type) - bug #1 fix 7/2/2003 */
+ {
+ default: error(a, 34);
+ case t_string:
+ a->mode = m_forward;
+ a->modifyable = true;
+ p = new_node(a, c_dollar);
+ p->left = read_C(a); break;
+ case t_integer:
+ /* a->mode = m_integer; */
+ p = new_node(a, read_AE_test(a));
+ p->AE = read_AE(a, 0); break;
+ }
+ p->name = q;
+ a->mode = mode;
+ a->modifyable = modifyable;
+ return p;
+ }
+ case c_name:
+ {
+ struct name * q = find_name(a);
+ struct node * p = new_node(a, c_name);
+ unless (q == 0) {
+ q->used = true;
+ switch (q->type) {
+ case t_boolean:
+ p->type = c_booltest; break;
+ case t_integer:
+ error(a, 35); /* integer name misplaced */
+ case t_string:
+ break;
+ case t_routine:
+ case t_external:
+ p->type = c_call;
+ check_routine_mode(a, q, a->mode);
+ break;
+ case t_grouping:
+ p->type = c_grouping; break;
+ }
+ }
+ p->name = q;
+ return p;
+ }
+ case c_non:
+ {
+ struct node * p = new_node(a, token);
+ read_token(t);
+ if (t->token == c_minus) read_token(t);
+ unless (check_token(a, c_name)) { omission_error(a, c_name); return p; }
+ name_to_node(a, p, 'g');
+ return p;
+ }
+ case c_literalstring:
+ return read_literalstring(a);
+ case c_among: return read_among(a);
+ case c_substring: return read_substring(a);
+ default: error(a, 1); return 0;
+ }
+}
+
+static int next_symbol(symbol * p, symbol * W, int utf8) {
+ if (utf8) {
+ int ch;
+ int j = get_utf8(p, & ch);
+ W[0] = ch; return j;
+ } else {
+ W[0] = p[0]; return 1;
+ }
+}
+
+static symbol * alter_grouping(symbol * p, symbol * q, int style, int utf8) {
+ int j = 0;
+ symbol W[1];
+ int width;
+ if (style == c_plus) {
+ while (j < SIZE(q)) {
+ width = next_symbol(q + j, W, utf8);
+ p = add_to_b(p, 1, W);
+ j += width;
+ }
+ } else {
+ while (j < SIZE(q)) {
+ int i;
+ width = next_symbol(q + j, W, utf8);
+ for (i = 0; i < SIZE(p); i++) {
+ if (p[i] == W[0]) {
+ memmove(p + i, p + i + 1, (SIZE(p) - i - 1) * sizeof(symbol));
+ SIZE(p)--;
+ }
+ }
+ j += width;
+ }
+ }
+ return p;
+}
+
+static void read_define_grouping(struct analyser * a, struct name * q) {
+ struct tokeniser * t = a->tokeniser;
+ int style = c_plus;
+ {
+ NEW(grouping, p);
+ if (a->groupings == 0) a->groupings = p; else a->groupings_end->next = p;
+ a->groupings_end = p;
+ q->grouping = p;
+ p->next = 0;
+ p->name = q;
+ p->number = q->count;
+ p->b = create_b(0);
+ repeat {
+ switch (read_token(t)) {
+ case c_name:
+ {
+ struct name * r = find_name(a);
+ unless (r == 0) {
+ check_name_type(a, r, 'g');
+ p->b = alter_grouping(p->b, r->grouping->b, style, false);
+ }
+ }
+ break;
+ case c_literalstring:
+ p->b = alter_grouping(p->b, t->b, style, a->utf8);
+ break;
+ default: error(a, 1); return;
+ }
+ switch (read_token(t)) {
+ case c_plus:
+ case c_minus: style = t->token; break;
+ default: goto label0;
+ }
+ }
+ label0:
+ {
+ int i;
+ int max = 0;
+ int min = 1<<16;
+ for (i = 0; i < SIZE(p->b); i++) {
+ if (p->b[i] > max) max = p->b[i];
+ if (p->b[i] < min) min = p->b[i];
+ }
+ p->largest_ch = max;
+ p->smallest_ch = min;
+ if (min == 1<<16) error(a, 16);
+ }
+ t->token_held = true; return;
+ }
+}
+
+static void read_define_routine(struct analyser * a, struct name * q) {
+ struct node * p = new_node(a, c_define);
+ a->amongvar_needed = false;
+ unless (q == 0) {
+ check_name_type(a, q, 'R');
+ if (q->definition != 0) error(a, 36);
+ if (q->mode < 0) q->mode = a->mode; else
+ if (q->mode != a->mode) error2(a, 32, q->mode);
+ }
+ p->name = q;
+ if (a->program == 0) a->program = p; else a->program_end->right = p;
+ a->program_end = p;
+ get_token(a, c_as);
+ p->left = read_C(a);
+ unless (q == 0) q->definition = p->left;
+
+ if (a->substring != 0) {
+ error2(a, 14, a->substring->line_number);
+ a->substring = 0;
+ }
+ p->amongvar_needed = a->amongvar_needed;
+}
+
+static void read_define(struct analyser * a) {
+ unless (get_token(a, c_name)) return;
+ {
+ struct name * q = find_name(a);
+ if (q != 0 && q->type == t_grouping) read_define_grouping(a, q);
+ else read_define_routine(a, q);
+ }
+}
+
+static void read_backwardmode(struct analyser * a) {
+ int mode = a->mode;
+ a->mode = m_backward;
+ if (get_token(a, c_bra)) {
+ read_program_(a, c_ket);
+ check_token(a, c_ket);
+ }
+ a->mode = mode;
+}
+
+static void read_program_(struct analyser * a, int terminator) {
+ struct tokeniser * t = a->tokeniser;
+ repeat {
+ switch (read_token(t)) {
+ case c_strings: read_names(a, t_string); break;
+ case c_booleans: read_names(a, t_boolean); break;
+ case c_integers: read_names(a, t_integer); break;
+ case c_routines: read_names(a, t_routine); break;
+ case c_externals: read_names(a, t_external); break;
+ case c_groupings: read_names(a, t_grouping); break;
+ case c_define: read_define(a); break;
+ case c_backwardmode:read_backwardmode(a); break;
+ case c_ket:
+ if (terminator == c_ket) return;
+ default:
+ error(a, 1); break;
+ case -1:
+ unless (terminator < 0) omission_error(a, c_ket);
+ return;
+ }
+ }
+}
+
+extern void read_program(struct analyser * a) {
+ read_program_(a, -1);
+ {
+ struct name * q = a->names;
+ until (q == 0) {
+ switch(q->type) {
+ case t_external: case t_routine:
+ if (q->used && q->definition == 0) error4(a, q); break;
+ case t_grouping:
+ if (q->used && q->grouping == 0) error4(a, q); break;
+ }
+ q = q->next;
+ }
+ }
+
+ if (a->tokeniser->error_count == 0) {
+ struct name * q = a->names;
+ int warned = false;
+ until (q == 0) {
+ unless (q->referenced) {
+ unless (warned) {
+ fprintf(stderr, "Declared but not used:");
+ warned = true;
+ }
+ fprintf(stderr, " "); report_b(stderr, q->b);
+ }
+ q = q->next;
+ }
+ if (warned) fprintf(stderr, "\n");
+
+ q = a->names;
+ warned = false;
+ until (q == 0) {
+ if (! q->used && (q->type == t_routine ||
+ q->type == t_grouping)) {
+ unless (warned) {
+ fprintf(stderr, "Declared and defined but not used:");
+ warned = true;
+ }
+ fprintf(stderr, " "); report_b(stderr, q->b);
+ }
+ q = q->next;
+ }
+ if (warned) fprintf(stderr, "\n");
+ }
+}
+
+extern struct analyser * create_analyser(struct tokeniser * t) {
+ NEW(analyser, a);
+ a->tokeniser = t;
+ a->nodes = 0;
+ a->names = 0;
+ a->literalstrings = 0;
+ a->program = 0;
+ a->amongs = 0;
+ a->among_count = 0;
+ a->groupings = 0;
+ a->mode = m_forward;
+ a->modifyable = true;
+ { int i; for (i = 0; i < t_size; i++) a->name_count[i] = 0; }
+ a->substring = 0;
+ return a;
+}
+
+extern void close_analyser(struct analyser * a) {
+ {
+ struct node * q = a->nodes;
+ until (q == 0) {
+ struct node * q_next = q->next;
+ FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct name * q = a->names;
+ until (q == 0) {
+ struct name * q_next = q->next;
+ lose_b(q->b); FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct literalstring * q = a->literalstrings;
+ until (q == 0) {
+ struct literalstring * q_next = q->next;
+ lose_b(q->b); FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct among * q = a->amongs;
+ until (q == 0) {
+ struct among * q_next = q->next;
+ FREE(q->b); FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct grouping * q = a->groupings;
+ until (q == 0) {
+ struct grouping * q_next = q->next;
+ lose_b(q->b); FREE(q);
+ q = q_next;
+ }
+ }
+ FREE(a);
+}
+
diff --git a/contrib/snowball/compiler/driver.c b/contrib/snowball/compiler/driver.c
new file mode 100644
index 000000000..fbb1e9cae
--- /dev/null
+++ b/contrib/snowball/compiler/driver.c
@@ -0,0 +1,257 @@
+#include <stdio.h> /* for fprintf etc */
+#include <stdlib.h> /* for free etc */
+#include <string.h> /* for strlen */
+#include "header.h"
+
+#define DEFAULT_PACKAGE "org.tartarus.snowball.ext"
+#define DEFAULT_BASE_CLASS "org.tartarus.snowball.SnowballProgram"
+#define DEFAULT_AMONG_CLASS "org.tartarus.snowball.Among"
+#define DEFAULT_STRING_CLASS "java.lang.StringBuilder"
+
+static int eq(const char * s1, const char * s2) {
+ int s1_len = strlen(s1);
+ int s2_len = strlen(s2);
+ return s1_len == s2_len && memcmp(s1, s2, s1_len) == 0;
+}
+
+static void print_arglist(void) {
+ fprintf(stderr, "Usage: snowball <file> [options]\n\n"
+ "options are: [-o[utput] file]\n"
+ " [-s[yntax]]\n"
+#ifndef DISABLE_JAVA
+ " [-j[ava]]\n"
+#endif
+ " [-c++]\n"
+ " [-w[idechars]]\n"
+ " [-u[tf8]]\n"
+ " [-n[ame] class name]\n"
+ " [-ep[refix] string]\n"
+ " [-vp[refix] string]\n"
+ " [-i[nclude] directory]\n"
+ " [-r[untime] path to runtime headers]\n"
+#ifndef DISABLE_JAVA
+ " [-p[arentclassname] fully qualified parent class name]\n"
+ " [-P[ackage] package name for stemmers]\n"
+ " [-S[tringclass] StringBuffer-compatible class]\n"
+ " [-a[mongclass] fully qualified name of the Among class]\n"
+#endif
+ );
+ exit(1);
+}
+
+static void check_lim(int i, int argc) {
+ if (i >= argc) {
+ fprintf(stderr, "argument list is one short\n");
+ print_arglist();
+ }
+}
+
+static FILE * get_output(symbol * b) {
+ char * s = b_to_s(b);
+ FILE * output = fopen(s, "w");
+ if (output == 0) {
+ fprintf(stderr, "Can't open output %s\n", s);
+ exit(1);
+ }
+ free(s);
+ return output;
+}
+
+static void read_options(struct options * o, int argc, char * argv[]) {
+ char * s;
+ int i = 2;
+
+ /* set defaults: */
+
+ o->output_file = 0;
+ o->syntax_tree = false;
+ o->externals_prefix = "";
+ o->variables_prefix = 0;
+ o->runtime_path = 0;
+ o->parent_class_name = DEFAULT_BASE_CLASS;
+ o->string_class = DEFAULT_STRING_CLASS;
+ o->among_class = DEFAULT_AMONG_CLASS;
+ o->package = DEFAULT_PACKAGE;
+ o->name = "";
+ o->make_lang = LANG_C;
+ o->widechars = false;
+ o->includes = 0;
+ o->includes_end = 0;
+ o->utf8 = false;
+
+ /* read options: */
+
+ repeat {
+ if (i >= argc) break;
+ s = argv[i++];
+ { if (eq(s, "-o") || eq(s, "-output")) {
+ check_lim(i, argc);
+ o->output_file = argv[i++];
+ continue;
+ }
+ if (eq(s, "-n") || eq(s, "-name")) {
+ check_lim(i, argc);
+ o->name = argv[i++];
+ continue;
+ }
+#ifndef DISABLE_JAVA
+ if (eq(s, "-j") || eq(s, "-java")) {
+ o->make_lang = LANG_JAVA;
+ o->widechars = true;
+ continue;
+ }
+#endif
+ if (eq(s, "-c++")) {
+ o->make_lang = LANG_CPLUSPLUS;
+ continue;
+ }
+ if (eq(s, "-w") || eq(s, "-widechars")) {
+ o->widechars = true;
+ o->utf8 = false;
+ continue;
+ }
+ if (eq(s, "-s") || eq(s, "-syntax")) {
+ o->syntax_tree = true;
+ continue;
+ }
+ if (eq(s, "-ep") || eq(s, "-eprefix")) {
+ check_lim(i, argc);
+ o->externals_prefix = argv[i++];
+ continue;
+ }
+ if (eq(s, "-vp") || eq(s, "-vprefix")) {
+ check_lim(i, argc);
+ o->variables_prefix = argv[i++];
+ continue;
+ }
+ if (eq(s, "-i") || eq(s, "-include")) {
+ check_lim(i, argc);
+
+ {
+ NEW(include, p);
+ symbol * b = add_s_to_b(0, argv[i++]);
+ b = add_s_to_b(b, "/");
+ p->next = 0; p->b = b;
+
+ if (o->includes == 0) o->includes = p; else
+ o->includes_end->next = p;
+ o->includes_end = p;
+ }
+ continue;
+ }
+ if (eq(s, "-r") || eq(s, "-runtime")) {
+ check_lim(i, argc);
+ o->runtime_path = argv[i++];
+ continue;
+ }
+ if (eq(s, "-u") || eq(s, "-utf8")) {
+ o->utf8 = true;
+ o->widechars = false;
+ continue;
+ }
+#ifndef DISABLE_JAVA
+ if (eq(s, "-p") || eq(s, "-parentclassname")) {
+ check_lim(i, argc);
+ o->parent_class_name = argv[i++];
+ continue;
+ }
+ if (eq(s, "-P") || eq(s, "-Package")) {
+ check_lim(i, argc);
+ o->package = argv[i++];
+ continue;
+ }
+ if (eq(s, "-S") || eq(s, "-stringclass")) {
+ check_lim(i, argc);
+ o->string_class = argv[i++];
+ continue;
+ }
+ if (eq(s, "-a") || eq(s, "-amongclass")) {
+ check_lim(i, argc);
+ o->among_class = argv[i++];
+ continue;
+ }
+#endif
+ fprintf(stderr, "'%s' misplaced\n", s);
+ print_arglist();
+ }
+ }
+}
+
+extern int main(int argc, char * argv[]) {
+
+ NEW(options, o);
+ if (argc == 1) print_arglist();
+ read_options(o, argc, argv);
+ {
+ symbol * filename = add_s_to_b(0, argv[1]);
+ char * file;
+ symbol * u = get_input(filename, &file);
+ if (u == 0) {
+ fprintf(stderr, "Can't open input %s\n", argv[1]);
+ exit(1);
+ }
+ {
+ struct tokeniser * t = create_tokeniser(u, file);
+ struct analyser * a = create_analyser(t);
+ t->widechars = o->widechars;
+ t->includes = o->includes;
+ a->utf8 = t->utf8 = o->utf8;
+ read_program(a);
+ if (t->error_count > 0) exit(1);
+ if (o->syntax_tree) print_program(a);
+ close_tokeniser(t);
+ unless (o->syntax_tree) {
+ struct generator * g;
+
+ char * s = o->output_file;
+ unless (s) {
+ fprintf(stderr, "Please include the -o option\n");
+ print_arglist();
+ exit(1);
+ }
+ if (o->make_lang == LANG_C || o->make_lang == LANG_CPLUSPLUS) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".h");
+ o->output_h = get_output(b);
+ b[SIZE(b) - 1] = 'c';
+ if (o->make_lang == LANG_CPLUSPLUS) {
+ b = add_s_to_b(b, "c");
+ }
+ o->output_c = get_output(b);
+ lose_b(b);
+
+ g = create_generator_c(a, o);
+ generate_program_c(g);
+ close_generator_c(g);
+ fclose(o->output_c);
+ fclose(o->output_h);
+ }
+#ifndef DISABLE_JAVA
+ if (o->make_lang == LANG_JAVA) {
+ symbol * b = add_s_to_b(0, s);
+ b = add_s_to_b(b, ".java");
+ o->output_java = get_output(b);
+ lose_b(b);
+ g = create_generator_java(a, o);
+ generate_program_java(g);
+ close_generator_java(g);
+ fclose(o->output_java);
+ }
+#endif
+ }
+ close_analyser(a);
+ }
+ lose_b(u);
+ lose_b(filename);
+ }
+ { struct include * p = o->includes;
+ until (p == 0)
+ { struct include * q = p->next;
+ lose_b(p->b); FREE(p); p = q;
+ }
+ }
+ FREE(o);
+ unless (space_count == 0) fprintf(stderr, "%d blocks unfreed\n", space_count);
+ return 0;
+}
+
diff --git a/contrib/snowball/compiler/generator.c b/contrib/snowball/compiler/generator.c
new file mode 100644
index 000000000..7294b0471
--- /dev/null
+++ b/contrib/snowball/compiler/generator.c
@@ -0,0 +1,1465 @@
+
+#include <limits.h> /* for INT_MAX */
+#include <stdio.h> /* for fprintf etc */
+#include <stdlib.h> /* for free etc */
+#include <string.h> /* for strlen */
+#include "header.h"
+
+/* Define this to get warning messages when optimisations can't be used. */
+/* #define OPTIMISATION_WARNINGS */
+
+/* recursive use: */
+
+static void generate(struct generator * g, struct node * p);
+
+enum special_labels {
+
+ x_return = -1
+
+};
+
+static int new_label(struct generator * g) {
+ return g->next_label++;
+}
+
+/* Output routines */
+static void output_str(FILE * outfile, struct str * str) {
+
+ char * s = b_to_s(str_data(str));
+ fprintf(outfile, "%s", s);
+ free(s);
+}
+
+static void wch(struct generator * g, int ch) {
+ str_append_ch(g->outbuf, ch); /* character */
+}
+
+static void wnl(struct generator * g) {
+ str_append_ch(g->outbuf, '\n'); /* newline */
+ g->line_count++;
+}
+
+static void ws(struct generator * g, const char * s) {
+ str_append_string(g->outbuf, s); /* string */
+}
+
+static void wi(struct generator * g, int i) {
+ str_append_int(g->outbuf, i); /* integer */
+}
+
+static void wh_ch(struct generator * g, int i) {
+ str_append_ch(g->outbuf, "0123456789ABCDEF"[i & 0xF]); /* hexchar */
+}
+
+static void wh(struct generator * g, int i) {
+ if (i >> 4) wh(g, i >> 4);
+ wh_ch(g, i); /* hex integer */
+}
+
+static void wi3(struct generator * g, int i) {
+ if (i < 100) wch(g, ' ');
+ if (i < 10) wch(g, ' ');
+ wi(g, i); /* integer (width 3) */
+}
+
+static void wvn(struct generator * g, struct name * p) { /* variable name */
+
+ int ch = "SBIrxg"[p->type];
+ switch (p->type) {
+ case t_string:
+ case t_boolean:
+ case t_integer:
+ wch(g, ch); wch(g, '['); wi(g, p->count); wch(g, ']'); return;
+ case t_external:
+ ws(g, g->options->externals_prefix); break;
+ default:
+ wch(g, ch); wch(g, '_');
+ }
+ str_append_b(g->outbuf, p->b);
+}
+
+static void wv(struct generator * g, struct name * p) { /* reference to variable */
+ if (p->type < t_routine) ws(g, "z->");
+ wvn(g, p);
+}
+
+static void wlitarray(struct generator * g, symbol * p) { /* write literal array */
+
+ ws(g, "{ ");
+ {
+ int i;
+ for (i = 0; i < SIZE(p); i++) {
+ int ch = p[i];
+ if (32 <= ch && ch < 127) {
+ wch(g, '\'');
+ switch (ch) {
+ case '\'':
+ case '\\': wch(g, '\\');
+ default: wch(g, ch);
+ }
+ wch(g, '\'');
+ } else {
+ wch(g, '0'); wch(g, 'x'); wh(g, ch);
+ }
+ if (i < SIZE(p) - 1) ws(g, ", ");
+ }
+ }
+ ws(g, " }");
+}
+
+static void wlitref(struct generator * g, symbol * p) { /* write ref to literal array */
+
+ if (SIZE(p) == 0) ws(g, "0"); else {
+ struct str * s = g->outbuf;
+ g->outbuf = g->declarations;
+ ws(g, "static const symbol s_"); wi(g, g->literalstring_count); ws(g, "[] = ");
+ wlitarray(g, p);
+ ws(g, ";\n");
+ g->outbuf = s;
+ ws(g, "s_"); wi(g, g->literalstring_count);
+ g->literalstring_count++;
+ }
+}
+
+
+static void wm(struct generator * g) { /* margin */
+ int i;
+ for (i = 0; i < g->margin; i++) ws(g, " ");
+}
+
+static void wc(struct generator * g, struct node * p) { /* comment */
+
+ ws(g, " /* ");
+ switch (p->type) {
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ if (p->name) {
+ wch(g, '$');
+ str_append_b(g->outbuf, p->name->b);
+ wch(g, ' ');
+ }
+ ws(g, name_of_token(p->type));
+ ws(g, " <integer expression>");
+ break;
+ default:
+ ws(g, name_of_token(p->type));
+ if (p->name) {
+ wch(g, ' ');
+ str_append_b(g->outbuf, p->name->b);
+ }
+ }
+ ws(g, ", line "); wi(g, p->line_number); ws(g, " */");
+ wnl(g);
+}
+
+static void wms(struct generator * g, const char * s) {
+ wm(g); ws(g, s); } /* margin + string */
+
+static void wbs(struct generator * g) { /* block start */
+ wms(g, "{ ");
+ g->margin++;
+}
+
+static void wbe(struct generator * g) { /* block end */
+
+ if (g->line_labelled == g->line_count) { wms(g, ";"); wnl(g); }
+ g->margin--;
+ wms(g, "}"); wnl(g);
+}
+
+static void wk(struct generator * g, struct node * p) { /* keep c */
+ ++g->keep_count;
+ if (p->mode == m_forward) {
+ ws(g, "int c"); wi(g, g->keep_count); ws(g, " = z->c;");
+ } else {
+ ws(g, "int m"); wi(g, g->keep_count); ws(g, " = z->l - z->c; (void)m");
+ wi(g, g->keep_count); ws(g, ";");
+ }
+}
+
+static void wrestore(struct generator * g, struct node * p, int keep_token) { /* restore c */
+ if (p->mode == m_forward) {
+ ws(g, "z->c = c");
+ } else {
+ ws(g, "z->c = z->l - m");
+ }
+ wi(g, keep_token); ws(g, ";");
+}
+
+static void winc(struct generator * g, struct node * p) { /* increment c */
+ ws(g, p->mode == m_forward ? "z->c++;" :
+ "z->c--;");
+}
+
+static void wsetl(struct generator * g, int n) {
+
+ g->margin--;
+ wms(g, "lab"); wi(g, n); wch(g, ':'); wnl(g);
+ g->line_labelled = g->line_count;
+ g->margin++;
+}
+
+static void wgotol(struct generator * g, int n) {
+ wms(g, "goto lab"); wi(g, n); wch(g, ';'); wnl(g);
+}
+
+static void wf(struct generator * g) { /* fail */
+ if (g->failure_string != 0) { ws(g, "{ "); ws(g, g->failure_string); wch(g, ' '); }
+ switch (g->failure_label)
+ {
+ case x_return:
+ ws(g, "return 0;");
+ break;
+ default:
+ ws(g, "goto lab");
+ wi(g, g->failure_label);
+ wch(g, ';');
+ g->label_used = 1;
+ }
+ if (g->failure_string != 0) ws(g, " }");
+}
+
+static void wlim(struct generator * g, struct node * p) { /* if at limit fail */
+
+ ws(g, p->mode == m_forward ? "if (z->c >= z->l) " :
+ "if (z->c <= z->lb) ");
+ wf(g);
+}
+
+static void wp(struct generator * g, const char * s, struct node * p) { /* formatted write */
+ int i = 0;
+ int l = strlen(s);
+ until (i >= l) {
+ int ch = s[i++];
+ if (ch != '~') wch(g, ch); else
+ switch(s[i++]) {
+ default: wch(g, s[i - 1]); continue;
+ case 'C': wc(g, p); continue;
+ case 'k': wk(g, p); continue;
+ case 'K': /* keep for c_test */
+ ws(g, p->mode == m_forward ? "int c_test = z->c;" :
+ "int m_test = z->l - z->c;");
+ continue;
+ case 'R': /* restore for c_test */
+ ws(g, p->mode == m_forward ? "z->c = c_test;" :
+ "z->c = z->l - m_test;");
+ continue;
+ case 'i': winc(g, p); continue;
+ case 'l': wlim(g, p); continue;
+ case 'f': wf(g); continue;
+ case 'M': wm(g); continue;
+ case 'N': wnl(g); continue;
+ case '{': wbs(g); continue;
+ case '}': wbe(g); continue;
+ case 'S': ws(g, g->S[s[i++] - '0']); continue;
+ case 'I': wi(g, g->I[s[i++] - '0']); continue;
+ case 'J': wi3(g, g->I[s[i++] - '0']); continue;
+ case 'V': wv(g, g->V[s[i++] - '0']); continue;
+ case 'W': wvn(g, g->V[s[i++] - '0']); continue;
+ case 'L': wlitref(g, g->L[s[i++] - '0']); continue;
+ case 'A': wlitarray(g, g->L[s[i++] - '0']); continue;
+ case '+': g->margin++; continue;
+ case '-': g->margin--; continue;
+ case '$': /* insert_s, insert_v etc */
+ wch(g, p->literalstring == 0 ? 'v' : 's');
+ continue;
+ case 'p': ws(g, g->options->externals_prefix); continue;
+ }
+ }
+}
+
+static void w(struct generator * g, const char * s) { wp(g, s, 0); }
+
+static void generate_AE(struct generator * g, struct node * p) {
+ char * s;
+ switch (p->type) {
+ case c_name:
+ wv(g, p->name); break;
+ case c_number:
+ wi(g, p->number); break;
+ case c_maxint:
+ ws(g, "MAXINT"); break;
+ case c_minint:
+ ws(g, "MININT"); break;
+ case c_neg:
+ wch(g, '-'); generate_AE(g, p->right); break;
+ case c_multiply:
+ s = " * "; goto label0;
+ case c_plus:
+ s = " + "; goto label0;
+ case c_minus:
+ s = " - "; goto label0;
+ case c_divide:
+ s = " / ";
+ label0:
+ wch(g, '('); generate_AE(g, p->left);
+ ws(g, s); generate_AE(g, p->right); wch(g, ')'); break;
+ case c_sizeof:
+ g->V[0] = p->name;
+ w(g, "SIZE(~V0)"); break;
+ case c_cursor:
+ w(g, "z->c"); break;
+ case c_limit:
+ w(g, p->mode == m_forward ? "z->l" : "z->lb"); break;
+ case c_size:
+ w(g, "SIZE(z->p)"); break;
+ }
+}
+
+/* K_needed() tests to see if we really need to keep c. Not true when the
+ the command does not touch the cursor. This and repeat_score() could be
+ elaborated almost indefinitely.
+*/
+
+static int K_needed(struct generator * g, struct node * p) {
+ until (p == 0) {
+ switch (p->type) {
+ case c_dollar:
+ case c_leftslice:
+ case c_rightslice:
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ case c_sliceto:
+ case c_true:
+ case c_false:
+ case c_debug:
+ break;
+
+ case c_call:
+ if (K_needed(g, p->name->definition)) return true;
+ break;
+
+ case c_bra:
+ if (K_needed(g, p->left)) return true;
+ break;
+
+ default: return true;
+ }
+ p = p->right;
+ }
+ return false;
+}
+
+static int repeat_score(struct generator * g, struct node * p) {
+ int score = 0;
+ until (p == 0)
+ {
+ switch (p->type) {
+ case c_dollar:
+ case c_leftslice:
+ case c_rightslice:
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ case c_sliceto: /* case c_not: must not be included here! */
+ case c_debug:
+ break;
+
+ case c_call:
+ score += repeat_score(g, p->name->definition);
+ break;
+
+ case c_bra:
+ score += repeat_score(g, p->left);
+ break;
+
+ case c_name:
+ case c_literalstring:
+ case c_next:
+ case c_grouping:
+ case c_non:
+ case c_hop:
+ score = score + 1; break;
+
+ default: score = 2; break;
+ }
+ p = p->right;
+ }
+ return score;
+}
+
+/* tests if an expression requires cursor reinstatement in a repeat */
+
+static int repeat_restore(struct generator * g, struct node * p) {
+ return repeat_score(g, p) >= 2;
+}
+
+static void generate_bra(struct generator * g, struct node * p) {
+ p = p->left;
+ until (p == 0) { generate(g, p); p = p->right; }
+}
+
+static void generate_and(struct generator * g, struct node * p) {
+ int keep_c = 0;
+ if (K_needed(g, p->left)) {
+ wp(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ wp(g, "~M~C", p);
+ }
+ p = p->left;
+ until (p == 0) {
+ generate(g, p);
+ if (keep_c && p->right != 0) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+ p = p->right;
+ }
+ if (keep_c) w(g, "~}");
+}
+
+static void generate_or(struct generator * g, struct node * p) {
+ int keep_c = 0;
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ const char * a1 = g->failure_string;
+
+ int out_lab = new_label(g);
+
+ if (K_needed(g, p->left)) {
+ wp(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ wp(g, "~M~C", p);
+ }
+ p = p->left;
+ g->failure_string = 0;
+ until (p->right == 0) {
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ generate(g, p);
+ wgotol(g, out_lab);
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+ p = p->right;
+ }
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_string = a1;
+
+ generate(g, p);
+ if (keep_c) w(g, "~}");
+ wsetl(g, out_lab);
+}
+
+static void generate_backwards(struct generator * g, struct node * p) {
+
+ wp(g,"~Mz->lb = z->c; z->c = z->l;~C~N", p);
+ generate(g, p->left);
+ w(g, "~Mz->c = z->lb;~N");
+}
+
+
+static void generate_not(struct generator * g, struct node * p) {
+ int keep_c = 0;
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ const char * a1 = g->failure_string;
+
+ if (K_needed(g, p->left)) {
+ wp(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ wp(g, "~M~C", p);
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_string = 0;
+ generate(g, p->left);
+
+ {
+ int l = g->failure_label;
+ int u = g->label_used;
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_string = a1;
+
+ w(g, "~M~f~N");
+ if (u)
+ wsetl(g, l);
+ }
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N~}");
+ }
+}
+
+
+static void generate_try(struct generator * g, struct node * p) {
+ int keep_c = K_needed(g, p->left);
+
+ if (keep_c) {
+ if (p->mode == m_forward) {
+ wp(g, "~{int c_keep = z->c;~C", p);
+ g->failure_string = "z->c = c_keep;";
+ } else {
+ wp(g, "~{int m_keep = z->l - z->c;/* (void) m_keep;*/~C", p);
+ g->failure_string = "z->c = z->l - m_keep;";
+ }
+ } else {
+ wp(g, "~M~C", p);
+ g->failure_string = 0;
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ generate(g, p->left);
+
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+
+ if (keep_c) w(g, "~}");
+}
+
+static void generate_set(struct generator * g, struct node * p) {
+ g->V[0] = p->name; wp(g, "~M~V0 = 1;~C", p);
+}
+
+static void generate_unset(struct generator * g, struct node * p) {
+ g->V[0] = p->name; wp(g, "~M~V0 = 0;~C", p);
+}
+
+static void generate_fail(struct generator * g, struct node * p) {
+ generate(g, p->left);
+ wp(g, "~M~f~C", p);
+}
+
+/* generate_test() also implements 'reverse' */
+
+static void generate_test(struct generator * g, struct node * p) {
+ int keep_c = K_needed(g, p->left);
+ if (keep_c) wp(g, "~{~K~C", p);
+ else wp(g, "~M~C", p);
+
+ generate(g, p->left);
+
+ if (keep_c) wp(g, "~M~R~N"
+ "~}", p);
+}
+
+static void generate_do(struct generator * g, struct node * p) {
+ int keep_c = 0;
+ if (K_needed(g, p->left)) {
+ wp(g, "~{~k~C", p);
+ keep_c = g->keep_count;
+ } else {
+ wp(g, "~M~C", p);
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_string = 0;
+ generate(g, p->left);
+
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c);
+ w(g, "~N~}");
+ }
+}
+
+static void generate_next(struct generator * g, struct node * p) {
+ if (g->options->utf8) {
+ if (p->mode == m_forward)
+ w(g, "~{int ret = skip_utf8(z->p, z->c, 0, z->l, 1");
+ else
+ w(g, "~{int ret = skip_utf8(z->p, z->c, z->lb, 0, -1");
+ wp(g, ");~N"
+ "~Mif (ret < 0) ~f~N"
+ "~Mz->c = ret;~C"
+ "~}", p);
+ } else
+ wp(g, "~M~l~N"
+ "~M~i~C", p);
+}
+
+static void generate_GO_grouping(struct generator * g, struct node * p, int is_goto, int complement) {
+
+ struct grouping * q = p->name->grouping;
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->S[1] = complement ? "in" : "out";
+ g->S[2] = g->options->utf8 ? "_U" : "";
+ g->V[0] = p->name;
+ g->I[0] = q->smallest_ch;
+ g->I[1] = q->largest_ch;
+ if (is_goto) {
+ wp(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1) < 0) ~f /* goto */~C", p);
+ } else {
+ wp(g, "~{ /* gopast */~C"
+ "~Mint ret = ~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1);~N"
+ "~Mif (ret < 0) ~f~N", p);
+ if (p->mode == m_forward)
+ w(g, "~Mz->c += ret;~N");
+ else
+ w(g, "~Mz->c -= ret;~N");
+ w(g, "~}");
+ }
+}
+
+static void generate_GO(struct generator * g, struct node * p, int style) {
+ int keep_c = 0;
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ const char * a1 = g->failure_string;
+
+ if (p->left->type == c_grouping || p->left->type == c_non) {
+ /* Special case for "goto" or "gopast" when used on a grouping or an
+ * inverted grouping - the movement of c by the matching action is
+ * exactly what we want! */
+#ifdef OPTIMISATION_WARNINGS
+ printf("Optimising %s %s\n", style ? "goto" : "gopast", p->left->type == c_non ? "non" : "grouping");
+#endif
+ generate_GO_grouping(g, p->left, style, p->left->type == c_non);
+ return;
+ }
+
+ w(g, "~Mwhile(1) {"); wp(g, "~C~+", p);
+
+ if (style == 1 || repeat_restore(g, p->left)) {
+ wp(g, "~M~k~N", p);
+ keep_c = g->keep_count;
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ generate(g, p->left);
+
+ if (style == 1) {
+ /* include for goto; omit for gopast */
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+ w(g, "~Mbreak;~N");
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_string = a1;
+
+/* wp(g, "~M~l~N"
+ "~M~i~N", p); */
+ generate_next(g, p);
+ w(g, "~}");
+}
+
+static void generate_loop(struct generator * g, struct node * p) {
+ w(g, "~{int i; for (i = "); generate_AE(g, p->AE); wp(g, "; i > 0; i--)~C"
+ "~{", p);
+
+ generate(g, p->left);
+
+ w(g, "~}"
+ "~}");
+}
+
+static void generate_repeat(struct generator * g, struct node * p, int atleast_case) {
+ int keep_c = 0;
+ wp(g, "~Mwhile(1) {~C~+", p);
+
+ if (repeat_restore(g, p->left)) {
+ wp(g, "~M~k~N", p);
+ keep_c = g->keep_count;
+ }
+
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_string = 0;
+ generate(g, p->left);
+
+ if (atleast_case) w(g, "~Mi--;~N");
+
+ w(g, "~Mcontinue;~N");
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+
+ if (keep_c) {
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ }
+
+ w(g, "~Mbreak;~N"
+ "~}");
+}
+
+static void generate_atleast(struct generator * g, struct node * p) {
+ w(g, "~{int i = "); generate_AE(g, p->AE); w(g, ";~N");
+ {
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ const char * a1 = g->failure_string;
+
+ generate_repeat(g, p, true);
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_string = a1;
+ }
+ w(g, "~Mif (i > 0) ~f~N"
+ "~}");
+}
+
+static void generate_setmark(struct generator * g, struct node * p) {
+ g->V[0] = p->name;
+ wp(g, "~M~V0 = z->c;~C", p);
+}
+
+static void generate_tomark(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? ">" : "<";
+
+ w(g, "~Mif (z->c ~S0 "); generate_AE(g, p->AE); w(g, ") ~f~N");
+ w(g, "~Mz->c = "); generate_AE(g, p->AE); wp(g, ";~C", p);
+}
+
+static void generate_atmark(struct generator * g, struct node * p) {
+
+ w(g, "~Mif (z->c != "); generate_AE(g, p->AE); wp(g, ") ~f~C", p);
+}
+
+static void generate_hop(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "+" : "-";
+ g->S[1] = p->mode == m_forward ? "0" : "z->lb";
+ if (g->options->utf8) {
+ w(g, "~{int ret = skip_utf8(z->p, z->c, ~S1, z->l, ~S0 ");
+ generate_AE(g, p->AE); wp(g, ");~C", p);
+ w(g, "~Mif (ret < 0) ~f~N");
+ } else {
+ w(g, "~{int ret = z->c ~S0 ");
+ generate_AE(g, p->AE); wp(g, ";~C", p);
+ w(g, "~Mif (~S1 > ret || ret > z->l) ~f~N");
+ }
+ wp(g, "~Mz->c = ret;~C"
+ "~}", p);
+}
+
+static void generate_delete(struct generator * g, struct node * p) {
+ wp(g, "~{int ret = slice_del(z);~C", p);
+ wp(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+static void generate_tolimit(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "" : "b";
+ wp(g, "~Mz->c = z->l~S0;~C", p);
+}
+
+static void generate_atlimit(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "" : "b";
+ g->S[1] = p->mode == m_forward ? "<" : ">";
+ wp(g, "~Mif (z->c ~S1 z->l~S0) ~f~C", p);
+}
+
+static void generate_leftslice(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "bra" : "ket";
+ wp(g, "~Mz->~S0 = z->c;~C", p);
+}
+
+static void generate_rightslice(struct generator * g, struct node * p) {
+ g->S[0] = p->mode == m_forward ? "ket" : "bra";
+ wp(g, "~Mz->~S0 = z->c;~C", p);
+}
+
+static void generate_assignto(struct generator * g, struct node * p) {
+ g->V[0] = p->name;
+ wp(g, "~M~V0 = assign_to(z, ~V0);~C"
+ "~Mif (~V0 == 0) return -1;~C", p);
+}
+
+static void generate_sliceto(struct generator * g, struct node * p) {
+ g->V[0] = p->name;
+ wp(g, "~M~V0 = slice_to(z, ~V0);~C"
+ "~Mif (~V0 == 0) return -1;~C", p);
+}
+
+static void generate_data_address(struct generator * g, struct node * p) {
+
+ symbol * b = p->literalstring;
+ if (b != 0) {
+ wi(g, SIZE(b)); w(g, ", ");
+ wlitref(g, b);
+ } else
+ wv(g, p->name);
+}
+
+static void generate_insert(struct generator * g, struct node * p, int style) {
+
+ int keep_c = style == c_attach;
+ if (p->mode == m_backward) keep_c = !keep_c;
+ wp(g, "~{", p);
+ if (keep_c) w(g, "int c_keep = z->c;~N~M");
+ wp(g, "int ret = insert_~$(z, z->c, z->c, ", p);
+ generate_data_address(g, p);
+ wp(g, ");~C", p);
+ if (keep_c) w(g, "~Mz->c = c_keep;~N");
+ wp(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+static void generate_assignfrom(struct generator * g, struct node * p) {
+
+ int keep_c = p->mode == m_forward; /* like 'attach' */
+ wp(g, "~{", p);
+ if (keep_c) wp(g, "int c_keep = z->c;~N"
+ "~Mret = insert_~$(z, z->c, z->l, ", p);
+ else wp(g, "ret = insert_~$(z, z->lb, z->c, ", p);
+ generate_data_address(g, p);
+ wp(g, ");~C", p);
+ if (keep_c) w(g, "~Mz->c = c_keep;~N");
+ wp(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+/* bugs marked <======= fixed 22/7/02. Similar fixes required for Java */
+
+static void generate_slicefrom(struct generator * g, struct node * p) {
+
+/* w(g, "~Mslice_from_s(z, "); <============= bug! should be: */
+ wp(g, "~{int ret = slice_from_~$(z, ", p);
+ generate_data_address(g, p);
+ wp(g, ");~C", p);
+ wp(g, "~Mif (ret < 0) return ret;~N"
+ "~}", p);
+}
+
+static void generate_setlimit(struct generator * g, struct node * p) {
+ int keep_c;
+ wp(g, "~{int mlimit;~C"
+ "~M~k~N"
+ , p);
+ keep_c = g->keep_count;
+ generate(g, p->left);
+ if (p->mode == m_forward) w(g, "~Mmlimit = z->l - z->c; z->l = z->c;~N");
+ else w(g, "~Mmlimit = z->lb; z->lb = z->c;~N");
+ w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N");
+ g->failure_string = p->mode == m_forward ? "z->l += mlimit;" :
+ "z->lb = mlimit;";
+ generate(g, p->aux);
+ wms(g, g->failure_string);
+ w(g, "~N"
+ "~}");
+}
+
+static void generate_dollar(struct generator * g, struct node * p) {
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ const char * a1 = g->failure_string;
+ g->failure_label = new_label(g);
+ g->label_used = 0;
+ g->failure_string = 0;
+
+ g->V[0] = p->name;
+ wp(g, "~{struct SN_env env = * z;~C"
+ "~Mint failure = 1; /* assume failure */~N"
+ "~Mz->p = ~V0;~N"
+ "~Mz->lb = z->c = 0;~N"
+ "~Mz->l = SIZE(z->p);~N", p);
+ generate(g, p->left);
+ w(g, "~Mfailure = 0; /* mark success */~N");
+ if (g->label_used)
+ wsetl(g, g->failure_label);
+ g->V[0] = p->name; /* necessary */
+
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_string = a1;
+
+ w(g, "~M~V0 = z->p;~N"
+ "~M* z = env;~N"
+ "~Mif (failure) ~f~N~}");
+}
+
+static void generate_integer_assign(struct generator * g, struct node * p, char * s) {
+
+ g->V[0] = p->name;
+ g->S[0] = s;
+ w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); wp(g, ";~C", p);
+}
+
+static void generate_integer_test(struct generator * g, struct node * p, char * s) {
+
+ g->V[0] = p->name;
+ g->S[0] = s;
+ w(g, "~Mif (!(~V0 ~S0 "); generate_AE(g, p->AE); wp(g, ")) ~f~C", p);
+}
+
+static void generate_call(struct generator * g, struct node * p) {
+
+ g->V[0] = p->name;
+ wp(g, "~{int ret = ~V0(z);~C"
+ "~Mif (ret == 0) ~f~N"
+ "~Mif (ret < 0) return ret;~N~}", p);
+}
+
+static void generate_grouping(struct generator * g, struct node * p, int complement) {
+
+ struct grouping * q = p->name->grouping;
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->S[1] = complement ? "out" : "in";
+ g->S[2] = g->options->utf8 ? "_U" : "";
+ g->V[0] = p->name;
+ g->I[0] = q->smallest_ch;
+ g->I[1] = q->largest_ch;
+ wp(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 0)) ~f~C", p);
+}
+
+static void generate_namedstring(struct generator * g, struct node * p) {
+
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->V[0] = p->name;
+ wp(g, "~Mif (!(eq_v~S0(z, ~V0))) ~f~C", p);
+}
+
+static void generate_literalstring(struct generator * g, struct node * p) {
+ symbol * b = p->literalstring;
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->I[0] = SIZE(b);
+ g->L[0] = b;
+
+ wp(g, "~Mif (!(eq_s~S0(z, ~I0, ~L0))) ~f~C", p);
+}
+
+static void generate_define(struct generator * g, struct node * p) {
+ struct name * q = p->name;
+ g->next_label = 0;
+
+ g->S[0] = q->type == t_routine ? "static" : "extern";
+ g->V[0] = q;
+
+ w(g, "~N~S0 int ~V0(struct SN_env * z) {~N~+");
+ if (p->amongvar_needed) w(g, "~Mint among_var;~N");
+ g->failure_string = 0;
+ g->failure_label = x_return;
+ g->label_used = 0;
+ g->keep_count = 0;
+ generate(g, p->left);
+ w(g, "~Mreturn 1;~N~}");
+}
+
+static void generate_substring(struct generator * g, struct node * p) {
+
+ struct among * x = p->among;
+ int block = -1;
+ unsigned int bitmap = 0;
+ struct amongvec * among_cases = x->b;
+ int c;
+ int empty_case = -1;
+ int n_cases = 0;
+ symbol cases[2];
+ int shortest_size = INT_MAX;
+
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->I[0] = x->number;
+ g->I[1] = x->literalstring_count;
+
+ /* In forward mode with non-ASCII UTF-8 characters, the first character
+ * of the string will often be the same, so instead look at the last
+ * common character position.
+ *
+ * In backward mode, we can't match if there are fewer characters before
+ * the current position than the minimum length.
+ */
+ for (c = 0; c < x->literalstring_count; ++c) {
+ int size = among_cases[c].size;
+ if (size != 0 && size < shortest_size) {
+ shortest_size = size;
+ }
+ }
+
+ for (c = 0; c < x->literalstring_count; ++c) {
+ symbol ch;
+ if (among_cases[c].size == 0) {
+ empty_case = c;
+ continue;
+ }
+ if (p->mode == m_forward) {
+ ch = among_cases[c].b[shortest_size - 1];
+ } else {
+ ch = among_cases[c].b[among_cases[c].size - 1];
+ }
+ if (n_cases == 0) {
+ block = ch >> 5;
+ } else if (ch >> 5 != block) {
+ block = -1;
+ if (n_cases > 2) break;
+ }
+ if (block == -1) {
+ if (ch == cases[0]) continue;
+ if (n_cases < 2) {
+ cases[n_cases++] = ch;
+ } else if (ch != cases[1]) {
+ ++n_cases;
+ break;
+ }
+ } else {
+ if ((bitmap & (1u << (ch & 0x1f))) == 0) {
+ bitmap |= 1u << (ch & 0x1f);
+ if (n_cases < 2)
+ cases[n_cases] = ch;
+ ++n_cases;
+ }
+ }
+ }
+
+ if (block != -1 || n_cases <= 2) {
+ char buf[64];
+ g->I[2] = block;
+ g->I[3] = bitmap;
+ g->I[4] = shortest_size - 1;
+ if (p->mode == m_forward) {
+ sprintf(buf, "z->p[z->c + %d]", shortest_size - 1);
+ g->S[1] = buf;
+ if (shortest_size == 1) {
+ wp(g, "~Mif (z->c >= z->l || ", p);
+ } else {
+ wp(g, "~Mif (z->c + ~I4 >= z->l || ", p);
+ }
+ } else {
+ g->S[1] = "z->p[z->c - 1]";
+ if (shortest_size == 1) {
+ wp(g, "~Mif (z->c <= z->lb || ", p);
+ } else {
+ wp(g, "~Mif (z->c - ~I4 <= z->lb || ", p);
+ }
+ }
+ if (n_cases == 0) {
+ /* We get this for the degenerate case: among { '' }
+ * This doesn't seem to be a useful construct, but it is
+ * syntactically valid.
+ */
+ wp(g, "0", p);
+ } else if (n_cases == 1) {
+ g->I[4] = cases[0];
+ wp(g, "~S1 != ~I4", p);
+ } else if (n_cases == 2) {
+ g->I[4] = cases[0];
+ g->I[5] = cases[1];
+ wp(g, "(~S1 != ~I4 && ~S1 != ~I5)", p);
+ } else {
+ wp(g, "~S1 >> 5 != ~I2 || !((~I3 >> (~S1 & 0x1f)) & 1)", p);
+ }
+ ws(g, ") ");
+ if (empty_case != -1) {
+ /* If the among includes the empty string, it can never fail
+ * so not matching the bitmap means we match the empty string.
+ */
+ g->I[4] = among_cases[empty_case].result;
+ wp(g, "among_var = ~I4; else~C", p);
+ } else {
+ wp(g, "~f~C", p);
+ }
+ } else {
+#ifdef OPTIMISATION_WARNINGS
+ printf("Couldn't shortcut among %d\n", x->number);
+#endif
+ }
+
+ if (x->command_count == 0 && x->starter == 0)
+ wp(g, "~Mif (!(find_among~S0(z, a_~I0, ~I1))) ~f~C", p);
+ else
+ wp(g, "~Mamong_var = find_among~S0(z, a_~I0, ~I1);~C"
+ "~Mif (!(among_var)) ~f~N", p);
+}
+
+static void generate_among(struct generator * g, struct node * p) {
+
+ struct among * x = p->among;
+ int case_number = 1;
+
+ if (x->substring == 0) generate_substring(g, p);
+ if (x->command_count == 0 && x->starter == 0) return;
+
+ unless (x->starter == 0) generate(g, x->starter);
+
+ p = p->left;
+ if (p != 0 && p->type != c_literalstring) p = p->right;
+ w(g, "~Mswitch(among_var) {~N~+"
+ "~Mcase 0: ~f~N");
+
+ until (p == 0) {
+ if (p->type == c_bra && p->left != 0) {
+ g->I[0] = case_number++;
+ w(g, "~Mcase ~I0:~N~+"); generate(g, p); w(g, "~Mbreak;~N~-");
+ }
+ p = p->right;
+ }
+ w(g, "~}");
+}
+
+static void generate_booltest(struct generator * g, struct node * p) {
+
+ g->V[0] = p->name;
+ wp(g, "~Mif (!(~V0)) ~f~C", p);
+}
+
+static void generate_false(struct generator * g, struct node * p) {
+
+ wp(g, "~M~f~C", p);
+}
+
+static void generate_debug(struct generator * g, struct node * p) {
+
+ g->I[0] = g->debug_count++;
+ g->I[1] = p->line_number;
+ wp(g, "~Mdebug(z, ~I0, ~I1);~C", p);
+
+}
+
+static void generate(struct generator * g, struct node * p) {
+
+ int used = g->label_used;
+ int a0 = g->failure_label;
+ const char * a1 = g->failure_string;
+
+ switch (p->type)
+ {
+ case c_define: generate_define(g, p); break;
+ case c_bra: generate_bra(g, p); break;
+ case c_and: generate_and(g, p); break;
+ case c_or: generate_or(g, p); break;
+ case c_backwards: generate_backwards(g, p); break;
+ case c_not: generate_not(g, p); break;
+ case c_set: generate_set(g, p); break;
+ case c_unset: generate_unset(g, p); break;
+ case c_try: generate_try(g, p); break;
+ case c_fail: generate_fail(g, p); break;
+ case c_reverse:
+ case c_test: generate_test(g, p); break;
+ case c_do: generate_do(g, p); break;
+ case c_goto: generate_GO(g, p, 1); break;
+ case c_gopast: generate_GO(g, p, 0); break;
+ case c_repeat: generate_repeat(g, p, false); break;
+ case c_loop: generate_loop(g, p); break;
+ case c_atleast: generate_atleast(g, p); break;
+ case c_setmark: generate_setmark(g, p); break;
+ case c_tomark: generate_tomark(g, p); break;
+ case c_atmark: generate_atmark(g, p); break;
+ case c_hop: generate_hop(g, p); break;
+ case c_delete: generate_delete(g, p); break;
+ case c_next: generate_next(g, p); break;
+ case c_tolimit: generate_tolimit(g, p); break;
+ case c_atlimit: generate_atlimit(g, p); break;
+ case c_leftslice: generate_leftslice(g, p); break;
+ case c_rightslice: generate_rightslice(g, p); break;
+ case c_assignto: generate_assignto(g, p); break;
+ case c_sliceto: generate_sliceto(g, p); break;
+ case c_assign: generate_assignfrom(g, p); break;
+ case c_insert:
+ case c_attach: generate_insert(g, p, p->type); break;
+ case c_slicefrom: generate_slicefrom(g, p); break;
+ case c_setlimit: generate_setlimit(g, p); break;
+ case c_dollar: generate_dollar(g, p); break;
+ case c_mathassign: generate_integer_assign(g, p, "="); break;
+ case c_plusassign: generate_integer_assign(g, p, "+="); break;
+ case c_minusassign: generate_integer_assign(g, p, "-="); break;
+ case c_multiplyassign:generate_integer_assign(g, p, "*="); break;
+ case c_divideassign: generate_integer_assign(g, p, "/="); break;
+ case c_eq: generate_integer_test(g, p, "=="); break;
+ case c_ne: generate_integer_test(g, p, "!="); break;
+ case c_gr: generate_integer_test(g, p, ">"); break;
+ case c_ge: generate_integer_test(g, p, ">="); break;
+ case c_ls: generate_integer_test(g, p, "<"); break;
+ case c_le: generate_integer_test(g, p, "<="); break;
+ case c_call: generate_call(g, p); break;
+ case c_grouping: generate_grouping(g, p, false); break;
+ case c_non: generate_grouping(g, p, true); break;
+ case c_name: generate_namedstring(g, p); break;
+ case c_literalstring: generate_literalstring(g, p); break;
+ case c_among: generate_among(g, p); break;
+ case c_substring: generate_substring(g, p); break;
+ case c_booltest: generate_booltest(g, p); break;
+ case c_false: generate_false(g, p); break;
+ case c_true: break;
+ case c_debug: generate_debug(g, p); break;
+ default: fprintf(stderr, "%d encountered\n", p->type);
+ exit(1);
+ }
+
+ if (g->failure_label != a0)
+ g->label_used = used;
+ g->failure_label = a0;
+ g->failure_string = a1;
+}
+
+static void generate_start_comment(struct generator * g) {
+
+ w(g, "~N/* This file was generated automatically by the Snowball to ANSI C compiler */~N");
+}
+
+static void generate_head(struct generator * g) {
+
+ if (g->options->runtime_path == 0) {
+ w(g, "~N#include \"header.h\"~N~N");
+ } else {
+ w(g, "~N#include \"");
+ ws(g, g->options->runtime_path);
+ if (g->options->runtime_path[strlen(g->options->runtime_path) - 1] != '/')
+ wch(g, '/');
+ w(g, "header.h\"~N~N");
+ }
+}
+
+static void generate_routine_headers(struct generator * g) {
+ struct name * q = g->analyser->names;
+ until (q == 0) {
+ g->V[0] = q;
+ switch (q->type) {
+ case t_routine:
+ w(g, "static int ~W0(struct SN_env * z);~N");
+ break;
+ case t_external:
+ w(g,
+ "#ifdef __cplusplus~N"
+ "extern \"C\" {~N"
+ "#endif~N"
+ "extern int ~W0(struct SN_env * z);~N"
+ "#ifdef __cplusplus~N"
+ "}~N"
+ "#endif~N"
+ );
+ break;
+ }
+ q = q->next;
+ }
+}
+
+static void generate_among_table(struct generator * g, struct among * x) {
+
+ struct amongvec * v = x->b;
+
+ g->I[0] = x->number;
+ {
+ int i;
+ for (i = 0; i < x->literalstring_count; i++)
+ {
+ g->I[1] = i;
+ g->I[2] = v->size;
+ g->L[0] = v->b;
+ unless (v->size == 0)
+ w(g, "static const symbol s_~I0_~I1[~I2] = ~A0;~N");
+ v++;
+ }
+ }
+
+ g->I[1] = x->literalstring_count;
+ w(g, "~N~Mstatic const struct among a_~I0[~I1] =~N{~N");
+
+ v = x->b;
+ {
+ int i;
+ for (i = 0; i < x->literalstring_count; i++) {
+ g->I[1] = i;
+ g->I[2] = v->size;
+ g->I[3] = v->i;
+ g->I[4] = v->result;
+ g->S[0] = i < x->literalstring_count - 1 ? "," : "";
+
+ w(g, "/*~J1 */ { ~I2, ");
+ if (v->size == 0) w(g, "0,");
+ else w(g, "s_~I0_~I1,");
+ w(g, " ~I3, ~I4, ");
+ if (v->function == 0) w(g, "0"); else
+ wvn(g, v->function);
+ w(g, "}~S0~N");
+ v++;
+ }
+ }
+ w(g, "};~N~N");
+}
+
+static void generate_amongs(struct generator * g) {
+ struct among * x = g->analyser->amongs;
+ until (x == 0) {
+ generate_among_table(g, x);
+ x = x->next;
+ }
+}
+
+static void set_bit(symbol * b, int i) { b[i/8] |= 1 << i%8; }
+
+static void generate_grouping_table(struct generator * g, struct grouping * q) {
+
+ int range = q->largest_ch - q->smallest_ch + 1;
+ int size = (range + 7)/ 8; /* assume 8 bits per symbol */
+ symbol * b = q->b;
+ symbol * map = create_b(size);
+ int i;
+ for (i = 0; i < size; i++) map[i] = 0;
+
+ for (i = 0; i < SIZE(b); i++) set_bit(map, b[i] - q->smallest_ch);
+
+ {
+ g->V[0] = q->name;
+
+ w(g, "static const unsigned char ~V0[] = { ");
+ for (i = 0; i < size; i++) {
+ wi(g, map[i]);
+ if (i < size - 1) w(g, ", ");
+ }
+ w(g, " };~N~N");
+ }
+ lose_b(map);
+}
+
+static void generate_groupings(struct generator * g) {
+ struct grouping * q = g->analyser->groupings;
+ until (q == 0) {
+ generate_grouping_table(g, q);
+ q = q->next;
+ }
+}
+
+static void generate_create(struct generator * g) {
+
+ int * p = g->analyser->name_count;
+ g->I[0] = p[t_string];
+ g->I[1] = p[t_integer];
+ g->I[2] = p[t_boolean];
+ w(g, "~N"
+ "extern struct SN_env * ~pcreate_env(void) { return SN_create_env(~I0, ~I1, ~I2); }"
+ "~N");
+}
+
+static void generate_close(struct generator * g) {
+
+ int * p = g->analyser->name_count;
+ g->I[0] = p[t_string];
+ w(g, "~Nextern void ~pclose_env(struct SN_env * z) { SN_close_env(z, ~I0); }~N~N");
+}
+
+static void generate_create_and_close_templates(struct generator * g) {
+ w(g, "~N"
+ "extern struct SN_env * ~pcreate_env(void);~N"
+ "extern void ~pclose_env(struct SN_env * z);~N"
+ "~N");
+}
+
+static void generate_header_file(struct generator * g) {
+
+ struct name * q = g->analyser->names;
+ char * vp = g->options->variables_prefix;
+ g->S[0] = vp;
+
+ w(g, "~N"
+ "#ifdef __cplusplus~N"
+ "extern \"C\" {~N"
+ "#endif~N"); /* for C++ */
+
+ generate_create_and_close_templates(g);
+ until (q == 0) {
+ g->V[0] = q;
+ switch (q->type)
+ {
+ case t_external:
+ w(g, "extern int ~W0(struct SN_env * z);~N");
+ break;
+ case t_string: g->S[1] = "S"; goto label0;
+ case t_integer: g->S[1] = "I"; goto label0;
+ case t_boolean: g->S[1] = "B";
+ label0:
+ if (vp) {
+ g->I[0] = q->count;
+ w(g, "#define ~S0");
+ str_append_b(g->outbuf, q->b);
+ w(g, " (~S1[~I0])~N");
+ }
+ break;
+ }
+ q = q->next;
+ }
+
+ w(g, "~N"
+ "#ifdef __cplusplus~N"
+ "}~N"
+ "#endif~N"); /* for C++ */
+
+ w(g, "~N");
+}
+
+extern void generate_program_c(struct generator * g) {
+
+ g->outbuf = str_new();
+ generate_start_comment(g);
+ generate_head(g);
+ generate_routine_headers(g);
+ w(g, "#ifdef __cplusplus~N"
+ "extern \"C\" {~N"
+ "#endif~N"
+ "~N");
+ generate_create_and_close_templates(g);
+ w(g, "~N"
+ "#ifdef __cplusplus~N"
+ "}~N"
+ "#endif~N");
+ generate_amongs(g);
+ generate_groupings(g);
+ g->declarations = g->outbuf;
+ g->outbuf = str_new();
+ g->literalstring_count = 0;
+ {
+ struct node * p = g->analyser->program;
+ until (p == 0) { generate(g, p); p = p->right; }
+ }
+ generate_create(g);
+ generate_close(g);
+ output_str(g->options->output_c, g->declarations);
+ str_delete(g->declarations);
+ output_str(g->options->output_c, g->outbuf);
+ str_clear(g->outbuf);
+
+ generate_start_comment(g);
+ generate_header_file(g);
+ output_str(g->options->output_h, g->outbuf);
+ str_delete(g->outbuf);
+}
+
+extern struct generator * create_generator_c(struct analyser * a, struct options * o) {
+ NEW(generator, g);
+ g->analyser = a;
+ g->options = o;
+ g->margin = 0;
+ g->debug_count = 0;
+ g->line_count = 0;
+ return g;
+}
+
+extern void close_generator_c(struct generator * g) {
+
+ FREE(g);
+}
+
diff --git a/contrib/snowball/compiler/generator_java.c b/contrib/snowball/compiler/generator_java.c
new file mode 100644
index 000000000..07a4b8e2e
--- /dev/null
+++ b/contrib/snowball/compiler/generator_java.c
@@ -0,0 +1,1452 @@
+
+#include <stdlib.h> /* for exit */
+#include <string.h> /* for strlen */
+#include <stdio.h> /* for fprintf etc */
+#include "header.h"
+
+/* prototypes */
+
+static void generate(struct generator * g, struct node * p);
+static void w(struct generator * g, const char * s);
+static void writef(struct generator * g, const char * s, struct node * p);
+
+
+enum special_labels {
+ x_return = -1
+};
+
+static int new_label(struct generator * g) {
+
+ return g->next_label++;
+}
+
+static struct str * vars_newname(struct generator * g) {
+
+ struct str * output;
+ g->var_number ++;
+ output = str_new();
+ str_append_string(output, "v_");
+ str_append_int(output, g->var_number);
+ return output;
+}
+
+/* Output routines */
+static void output_str(FILE * outfile, struct str * str) {
+
+ char * s = b_to_s(str_data(str));
+ fprintf(outfile, "%s", s);
+ free(s);
+}
+
+/* Write routines for simple entities */
+
+static void write_char(struct generator * g, int ch) {
+
+ str_append_ch(g->outbuf, ch);
+}
+
+static void write_newline(struct generator * g) {
+
+ str_append_string(g->outbuf, "\n");
+}
+
+static void write_string(struct generator * g, const char * s) {
+
+ str_append_string(g->outbuf, s);
+}
+
+static void write_b(struct generator * g, symbol * b) {
+
+ str_append_b(g->outbuf, b);
+}
+
+static void write_str(struct generator * g, struct str * str) {
+
+ str_append(g->outbuf, str);
+}
+
+static void write_int(struct generator * g, int i) {
+
+ str_append_int(g->outbuf, i);
+}
+
+
+/* Write routines for items from the syntax tree */
+
+static void write_varname(struct generator * g, struct name * p) {
+
+ int ch = "SBIrxg"[p->type];
+ if (p->type != t_external)
+ {
+ write_char(g, ch);
+ write_char(g, '_');
+ }
+ str_append_b(g->outbuf, p->b);
+}
+
+static void write_varref(struct generator * g, struct name * p) {
+
+ /* In java, references look just the same */
+ write_varname(g, p);
+}
+
+static void write_hexdigit(struct generator * g, int n) {
+
+ write_char(g, n < 10 ? n + '0' : n - 10 + 'A');
+}
+
+static void write_hex(struct generator * g, int ch) {
+
+ write_string(g, "\\u");
+ {
+ int i;
+ for (i = 12; i >= 0; i -= 4) write_hexdigit(g, ch >> i & 0xf);
+ }
+}
+
+static void write_literal_string(struct generator * g, symbol * p) {
+
+ int i;
+ write_string(g, "\"");
+ for (i = 0; i < SIZE(p); i++) {
+ int ch = p[i];
+ if (32 <= ch && ch <= 127) {
+ if (ch == '\"' || ch == '\\') write_string(g, "\\");
+ write_char(g, ch);
+ } else {
+ write_hex(g, ch);
+ }
+ }
+ write_string(g, "\"");
+}
+
+static void write_margin(struct generator * g) {
+
+ int i;
+ for (i = 0; i < g->margin; i++) write_string(g, " ");
+}
+
+/* Write a variable declaration. */
+static void write_declare(struct generator * g,
+ char * declaration,
+ struct node * p) {
+
+ struct str * temp = g->outbuf;
+ g->outbuf = g->declarations;
+ write_string(g, " ");
+ writef(g, declaration, p);
+ write_string(g, ";");
+ write_newline(g);
+ g->outbuf = temp;
+}
+
+static void write_comment(struct generator * g, struct node * p) {
+
+ write_margin(g);
+ write_string(g, "// ");
+ write_string(g, name_of_token(p->type));
+ if (p->name != 0) {
+ write_string(g, " ");
+ str_append_b(g->outbuf, p->name->b);
+ }
+ write_string(g, ", line ");
+ write_int(g, p->line_number);
+ write_newline(g);
+}
+
+static void write_block_start(struct generator * g) {
+
+ w(g, "~M{~+~N");
+}
+
+static void write_block_end(struct generator * g) /* block end */ {
+
+ w(g, "~-~M}~N");
+}
+
+static void write_savecursor(struct generator * g, struct node * p,
+ struct str * savevar) {
+
+ g->B[0] = str_data(savevar);
+ g->S[1] = "";
+ if (p->mode != m_forward) g->S[1] = "limit - ";
+ write_declare(g, "int ~B0", p);
+ writef(g, "~M~B0 = ~S1cursor;~N" , p);
+}
+
+static void restore_string(struct node * p, struct str * out, struct str * savevar) {
+
+ str_clear(out);
+ str_append_string(out, "cursor = ");
+ if (p->mode != m_forward) str_append_string(out, "limit - ");
+ str_append(out, savevar);
+ str_append_string(out, ";");
+}
+
+static void write_restorecursor(struct generator * g, struct node * p,
+ struct str * savevar) {
+
+ struct str * temp = str_new();
+ write_margin(g);
+ restore_string(p, temp, savevar);
+ write_str(g, temp);
+ write_newline(g);
+ str_delete(temp);
+}
+
+static void write_inc_cursor(struct generator * g, struct node * p) {
+
+ write_margin(g);
+ write_string(g, p->mode == m_forward ? "cursor++;" : "cursor--;");
+ write_newline(g);
+}
+
+static void wsetlab_begin(struct generator * g, int n) {
+
+ w(g, "~Mlab");
+ write_int(g, n);
+ w(g, ": do {~+~N");
+}
+
+static void wsetlab_end(struct generator * g) {
+
+ w(g, "~-~M} while (false);~N");
+}
+
+static void wgotol(struct generator * g, int n) {
+
+ write_margin(g);
+ write_string(g, "break lab");
+ write_int(g, n);
+ write_string(g, ";");
+ write_newline(g);
+}
+
+static void write_failure(struct generator * g) {
+
+ if (str_len(g->failure_str) != 0) {
+ write_margin(g);
+ write_str(g, g->failure_str);
+ write_newline(g);
+ }
+ write_margin(g);
+ switch (g->failure_label)
+ {
+ case x_return:
+ write_string(g, "return false;");
+ break;
+ default:
+ write_string(g, "break lab");
+ write_int(g, g->failure_label);
+ write_string(g, ";");
+ g->unreachable = true;
+ }
+ write_newline(g);
+}
+
+static void write_failure_if(struct generator * g, char * s, struct node * p) {
+
+ writef(g, "~Mif (", p);
+ writef(g, s, p);
+ writef(g, ")~N", p);
+ write_block_start(g);
+ write_failure(g);
+ write_block_end(g);
+ g->unreachable = false;
+}
+
+/* if at limit fail */
+static void write_check_limit(struct generator * g, struct node * p) {
+
+ if (p->mode == m_forward) {
+ write_failure_if(g, "cursor >= limit", p);
+ } else {
+ write_failure_if(g, "cursor <= limit_backward", p);
+ }
+}
+
+/* Formatted write. */
+static void writef(struct generator * g, const char * input, struct node * p) {
+
+ int i = 0;
+ int l = strlen(input);
+
+ while (i < l) {
+ int ch = input[i++];
+ if (ch == '~') {
+ switch(input[i++]) {
+ default: write_char(g, input[i - 1]); continue;
+ case 'C': write_comment(g, p); continue;
+ case 'f': write_block_start(g);
+ write_failure(g);
+ g->unreachable = false;
+ write_block_end(g);
+ continue;
+ case 'M': write_margin(g); continue;
+ case 'N': write_newline(g); continue;
+ case '{': write_block_start(g); continue;
+ case '}': write_block_end(g); continue;
+ case 'S': write_string(g, g->S[input[i++] - '0']); continue;
+ case 'B': write_b(g, g->B[input[i++] - '0']); continue;
+ case 'I': write_int(g, g->I[input[i++] - '0']); continue;
+ case 'V': write_varref(g, g->V[input[i++] - '0']); continue;
+ case 'W': write_varname(g, g->V[input[i++] - '0']); continue;
+ case 'L': write_literal_string(g, g->L[input[i++] - '0']); continue;
+ case '+': g->margin++; continue;
+ case '-': g->margin--; continue;
+ case 'n': write_string(g, g->options->name); continue;
+ }
+ } else {
+ write_char(g, ch);
+ }
+ }
+}
+
+static void w(struct generator * g, const char * s) {
+ writef(g, s, 0);
+}
+
+static void generate_AE(struct generator * g, struct node * p) {
+ char * s;
+ switch (p->type) {
+ case c_name:
+ write_varref(g, p->name); break;
+ case c_number:
+ write_int(g, p->number); break;
+ case c_maxint:
+ write_string(g, "MAXINT"); break;
+ case c_minint:
+ write_string(g, "MININT"); break;
+ case c_neg:
+ write_string(g, "-"); generate_AE(g, p->right); break;
+ case c_multiply:
+ s = " * "; goto label0;
+ case c_plus:
+ s = " + "; goto label0;
+ case c_minus:
+ s = " - "; goto label0;
+ case c_divide:
+ s = " / ";
+ label0:
+ write_string(g, "("); generate_AE(g, p->left);
+ write_string(g, s); generate_AE(g, p->right); write_string(g, ")"); break;
+ case c_sizeof:
+ g->V[0] = p->name;
+ w(g, "(~V0.length())"); break;
+ case c_cursor:
+ w(g, "cursor"); break;
+ case c_limit:
+ w(g, p->mode == m_forward ? "limit" : "limit_backward"); break;
+ case c_size:
+ w(g, "(current.length())"); break;
+ }
+}
+
+/* K_needed() tests to see if we really need to keep c. Not true when the
+ the command does not touch the cursor. This and repeat_score() could be
+ elaborated almost indefinitely.
+*/
+
+static int K_needed(struct generator * g, struct node * p) {
+
+ while (p != 0) {
+ switch (p->type) {
+ case c_dollar:
+ case c_leftslice:
+ case c_rightslice:
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ case c_sliceto:
+ case c_booltest:
+ case c_true:
+ case c_false:
+ case c_debug:
+ break;
+
+ case c_call:
+ if (K_needed(g, p->name->definition)) return true;
+ break;
+
+ case c_bra:
+ if (K_needed(g, p->left)) return true;
+ break;
+
+ default: return true;
+ }
+ p = p->right;
+ }
+ return false;
+}
+
+static int repeat_score(struct generator * g, struct node * p) {
+
+ int score = 0;
+ while (p != 0) {
+ switch (p->type) {
+ case c_dollar:
+ case c_leftslice:
+ case c_rightslice:
+ case c_mathassign:
+ case c_plusassign:
+ case c_minusassign:
+ case c_multiplyassign:
+ case c_divideassign:
+ case c_eq:
+ case c_ne:
+ case c_gr:
+ case c_ge:
+ case c_ls:
+ case c_le:
+ case c_sliceto: /* case c_not: must not be included here! */
+ case c_debug:
+ break;
+
+ case c_call:
+ score += repeat_score(g, p->name->definition);
+ break;
+
+ case c_bra:
+ score += repeat_score(g, p->left);
+ break;
+
+ case c_name:
+ case c_literalstring:
+ case c_next:
+ case c_grouping:
+ case c_non:
+ case c_hop:
+ score = score + 1;
+ break;
+
+ default:
+ score = 2;
+ break;
+ }
+ p = p->right;
+ }
+ return score;
+}
+
+/* tests if an expression requires cursor reinstatement in a repeat */
+
+static int repeat_restore(struct generator * g, struct node * p) {
+
+ return repeat_score(g, p) >= 2;
+}
+
+static void generate_bra(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ p = p->left;
+ while (p != 0) {
+ generate(g, p);
+ p = p->right;
+ }
+}
+
+static void generate_and(struct generator * g, struct node * p) {
+
+ struct str * savevar = vars_newname(g);
+ int keep_c = K_needed(g, p->left);
+
+ write_comment(g, p);
+
+ if (keep_c) write_savecursor(g, p, savevar);
+
+ p = p->left;
+ while (p != 0) {
+ generate(g, p);
+ if (g->unreachable) break;
+ if (keep_c && p->right != 0) write_restorecursor(g, p, savevar);
+ p = p->right;
+ }
+ str_delete(savevar);
+}
+
+static void generate_or(struct generator * g, struct node * p) {
+
+ struct str * savevar = vars_newname(g);
+ int keep_c = K_needed(g, p->left);
+
+ int a0 = g->failure_label;
+ struct str * a1 = str_copy(g->failure_str);
+
+ int out_lab = new_label(g);
+ write_comment(g, p);
+ wsetlab_begin(g, out_lab);
+
+ if (keep_c) write_savecursor(g, p, savevar);
+
+ p = p->left;
+ str_clear(g->failure_str);
+
+ if (p == 0) {
+ /* p should never be 0 after an or: there should be at least two
+ * sub nodes. */
+ fprintf(stderr, "Error: \"or\" node without children nodes.");
+ exit (1);
+ }
+ while (p->right != 0) {
+ g->failure_label = new_label(g);
+ wsetlab_begin(g, g->failure_label);
+ generate(g, p);
+ if (!g->unreachable) wgotol(g, out_lab);
+ wsetlab_end(g);
+ g->unreachable = false;
+ if (keep_c) write_restorecursor(g, p, savevar);
+ p = p->right;
+ }
+
+ g->failure_label = a0;
+ str_delete(g->failure_str);
+ g->failure_str = a1;
+
+ generate(g, p);
+ wsetlab_end(g);
+ str_delete(savevar);
+}
+
+static void generate_backwards(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ writef(g,"~Mlimit_backward = cursor; cursor = limit;~N", p);
+ generate(g, p->left);
+ w(g, "~Mcursor = limit_backward;");
+}
+
+
+static void generate_not(struct generator * g, struct node * p) {
+
+ struct str * savevar = vars_newname(g);
+ int keep_c = K_needed(g, p->left);
+
+ int a0 = g->failure_label;
+ struct str * a1 = str_copy(g->failure_str);
+
+ write_comment(g, p);
+ if (keep_c) {
+ write_block_start(g);
+ write_savecursor(g, p, savevar);
+ }
+
+ g->failure_label = new_label(g);
+ str_clear(g->failure_str);
+
+ wsetlab_begin(g, g->failure_label);
+
+ generate(g, p->left);
+
+ g->failure_label = a0;
+ str_delete(g->failure_str);
+ g->failure_str = a1;
+
+ if (!g->unreachable) write_failure(g);
+
+ wsetlab_end(g);
+ g->unreachable = false;
+
+ if (keep_c) write_restorecursor(g, p, savevar);
+ if (keep_c) write_block_end(g);
+ str_delete(savevar);
+}
+
+
+static void generate_try(struct generator * g, struct node * p) {
+
+ struct str * savevar = vars_newname(g);
+ int keep_c = K_needed(g, p->left);
+
+ write_comment(g, p);
+ if (keep_c) write_savecursor(g, p, savevar);
+
+ g->failure_label = new_label(g);
+ if (keep_c) restore_string(p, g->failure_str, savevar);
+
+ wsetlab_begin(g, g->failure_label);
+ generate(g, p->left);
+ wsetlab_end(g);
+ g->unreachable = false;
+
+ str_delete(savevar);
+}
+
+static void generate_set(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = true;~N", p);
+}
+
+static void generate_unset(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = false;~N", p);
+}
+
+static void generate_fail(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ generate(g, p->left);
+ if (!g->unreachable) write_failure(g);
+}
+
+/* generate_test() also implements 'reverse' */
+
+static void generate_test(struct generator * g, struct node * p) {
+
+ struct str * savevar = vars_newname(g);
+ int keep_c = K_needed(g, p->left);
+
+ write_comment(g, p);
+
+ if (keep_c) {
+ write_savecursor(g, p, savevar);
+ }
+
+ generate(g, p->left);
+
+ if (!g->unreachable) {
+ if (keep_c) {
+ write_restorecursor(g, p, savevar);
+ }
+ }
+ str_delete(savevar);
+}
+
+static void generate_do(struct generator * g, struct node * p) {
+
+ struct str * savevar = vars_newname(g);
+ int keep_c = K_needed(g, p->left);
+ write_comment(g, p);
+ if (keep_c) write_savecursor(g, p, savevar);
+
+ g->failure_label = new_label(g);
+ str_clear(g->failure_str);
+
+ wsetlab_begin(g, g->failure_label);
+ generate(g, p->left);
+ wsetlab_end(g);
+ g->unreachable = false;
+
+ if (keep_c) write_restorecursor(g, p, savevar);
+ str_delete(savevar);
+}
+
+static void generate_GO(struct generator * g, struct node * p, int style) {
+
+ int end_unreachable = false;
+ struct str * savevar = vars_newname(g);
+ int keep_c = style == 1 || repeat_restore(g, p->left);
+
+ int a0 = g->failure_label;
+ struct str * a1 = str_copy(g->failure_str);
+
+ int golab = new_label(g);
+ g->I[0] = golab;
+ write_comment(g, p);
+ w(g, "~Mgolab~I0: while(true)~N");
+ w(g, "~{");
+
+ if (keep_c) write_savecursor(g, p, savevar);
+
+ g->failure_label = new_label(g);
+ wsetlab_begin(g, g->failure_label);
+ generate(g, p->left);
+
+ if (g->unreachable) {
+ /* Cannot break out of this loop: therefore the code after the
+ * end of the loop is unreachable.*/
+ end_unreachable = true;
+ } else {
+ /* include for goto; omit for gopast */
+ if (style == 1) write_restorecursor(g, p, savevar);
+ g->I[0] = golab;
+ w(g, "~Mbreak golab~I0;~N");
+ }
+ g->unreachable = false;
+ wsetlab_end(g);
+ if (keep_c) write_restorecursor(g, p, savevar);
+
+ g->failure_label = a0;
+ str_delete(g->failure_str);
+ g->failure_str = a1;
+
+ write_check_limit(g, p);
+ write_inc_cursor(g, p);
+ write_block_end(g);
+ str_delete(savevar);
+ g->unreachable = end_unreachable;
+}
+
+static void generate_loop(struct generator * g, struct node * p) {
+
+ struct str * loopvar = vars_newname(g);
+ write_comment(g, p);
+ g->B[0] = str_data(loopvar);
+ write_declare(g, "int ~B0", p);
+ w(g, "~Mfor (~B0 = ");
+ generate_AE(g, p->AE);
+ g->B[0] = str_data(loopvar);
+ writef(g, "; ~B0 > 0; ~B0--)~N", p);
+ writef(g, "~{", p);
+
+ generate(g, p->left);
+
+ w(g, "~}");
+ str_delete(loopvar);
+ g->unreachable = false;
+}
+
+static void generate_repeat(struct generator * g, struct node * p, struct str * loopvar) {
+
+ struct str * savevar = vars_newname(g);
+ int keep_c = repeat_restore(g, p->left);
+ int replab = new_label(g);
+ g->I[0] = replab;
+ write_comment(g, p);
+ writef(g, "~Mreplab~I0: while(true)~N~{", p);
+
+ if (keep_c) write_savecursor(g, p, savevar);
+
+ g->failure_label = new_label(g);
+ str_clear(g->failure_str);
+ wsetlab_begin(g, g->failure_label);
+ generate(g, p->left);
+
+ if (!g->unreachable) {
+ if (loopvar != 0) {
+ g->B[0] = str_data(loopvar);
+ w(g, "~M~B0--;~N");
+ }
+
+ g->I[0] = replab;
+ w(g, "~Mcontinue replab~I0;~N");
+ }
+
+ wsetlab_end(g);
+ g->unreachable = false;
+
+ if (keep_c) write_restorecursor(g, p, savevar);
+
+ g->I[0] = replab;
+ w(g, "~Mbreak replab~I0;~N~}");
+ str_delete(savevar);
+}
+
+static void generate_atleast(struct generator * g, struct node * p) {
+
+ struct str * loopvar = vars_newname(g);
+ write_comment(g, p);
+ w(g, "~{");
+ g->B[0] = str_data(loopvar);
+ w(g, "~Mint ~B0 = ");
+ generate_AE(g, p->AE);
+ w(g, ";~N");
+ {
+ int a0 = g->failure_label;
+ struct str * a1 = str_copy(g->failure_str);
+
+ generate_repeat(g, p, loopvar);
+
+ g->failure_label = a0;
+ str_delete(g->failure_str);
+ g->failure_str = a1;
+ }
+ g->B[0] = str_data(loopvar);
+ write_failure_if(g, "~B0 > 0", p);
+ w(g, "~}");
+ str_delete(loopvar);
+}
+
+static void generate_setmark(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = cursor;~N", p);
+}
+
+static void generate_tomark(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->S[0] = p->mode == m_forward ? ">" : "<";
+
+ w(g, "~Mif (cursor ~S0 "); generate_AE(g, p->AE); w(g, ")~N");
+ write_block_start(g);
+ write_failure(g);
+ write_block_end(g);
+ g->unreachable = false;
+ w(g, "~Mcursor = "); generate_AE(g, p->AE); writef(g, ";~N", p);
+}
+
+static void generate_atmark(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ w(g, "~Mif (cursor != "); generate_AE(g, p->AE); writef(g, ")~N", p);
+ write_block_start(g);
+ write_failure(g);
+ write_block_end(g);
+ g->unreachable = false;
+}
+
+
+static void generate_hop(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->S[0] = p->mode == m_forward ? "+" : "-";
+
+ w(g, "~{~Mint c = cursor ~S0 ");
+ generate_AE(g, p->AE);
+ w(g, ";~N");
+
+ g->S[0] = p->mode == m_forward ? "0" : "limit_backward";
+
+ write_failure_if(g, "~S0 > c || c > limit", p);
+ writef(g, "~Mcursor = c;~N", p);
+ writef(g, "~}", p);
+}
+
+static void generate_delete(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ writef(g, "~Mslice_del();~N", p);
+}
+
+
+static void generate_next(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ write_check_limit(g, p);
+ write_inc_cursor(g, p);
+}
+
+static void generate_tolimit(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->S[0] = p->mode == m_forward ? "limit" : "limit_backward";
+ writef(g, "~Mcursor = ~S0;~N", p);
+}
+
+static void generate_atlimit(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->S[0] = p->mode == m_forward ? "limit" : "limit_backward";
+ g->S[1] = p->mode == m_forward ? "<" : ">";
+ write_failure_if(g, "cursor ~S1 ~S0", p);
+}
+
+static void generate_leftslice(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->S[0] = p->mode == m_forward ? "bra" : "ket";
+ writef(g, "~M~S0 = cursor;~N", p);
+}
+
+static void generate_rightslice(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->S[0] = p->mode == m_forward ? "ket" : "bra";
+ writef(g, "~M~S0 = cursor;~N", p);
+}
+
+static void generate_assignto(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = assign_to(~V0);~N", p);
+}
+
+static void generate_sliceto(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->V[0] = p->name;
+ writef(g, "~M~V0 = slice_to(~V0);~N", p);
+}
+
+static void generate_address(struct generator * g, struct node * p) {
+
+ symbol * b = p->literalstring;
+ if (b != 0) {
+ write_literal_string(g, b);
+ } else {
+ write_varref(g, p->name);
+ }
+}
+
+static void generate_insert(struct generator * g, struct node * p, int style) {
+
+ int keep_c = style == c_attach;
+ write_comment(g, p);
+ if (p->mode == m_backward) keep_c = !keep_c;
+ if (keep_c) w(g, "~{~Mint c = cursor;~N");
+ writef(g, "~Minsert(cursor, cursor, ", p);
+ generate_address(g, p);
+ writef(g, ");~N", p);
+ if (keep_c) w(g, "~Mcursor = c;~N~}");
+}
+
+static void generate_assignfrom(struct generator * g, struct node * p) {
+
+ int keep_c = p->mode == m_forward; /* like 'attach' */
+
+ write_comment(g, p);
+ if (keep_c) writef(g, "~{~Mint c = cursor;~N", p);
+ if (p->mode == m_forward) {
+ writef(g, "~Minsert(cursor, limit, ", p);
+ } else {
+ writef(g, "~Minsert(limit_backward, cursor, ", p);
+ }
+ generate_address(g, p);
+ writef(g, ");~N", p);
+ if (keep_c) w(g, "~Mcursor = c;~N~}");
+}
+
+
+static void generate_slicefrom(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ w(g, "~Mslice_from(");
+ generate_address(g, p);
+ writef(g, ");~N", p);
+}
+
+static void generate_setlimit(struct generator * g, struct node * p) {
+
+ struct str * savevar = vars_newname(g);
+ struct str * varname = vars_newname(g);
+ write_comment(g, p);
+ write_savecursor(g, p, savevar);
+ generate(g, p->left);
+
+ if (!g->unreachable) {
+ g->B[0] = str_data(varname);
+ write_declare(g, "int ~B0", p);
+ if (p->mode == m_forward) {
+ w(g, "~M~B0 = limit - cursor;~N");
+ w(g, "~Mlimit = cursor;~N");
+ } else {
+ w(g, "~M~B0 = limit_backward;~N");
+ w(g, "~Mlimit_backward = cursor;~N");
+ }
+ write_restorecursor(g, p, savevar);
+
+ if (p->mode == m_forward) {
+ str_assign(g->failure_str, "limit += ");
+ str_append(g->failure_str, varname);
+ str_append_ch(g->failure_str, ';');
+ } else {
+ str_assign(g->failure_str, "limit_backward = ");
+ str_append(g->failure_str, varname);
+ str_append_ch(g->failure_str, ';');
+ }
+ generate(g, p->aux);
+
+ if (!g->unreachable) {
+ write_margin(g);
+ write_str(g, g->failure_str);
+ write_newline(g);
+ }
+ }
+ str_delete(varname);
+ str_delete(savevar);
+}
+
+/* dollar sets snowball up to operate on a string variable as if it were the
+ * current string */
+static void generate_dollar(struct generator * g, struct node * p) {
+
+ struct str * savevar = vars_newname(g);
+ write_comment(g, p);
+ g->V[0] = p->name;
+
+ str_assign(g->failure_str, "copy_from(");
+ str_append(g->failure_str, savevar);
+ str_append_string(g->failure_str, ");");
+ g->B[0] = str_data(savevar);
+ writef(g, "~{~M~n ~B0 = this;~N"
+ "~Mcurrent = new StringBuffer(~V0.toString());~N"
+ "~Mcursor = 0;~N"
+ "~Mlimit = (current.length());~N", p);
+ generate(g, p->left);
+ if (!g->unreachable) {
+ write_margin(g);
+ write_str(g, g->failure_str);
+ write_newline(g);
+ }
+ w(g, "~}");
+ str_delete(savevar);
+}
+
+static void generate_integer_assign(struct generator * g, struct node * p, char * s) {
+
+ g->V[0] = p->name;
+ g->S[0] = s;
+ w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); w(g, ";~N");
+}
+
+static void generate_integer_test(struct generator * g, struct node * p, char * s) {
+
+ g->V[0] = p->name;
+ g->S[0] = s;
+ w(g, "~Mif (!(~V0 ~S0 "); generate_AE(g, p->AE); w(g, "))~N");
+ write_block_start(g);
+ write_failure(g);
+ write_block_end(g);
+ g->unreachable = false;
+}
+
+static void generate_call(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->V[0] = p->name;
+ write_failure_if(g, "!~V0()", p);
+}
+
+static void generate_grouping(struct generator * g, struct node * p, int complement) {
+
+ struct grouping * q = p->name->grouping;
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->S[1] = complement ? "out" : "in";
+ g->V[0] = p->name;
+ g->I[0] = q->smallest_ch;
+ g->I[1] = q->largest_ch;
+ if (q->no_gaps)
+ write_failure_if(g, "!(~S1_range~S0(~I0, ~I1))", p);
+ else
+ write_failure_if(g, "!(~S1_grouping~S0(~V0, ~I0, ~I1))", p);
+}
+
+static void generate_namedstring(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->V[0] = p->name;
+ write_failure_if(g, "!(eq_v~S0(~V0))", p);
+}
+
+static void generate_literalstring(struct generator * g, struct node * p) {
+
+ symbol * b = p->literalstring;
+ write_comment(g, p);
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->I[0] = SIZE(b);
+ g->L[0] = b;
+ write_failure_if(g, "!(eq_s~S0(~I0, ~L0))", p);
+}
+
+static void generate_define(struct generator * g, struct node * p) {
+
+ struct name * q = p->name;
+
+ struct str * saved_output = g->outbuf;
+ struct str * saved_declarations = g->declarations;
+
+ g->S[0] = q->type == t_routine ? "private" : "public";
+ g->V[0] = q;
+ w(g, "~+~+~N~M~S0 boolean ~V0() {~+~N");
+
+ g->outbuf = str_new();
+ g->declarations = str_new();
+
+ g->next_label = 0;
+ g->var_number = 0;
+
+ if (p->amongvar_needed) write_declare(g, "int among_var", p);
+ str_clear(g->failure_str);
+ g->failure_label = x_return;
+ g->unreachable = false;
+ generate(g, p->left);
+ if (!g->unreachable) w(g, "~Mreturn true;~N");
+ w(g, "~}~-~-");
+
+ str_append(saved_output, g->declarations);
+ str_append(saved_output, g->outbuf);
+ str_delete(g->declarations);
+ str_delete(g->outbuf);
+ g->declarations = saved_declarations;
+ g->outbuf = saved_output;
+}
+
+static void generate_substring(struct generator * g, struct node * p) {
+
+ struct among * x = p->among;
+
+ write_comment(g, p);
+
+ g->S[0] = p->mode == m_forward ? "" : "_b";
+ g->I[0] = x->number;
+ g->I[1] = x->literalstring_count;
+
+ if (x->command_count == 0 && x->starter == 0) {
+ write_failure_if(g, "find_among~S0(a_~I0, ~I1) == 0", p);
+ } else {
+ writef(g, "~Mamong_var = find_among~S0(a_~I0, ~I1);~N", p);
+ write_failure_if(g, "among_var == 0", p);
+ }
+}
+
+static void generate_among(struct generator * g, struct node * p) {
+
+ struct among * x = p->among;
+ int case_number = 1;
+
+ if (x->substring == 0) generate_substring(g, p);
+ if (x->command_count == 0 && x->starter == 0) return;
+
+ if (x->starter != 0) generate(g, x->starter);
+
+ p = p->left;
+ if (p != 0 && p->type != c_literalstring) p = p->right;
+ w(g, "~Mswitch(among_var) {~N~+");
+ w(g, "~Mcase 0:~N~+");
+ write_failure(g);
+ g->unreachable = false;
+ w(g, "~-");
+
+ while (p != 0) {
+ if (p->type == c_bra && p->left != 0) {
+ g->I[0] = case_number++;
+ w(g, "~Mcase ~I0:~N~+");
+ generate(g, p);
+ if (!g->unreachable) w(g, "~Mbreak;~N");
+ w(g, "~-");
+ g->unreachable = false;
+ }
+ p = p->right;
+ }
+ write_block_end(g);
+}
+
+static void generate_booltest(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->V[0] = p->name;
+ write_failure_if(g, "!(~V0)", p);
+}
+
+static void generate_false(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ write_failure(g);
+}
+
+static void generate_debug(struct generator * g, struct node * p) {
+
+ write_comment(g, p);
+ g->I[0] = g->debug_count++;
+ g->I[1] = p->line_number;
+ writef(g, "~Mdebug(~I0, ~I1);~N", p);
+}
+
+static void generate(struct generator * g, struct node * p) {
+
+ int a0;
+ struct str * a1;
+
+ if (g->unreachable) return;
+
+ a0 = g->failure_label;
+ a1 = str_copy(g->failure_str);
+
+ switch (p->type)
+ {
+ case c_define: generate_define(g, p); break;
+ case c_bra: generate_bra(g, p); break;
+ case c_and: generate_and(g, p); break;
+ case c_or: generate_or(g, p); break;
+ case c_backwards: generate_backwards(g, p); break;
+ case c_not: generate_not(g, p); break;
+ case c_set: generate_set(g, p); break;
+ case c_unset: generate_unset(g, p); break;
+ case c_try: generate_try(g, p); break;
+ case c_fail: generate_fail(g, p); break;
+ case c_reverse:
+ case c_test: generate_test(g, p); break;
+ case c_do: generate_do(g, p); break;
+ case c_goto: generate_GO(g, p, 1); break;
+ case c_gopast: generate_GO(g, p, 0); break;
+ case c_repeat: generate_repeat(g, p, 0); break;
+ case c_loop: generate_loop(g, p); break;
+ case c_atleast: generate_atleast(g, p); break;
+ case c_setmark: generate_setmark(g, p); break;
+ case c_tomark: generate_tomark(g, p); break;
+ case c_atmark: generate_atmark(g, p); break;
+ case c_hop: generate_hop(g, p); break;
+ case c_delete: generate_delete(g, p); break;
+ case c_next: generate_next(g, p); break;
+ case c_tolimit: generate_tolimit(g, p); break;
+ case c_atlimit: generate_atlimit(g, p); break;
+ case c_leftslice: generate_leftslice(g, p); break;
+ case c_rightslice: generate_rightslice(g, p); break;
+ case c_assignto: generate_assignto(g, p); break;
+ case c_sliceto: generate_sliceto(g, p); break;
+ case c_assign: generate_assignfrom(g, p); break;
+ case c_insert:
+ case c_attach: generate_insert(g, p, p->type); break;
+ case c_slicefrom: generate_slicefrom(g, p); break;
+ case c_setlimit: generate_setlimit(g, p); break;
+ case c_dollar: generate_dollar(g, p); break;
+ case c_mathassign: generate_integer_assign(g, p, "="); break;
+ case c_plusassign: generate_integer_assign(g, p, "+="); break;
+ case c_minusassign: generate_integer_assign(g, p, "-="); break;
+ case c_multiplyassign:generate_integer_assign(g, p, "*="); break;
+ case c_divideassign: generate_integer_assign(g, p, "/="); break;
+ case c_eq: generate_integer_test(g, p, "=="); break;
+ case c_ne: generate_integer_test(g, p, "!="); break;
+ case c_gr: generate_integer_test(g, p, ">"); break;
+ case c_ge: generate_integer_test(g, p, ">="); break;
+ case c_ls: generate_integer_test(g, p, "<"); break;
+ case c_le: generate_integer_test(g, p, "<="); break;
+ case c_call: generate_call(g, p); break;
+ case c_grouping: generate_grouping(g, p, false); break;
+ case c_non: generate_grouping(g, p, true); break;
+ case c_name: generate_namedstring(g, p); break;
+ case c_literalstring: generate_literalstring(g, p); break;
+ case c_among: generate_among(g, p); break;
+ case c_substring: generate_substring(g, p); break;
+ case c_booltest: generate_booltest(g, p); break;
+ case c_false: generate_false(g, p); break;
+ case c_true: break;
+ case c_debug: generate_debug(g, p); break;
+ default: fprintf(stderr, "%d encountered\n", p->type);
+ exit(1);
+ }
+
+ g->failure_label = a0;
+ str_delete(g->failure_str);
+ g->failure_str = a1;
+}
+
+static void generate_start_comment(struct generator * g) {
+
+ w(g, "// This file was generated automatically by the Snowball to Java compiler~N");
+ w(g, "~N");
+}
+
+static void generate_class_begin(struct generator * g) {
+
+ w(g, "package " );
+ w(g, g->options->package);
+ w(g, ";~N~N" );
+
+ w(g, "import ");
+ w(g, g->options->among_class );
+ w(g, ";~N"
+ "~N"
+ " /**~N"
+ " * This class was automatically generated by a Snowball to Java compiler ~N"
+ " * It implements the stemming algorithm defined by a snowball script.~N"
+ " */~N"
+ "~N"
+ "public class ~n extends ");
+
+ w(g, g->options->parent_class_name);
+ w(g, " {~N"
+ "~N"
+ "private static final long serialVersionUID = 1L;~N"
+ "~N"
+ "~+~+~Mprivate final static ~n methodObject = new ~n ();~N"
+ "~N");
+}
+
+static void generate_class_end(struct generator * g) {
+
+ w(g, "~N}");
+ w(g, "~N~N");
+}
+
+static void generate_equals(struct generator * g) {
+
+ w(g, "~N"
+ "~Mpublic boolean equals( Object o ) {~N"
+ "~+~Mreturn o instanceof ");
+ w(g, g->options->name);
+ w(g, ";~N~-~M}~N"
+ "~N"
+ "~Mpublic int hashCode() {~N"
+ "~+~Mreturn ");
+ w(g, g->options->name);
+ w(g, ".class.getName().hashCode();~N"
+ "~-~M}~N");
+ w(g, "~N~N");
+}
+
+static void generate_among_table(struct generator * g, struct among * x) {
+
+ struct amongvec * v = x->b;
+
+ g->I[0] = x->number;
+ g->I[1] = x->literalstring_count;
+
+ w(g, "~+~+~Mprivate final static Among a_~I0[] = {~N~+");
+ {
+ int i;
+ for (i = 0; i < x->literalstring_count; i++) {
+ g->I[0] = i;
+ g->I[1] = v->i;
+ g->I[2] = v->result;
+ g->L[0] = v->b;
+ g->S[0] = i < x->literalstring_count - 1 ? "," : "";
+
+ w(g, "~Mnew Among ( ~L0, ~I1, ~I2, \"");
+ if (v->function != 0) {
+ write_varname(g, v->function);
+ }
+ w(g, "\", methodObject )~S0~N");
+ v++;
+ }
+ }
+ w(g, "~-~M};~-~-~N~N");
+}
+
+static void generate_amongs(struct generator * g) {
+
+ struct among * x = g->analyser->amongs;
+ while (x != 0) {
+ generate_among_table(g, x);
+ x = x->next;
+ }
+}
+
+static void set_bit(symbol * b, int i) { b[i/8] |= 1 << i%8; }
+
+static int bit_is_set(symbol * b, int i) { return b[i/8] & 1 << i%8; }
+
+static void generate_grouping_table(struct generator * g, struct grouping * q) {
+
+ int range = q->largest_ch - q->smallest_ch + 1;
+ int size = (range + 7)/ 8; /* assume 8 bits per symbol */
+ symbol * b = q->b;
+ symbol * map = create_b(size);
+ int i;
+ for (i = 0; i < size; i++) map[i] = 0;
+
+ /* Using unicode would require revision here */
+
+ for (i = 0; i < SIZE(b); i++) set_bit(map, b[i] - q->smallest_ch);
+
+ q->no_gaps = true;
+ for (i = 0; i < range; i++) unless (bit_is_set(map, i)) q->no_gaps = false;
+
+ unless (q->no_gaps) {
+ g->V[0] = q->name;
+
+ w(g, "~+~+~Mprivate static final char ~V0[] = {");
+ for (i = 0; i < size; i++) {
+ write_int(g, map[i]);
+ if (i < size - 1) w(g, ", ");
+ }
+ w(g, " };~N~-~-~N");
+ }
+ lose_b(map);
+}
+
+static void generate_groupings(struct generator * g) {
+ struct grouping * q = g->analyser->groupings;
+ until (q == 0) {
+ generate_grouping_table(g, q);
+ q = q->next;
+ }
+}
+
+static void generate_members(struct generator * g) {
+
+ struct name * q = g->analyser->names;
+ until (q == 0) {
+ g->V[0] = q;
+ switch (q->type) {
+ case t_string:
+ w(g, " private ");
+ w(g, g->options->string_class );
+ w(g, " ~W0 = new ");
+ w(g, g->options->string_class);
+ w(g, "();~N");
+ break;
+ case t_integer:
+ w(g, " private int ~W0;~N");
+ break;
+ case t_boolean:
+ w(g, " private boolean ~W0;~N");
+ break;
+ }
+ q = q->next;
+ }
+ w(g, "~N");
+}
+
+static void generate_copyfrom(struct generator * g) {
+
+ struct name * q;
+ w(g, "~+~+~Mprivate void copy_from(~n other) {~+~N");
+ for (q = g->analyser->names; q != 0; q = q->next) {
+ g->V[0] = q;
+ switch (q->type) {
+ case t_string:
+ case t_integer:
+ case t_boolean:
+ w(g, "~M~W0 = other.~W0;~N");
+ break;
+ }
+ }
+ w(g, "~Msuper.copy_from(other);~N");
+ w(g, "~-~M}~-~-~N");
+}
+
+static void generate_methods(struct generator * g) {
+
+ struct node * p = g->analyser->program;
+ while (p != 0) {
+ generate(g, p);
+ g->unreachable = false;
+ p = p->right;
+ }
+}
+
+extern void generate_program_java(struct generator * g) {
+
+ g->outbuf = str_new();
+ g->failure_str = str_new();
+
+ generate_start_comment(g);
+ generate_class_begin(g);
+
+ generate_amongs(g);
+ generate_groupings(g);
+
+ generate_members(g);
+ generate_copyfrom(g);
+ generate_methods(g);
+ generate_equals(g);
+
+ generate_class_end(g);
+
+ output_str(g->options->output_java, g->outbuf);
+ str_delete(g->failure_str);
+ str_delete(g->outbuf);
+}
+
+extern struct generator * create_generator_java(struct analyser * a, struct options * o) {
+
+ NEW(generator, g);
+ g->analyser = a;
+ g->options = o;
+ g->margin = 0;
+ g->debug_count = 0;
+ g->unreachable = false;
+ return g;
+}
+
+extern void close_generator_java(struct generator * g) {
+
+ FREE(g);
+}
+
diff --git a/contrib/snowball/compiler/header.h b/contrib/snowball/compiler/header.h
new file mode 100644
index 000000000..9baf1d917
--- /dev/null
+++ b/contrib/snowball/compiler/header.h
@@ -0,0 +1,324 @@
+
+typedef unsigned char byte;
+typedef unsigned short symbol;
+
+#define true 1
+#define false 0
+#define repeat while(true)
+#define unless(C) if(!(C))
+#define until(C) while(!(C))
+
+#define MALLOC check_malloc
+#define FREE check_free
+
+#define NEW(type, p) struct type * p = (struct type *) MALLOC(sizeof(struct type))
+#define NEWVEC(type, p, n) struct type * p = (struct type *) MALLOC(sizeof(struct type) * n)
+
+#define STARTSIZE 10
+#define SIZE(p) ((int *)(p))[-1]
+#define CAPACITY(p) ((int *)(p))[-2]
+
+extern symbol * create_b(int n);
+extern void report_b(FILE * out, symbol * p);
+extern void lose_b(symbol * p);
+extern symbol * increase_capacity(symbol * p, int n);
+extern symbol * move_to_b(symbol * p, int n, symbol * q);
+extern symbol * add_to_b(symbol * p, int n, symbol * q);
+extern symbol * copy_b(symbol * p);
+extern char * b_to_s(symbol * p);
+extern symbol * add_s_to_b(symbol * p, const char * s);
+
+struct str; /* defined in space.c */
+
+extern struct str * str_new(void);
+extern void str_delete(struct str * str);
+extern void str_append(struct str * str, struct str * add);
+extern void str_append_ch(struct str * str, char add);
+extern void str_append_b(struct str * str, symbol * q);
+extern void str_append_string(struct str * str, const char * s);
+extern void str_append_int(struct str * str, int i);
+extern void str_clear(struct str * str);
+extern void str_assign(struct str * str, char * s);
+extern struct str * str_copy(struct str * old);
+extern symbol * str_data(struct str * str);
+extern int str_len(struct str * str);
+extern int get_utf8(const symbol * p, int * slot);
+extern int put_utf8(int ch, symbol * p);
+
+struct m_pair {
+
+ struct m_pair * next;
+ symbol * name;
+ symbol * value;
+
+};
+
+/* struct input must be a prefix of struct tokeniser. */
+struct input {
+
+ struct input * next;
+ symbol * p;
+ int c;
+ char * file;
+ int line_number;
+
+};
+
+struct include {
+
+ struct include * next;
+ symbol * b;
+
+};
+
+/* struct input must be a prefix of struct tokeniser. */
+struct tokeniser {
+
+ struct input * next;
+ symbol * p;
+ int c;
+ char * file;
+ int line_number;
+ symbol * b;
+ symbol * b2;
+ int number;
+ int m_start;
+ int m_end;
+ struct m_pair * m_pairs;
+ int get_depth;
+ int error_count;
+ int token;
+ int previous_token;
+ byte token_held;
+ byte widechars;
+ byte utf8;
+
+ int omission;
+ struct include * includes;
+
+};
+
+extern symbol * get_input(symbol * p, char ** p_file);
+extern struct tokeniser * create_tokeniser(symbol * b, char * file);
+extern int read_token(struct tokeniser * t);
+extern const char * name_of_token(int code);
+extern void close_tokeniser(struct tokeniser * t);
+
+enum token_codes {
+
+#include "syswords2.h"
+
+ c_mathassign,
+ c_name,
+ c_number,
+ c_literalstring,
+ c_neg,
+ c_call,
+ c_grouping,
+ c_booltest
+};
+
+extern int space_count;
+extern void * check_malloc(int n);
+extern void check_free(void * p);
+
+struct node;
+
+struct name {
+
+ struct name * next;
+ symbol * b;
+ int type; /* t_string etc */
+ int mode; /* )_ for routines, externals */
+ struct node * definition; /* ) */
+ int count; /* 0, 1, 2 for each type */
+ struct grouping * grouping; /* for grouping names */
+ byte referenced;
+ byte used;
+
+};
+
+struct literalstring {
+
+ struct literalstring * next;
+ symbol * b;
+
+};
+
+struct amongvec {
+
+ symbol * b; /* the string giving the case */
+ int size; /* - and its size */
+ struct node * p; /* the corresponding command */
+ int i; /* the amongvec index of the longest substring of b */
+ int result; /* the numeric result for the case */
+ struct name * function;
+
+};
+
+struct among {
+
+ struct among * next;
+ struct amongvec * b; /* pointer to the amongvec */
+ int number; /* amongs are numbered 0, 1, 2 ... */
+ int literalstring_count; /* in this among */
+ int command_count; /* in this among */
+ struct node * starter; /* i.e. among( (starter) 'string' ... ) */
+ struct node * substring; /* i.e. substring ... among ( ... ) */
+};
+
+struct grouping {
+
+ struct grouping * next;
+ int number; /* groupings are numbered 0, 1, 2 ... */
+ symbol * b; /* the characters of this group */
+ int largest_ch; /* character with max code */
+ int smallest_ch; /* character with min code */
+ byte no_gaps; /* not used in generator.c after 11/5/05 */
+ struct name * name; /* so g->name->grouping == g */
+};
+
+struct node {
+
+ struct node * next;
+ struct node * left;
+ struct node * aux; /* used in setlimit */
+ struct among * among; /* used in among */
+ struct node * right;
+ int type;
+ int mode;
+ struct node * AE;
+ struct name * name;
+ symbol * literalstring;
+ int number;
+ int line_number;
+ int amongvar_needed; /* used in routine definitions */
+};
+
+enum name_types {
+
+ t_size = 6,
+
+ t_string = 0, t_boolean = 1, t_integer = 2, t_routine = 3, t_external = 4,
+ t_grouping = 5
+
+/* If this list is extended, adjust wvn in generator.c */
+};
+
+/* In name_count[i] below, remember that
+ type is
+ ----+----
+ 0 | string
+ 1 | boolean
+ 2 | integer
+ 3 | routine
+ 4 | external
+ 5 | grouping
+*/
+
+struct analyser {
+
+ struct tokeniser * tokeniser;
+ struct node * nodes;
+ struct name * names;
+ struct literalstring * literalstrings;
+ int mode;
+ byte modifyable; /* false inside reverse(...) */
+ struct node * program;
+ struct node * program_end;
+ int name_count[t_size]; /* name_count[i] counts the number of names of type i */
+ struct among * amongs;
+ struct among * amongs_end;
+ int among_count;
+ int amongvar_needed; /* used in reading routine definitions */
+ struct grouping * groupings;
+ struct grouping * groupings_end;
+ struct node * substring; /* pending 'substring' in current routine definition */
+ byte utf8;
+};
+
+enum analyser_modes {
+
+ m_forward = 0, m_backward /*, m_integer */
+
+};
+
+extern void print_program(struct analyser * a);
+extern struct analyser * create_analyser(struct tokeniser * t);
+extern void close_analyser(struct analyser * a);
+
+extern void read_program(struct analyser * a);
+
+struct generator {
+
+ struct analyser * analyser;
+ struct options * options;
+ int unreachable; /* 0 if code can be reached, 1 if current code
+ * is unreachable. */
+ int var_number; /* Number of next variable to use. */
+ struct str * outbuf; /* temporary str to store output */
+ struct str * declarations; /* str storing variable declarations */
+ int next_label;
+ int margin;
+
+ const char * failure_string; /* String to output in case of a failure. */
+#ifndef DISABLE_JAVA
+ struct str * failure_str; /* This is used by the java generator instead of failure_string */
+#endif
+
+ int label_used; /* Keep track of whether the failure label is used. */
+ int failure_label;
+ int debug_count;
+
+ const char * S[10]; /* strings */
+ symbol * B[10]; /* blocks */
+ int I[10]; /* integers */
+ struct name * V[5]; /* variables */
+ symbol * L[5]; /* literals, used in formatted write */
+
+ int line_count; /* counts number of lines output */
+ int line_labelled; /* in ANSI C, will need extra ';' if it is a block end */
+ int literalstring_count;
+ int keep_count; /* used to number keep/restore pairs to avoid compiler warnings
+ about shadowed variables */
+};
+
+struct options {
+
+ /* for the command line: */
+
+ char * output_file;
+ char * name;
+ FILE * output_c;
+ FILE * output_h;
+#ifndef DISABLE_JAVA
+ FILE * output_java;
+#endif
+ byte syntax_tree;
+ byte widechars;
+ enum { LANG_JAVA, LANG_C, LANG_CPLUSPLUS } make_lang;
+ char * externals_prefix;
+ char * variables_prefix;
+ char * runtime_path;
+ char * parent_class_name;
+ char * package;
+ char * string_class;
+ char * among_class;
+ struct include * includes;
+ struct include * includes_end;
+ byte utf8;
+};
+
+/* Generator for C code. */
+extern struct generator * create_generator_c(struct analyser * a, struct options * o);
+extern void close_generator_c(struct generator * g);
+
+extern void generate_program_c(struct generator * g);
+
+#ifndef DISABLE_JAVA
+/* Generator for Java code. */
+extern struct generator * create_generator_java(struct analyser * a, struct options * o);
+extern void close_generator_java(struct generator * g);
+
+extern void generate_program_java(struct generator * g);
+#endif
diff --git a/contrib/snowball/compiler/space.c b/contrib/snowball/compiler/space.c
new file mode 100644
index 000000000..cd5fd863d
--- /dev/null
+++ b/contrib/snowball/compiler/space.c
@@ -0,0 +1,263 @@
+
+#include <stdio.h> /* for printf */
+#include <stdlib.h> /* malloc, free */
+#include <string.h> /* memmove */
+
+#include "header.h"
+
+#define HEAD 2*sizeof(int)
+#define EXTENDER 40
+
+
+/* This modules provides a simple mechanism for arbitrary length writable
+ strings, called 'blocks'. They are 'symbol *' items rather than 'char *'
+ items however.
+
+ The calls are:
+
+ symbol * b = create_b(n);
+ - create an empty block b with room for n symbols
+ b = increase_capacity(b, n);
+ - increase the capacity of block b by n symbols (b may change)
+ b2 = copy_b(b)
+ - copy block b into b2
+ lose_b(b);
+ - lose block b
+ b = move_to_b(b, n, p);
+ - set the data in b to be the n symbols at address p
+ b = add_to_b(b, n, p);
+ - add the n symbols at address p to the end of the data in b
+ SIZE(b)
+ - is the number of symbols in b
+ For example:
+
+ symbol * b = create_b(0);
+ { int i;
+ char p[10];
+ for (i = 0; i < 100; i++) {
+ sprintf(p, " %d", i);
+ add_s_to_b(b, p);
+ }
+ }
+
+ and b contains " 0 1 2 ... 99" spaced out as symbols.
+*/
+
+/* For a block b, SIZE(b) is the number of symbols so far written into it,
+ CAPACITY(b) the total number it can contain, so SIZE(b) <= CAPACITY(b).
+ In fact blocks have 1 extra character over the promised capacity so
+ they can be zero terminated by 'b[SIZE(b)] = 0;' without fear of
+ overwriting.
+*/
+
+extern symbol * create_b(int n) {
+ symbol * p = (symbol *) (HEAD + (char *) MALLOC(HEAD + (n + 1) * sizeof(symbol)));
+ CAPACITY(p) = n;
+ SIZE(p) = 0;
+ return p;
+}
+
+extern void report_b(FILE * out, symbol * p) {
+ int i;
+ for (i = 0; i < SIZE(p); i++) fprintf(out, "%c", p[i]);
+}
+
+extern void lose_b(symbol * p) {
+ if (p == 0) return;
+ FREE((char *) p - HEAD);
+}
+
+extern symbol * increase_capacity(symbol * p, int n) {
+ symbol * q = create_b(CAPACITY(p) + n + EXTENDER);
+ memmove(q, p, CAPACITY(p) * sizeof(symbol));
+ SIZE(q) = SIZE(p);
+ lose_b(p); return q;
+}
+
+extern symbol * move_to_b(symbol * p, int n, symbol * q) {
+ int x = n - CAPACITY(p);
+ if (x > 0) p = increase_capacity(p, x);
+ memmove(p, q, n * sizeof(symbol)); SIZE(p) = n; return p;
+}
+
+extern symbol * add_to_b(symbol * p, int n, symbol * q) {
+ int x = SIZE(p) + n - CAPACITY(p);
+ if (x > 0) p = increase_capacity(p, x);
+ memmove(p + SIZE(p), q, n * sizeof(symbol)); SIZE(p) += n; return p;
+}
+
+extern symbol * copy_b(symbol * p) {
+ int n = SIZE(p);
+ symbol * q = create_b(n);
+ move_to_b(q, n, p);
+ return q;
+}
+
+int space_count = 0;
+
+extern void * check_malloc(int n) {
+ space_count++;
+ return malloc(n);
+}
+
+extern void check_free(void * p) {
+ space_count--;
+ free(p);
+}
+
+/* To convert a block to a zero terminated string: */
+
+extern char * b_to_s(symbol * p) {
+ int n = SIZE(p);
+ char * s = (char *)malloc(n + 1);
+ {
+ int i;
+ for (i = 0; i < n; i++) {
+ if (p[i] > 255) {
+ printf("In b_to_s, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]);
+ exit(1);
+ }
+ s[i] = (char)p[i];
+ }
+ }
+ s[n] = 0;
+ return s;
+}
+
+/* To add a zero terminated string to a block. If p = 0 the
+ block is created. */
+
+extern symbol * add_s_to_b(symbol * p, const char * s) {
+ int n = strlen(s);
+ int k;
+ if (p == 0) p = create_b(n);
+ k = SIZE(p);
+ {
+ int x = k + n - CAPACITY(p);
+ if (x > 0) p = increase_capacity(p, x);
+ }
+ {
+ int i;
+ for (i = 0; i < n; i++) p[i + k] = s[i];
+ }
+ SIZE(p) += n;
+ return p;
+}
+
+/* The next section defines string handling capabilities in terms
+ of the lower level block handling capabilities of space.c */
+/* -------------------------------------------------------------*/
+
+struct str {
+ symbol * data;
+};
+
+/* Create a new string. */
+extern struct str * str_new() {
+
+ struct str * output = (struct str *) malloc(sizeof(struct str));
+ output->data = create_b(0);
+ return output;
+}
+
+/* Delete a string. */
+extern void str_delete(struct str * str) {
+
+ lose_b(str->data);
+ free(str);
+}
+
+/* Append a str to this str. */
+extern void str_append(struct str * str, struct str * add) {
+
+ symbol * q = add->data;
+ str->data = add_to_b(str->data, SIZE(q), q);
+}
+
+/* Append a character to this str. */
+extern void str_append_ch(struct str * str, char add) {
+
+ symbol q[1];
+ q[0] = add;
+ str->data = add_to_b(str->data, 1, q);
+}
+
+/* Append a low level block to a str. */
+extern void str_append_b(struct str * str, symbol * q) {
+
+ str->data = add_to_b(str->data, SIZE(q), q);
+}
+
+/* Append a (char *, null teminated) string to a str. */
+extern void str_append_string(struct str * str, const char * s) {
+
+ str->data = add_s_to_b(str->data, s);
+}
+
+/* Append an integer to a str. */
+extern void str_append_int(struct str * str, int i) {
+
+ char s[30];
+ sprintf(s, "%d", i);
+ str_append_string(str, s);
+}
+
+/* Clear a string */
+extern void str_clear(struct str * str) {
+
+ SIZE(str->data) = 0;
+}
+
+/* Set a string */
+extern void str_assign(struct str * str, char * s) {
+
+ str_clear(str);
+ str_append_string(str, s);
+}
+
+/* Copy a string. */
+extern struct str * str_copy(struct str * old) {
+
+ struct str * newstr = str_new();
+ str_append(newstr, old);
+ return newstr;
+}
+
+/* Get the data stored in this str. */
+extern symbol * str_data(struct str * str) {
+
+ return str->data;
+}
+
+/* Get the length of the str. */
+extern int str_len(struct str * str) {
+
+ return SIZE(str->data);
+}
+
+extern int get_utf8(const symbol * p, int * slot) {
+ int b0, b1;
+ b0 = *p++;
+ if (b0 < 0xC0) { /* 1100 0000 */
+ * slot = b0; return 1;
+ }
+ b1 = *p++;
+ if (b0 < 0xE0) { /* 1110 0000 */
+ * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
+ }
+ * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (*p & 0x3F); return 3;
+}
+
+extern int put_utf8(int ch, symbol * p) {
+ if (ch < 0x80) {
+ p[0] = ch; return 1;
+ }
+ if (ch < 0x800) {
+ p[0] = (ch >> 6) | 0xC0;
+ p[1] = (ch & 0x3F) | 0x80; return 2;
+ }
+ p[0] = (ch >> 12) | 0xE0;
+ p[1] = ((ch >> 6) & 0x3F) | 0x80;
+ p[2] = (ch & 0x3F) | 0x80; return 3;
+}
+
diff --git a/contrib/snowball/compiler/syswords.h b/contrib/snowball/compiler/syswords.h
new file mode 100644
index 000000000..917dd5751
--- /dev/null
+++ b/contrib/snowball/compiler/syswords.h
@@ -0,0 +1,84 @@
+static const struct system_word vocab[80+1] = {
+ { 0, (const byte *)"", 80+1},
+
+ { 1, (const byte *)"$", c_dollar },
+ { 1, (const byte *)"(", c_bra },
+ { 1, (const byte *)")", c_ket },
+ { 1, (const byte *)"*", c_multiply },
+ { 1, (const byte *)"+", c_plus },
+ { 1, (const byte *)"-", c_minus },
+ { 1, (const byte *)"/", c_divide },
+ { 1, (const byte *)"<", c_ls },
+ { 1, (const byte *)"=", c_assign },
+ { 1, (const byte *)">", c_gr },
+ { 1, (const byte *)"?", c_debug },
+ { 1, (const byte *)"[", c_leftslice },
+ { 1, (const byte *)"]", c_rightslice },
+ { 2, (const byte *)"!=", c_ne },
+ { 2, (const byte *)"*=", c_multiplyassign },
+ { 2, (const byte *)"+=", c_plusassign },
+ { 2, (const byte *)"-=", c_minusassign },
+ { 2, (const byte *)"->", c_sliceto },
+ { 2, (const byte *)"/*", c_comment2 },
+ { 2, (const byte *)"//", c_comment1 },
+ { 2, (const byte *)"/=", c_divideassign },
+ { 2, (const byte *)"<+", c_insert },
+ { 2, (const byte *)"<-", c_slicefrom },
+ { 2, (const byte *)"<=", c_le },
+ { 2, (const byte *)"==", c_eq },
+ { 2, (const byte *)"=>", c_assignto },
+ { 2, (const byte *)">=", c_ge },
+ { 2, (const byte *)"as", c_as },
+ { 2, (const byte *)"do", c_do },
+ { 2, (const byte *)"or", c_or },
+ { 3, (const byte *)"and", c_and },
+ { 3, (const byte *)"for", c_for },
+ { 3, (const byte *)"get", c_get },
+ { 3, (const byte *)"hex", c_hex },
+ { 3, (const byte *)"hop", c_hop },
+ { 3, (const byte *)"non", c_non },
+ { 3, (const byte *)"not", c_not },
+ { 3, (const byte *)"set", c_set },
+ { 3, (const byte *)"try", c_try },
+ { 4, (const byte *)"fail", c_fail },
+ { 4, (const byte *)"goto", c_goto },
+ { 4, (const byte *)"loop", c_loop },
+ { 4, (const byte *)"next", c_next },
+ { 4, (const byte *)"size", c_size },
+ { 4, (const byte *)"test", c_test },
+ { 4, (const byte *)"true", c_true },
+ { 5, (const byte *)"among", c_among },
+ { 5, (const byte *)"false", c_false },
+ { 5, (const byte *)"limit", c_limit },
+ { 5, (const byte *)"unset", c_unset },
+ { 6, (const byte *)"atmark", c_atmark },
+ { 6, (const byte *)"attach", c_attach },
+ { 6, (const byte *)"cursor", c_cursor },
+ { 6, (const byte *)"define", c_define },
+ { 6, (const byte *)"delete", c_delete },
+ { 6, (const byte *)"gopast", c_gopast },
+ { 6, (const byte *)"insert", c_insert },
+ { 6, (const byte *)"maxint", c_maxint },
+ { 6, (const byte *)"minint", c_minint },
+ { 6, (const byte *)"repeat", c_repeat },
+ { 6, (const byte *)"sizeof", c_sizeof },
+ { 6, (const byte *)"tomark", c_tomark },
+ { 7, (const byte *)"atleast", c_atleast },
+ { 7, (const byte *)"atlimit", c_atlimit },
+ { 7, (const byte *)"decimal", c_decimal },
+ { 7, (const byte *)"reverse", c_reverse },
+ { 7, (const byte *)"setmark", c_setmark },
+ { 7, (const byte *)"strings", c_strings },
+ { 7, (const byte *)"tolimit", c_tolimit },
+ { 8, (const byte *)"booleans", c_booleans },
+ { 8, (const byte *)"integers", c_integers },
+ { 8, (const byte *)"routines", c_routines },
+ { 8, (const byte *)"setlimit", c_setlimit },
+ { 9, (const byte *)"backwards", c_backwards },
+ { 9, (const byte *)"externals", c_externals },
+ { 9, (const byte *)"groupings", c_groupings },
+ { 9, (const byte *)"stringdef", c_stringdef },
+ { 9, (const byte *)"substring", c_substring },
+ { 12, (const byte *)"backwardmode", c_backwardmode },
+ { 13, (const byte *)"stringescapes", c_stringescapes }
+};
diff --git a/contrib/snowball/compiler/syswords2.h b/contrib/snowball/compiler/syswords2.h
new file mode 100644
index 000000000..eb8a91241
--- /dev/null
+++ b/contrib/snowball/compiler/syswords2.h
@@ -0,0 +1,13 @@
+ c_among = 4, c_and, c_as, c_assign, c_assignto, c_atleast,
+ c_atlimit, c_atmark, c_attach, c_backwardmode, c_backwards,
+ c_booleans, c_bra, c_comment1, c_comment2, c_cursor, c_debug,
+ c_decimal, c_define, c_delete, c_divide, c_divideassign, c_do,
+ c_dollar, c_eq, c_externals, c_fail, c_false, c_for, c_ge, c_get,
+ c_gopast, c_goto, c_gr, c_groupings, c_hex, c_hop, c_insert,
+ c_integers, c_ket, c_le, c_leftslice, c_limit, c_loop, c_ls,
+ c_maxint, c_minint, c_minus, c_minusassign, c_multiply,
+ c_multiplyassign, c_ne, c_next, c_non, c_not, c_or, c_plus,
+ c_plusassign, c_repeat, c_reverse, c_rightslice, c_routines,
+ c_set, c_setlimit, c_setmark, c_size, c_sizeof, c_slicefrom,
+ c_sliceto, c_stringdef, c_stringescapes, c_strings, c_substring,
+ c_test, c_tolimit, c_tomark, c_true, c_try, c_unset,
diff --git a/contrib/snowball/compiler/tokeniser.c b/contrib/snowball/compiler/tokeniser.c
new file mode 100644
index 000000000..3dae5f744
--- /dev/null
+++ b/contrib/snowball/compiler/tokeniser.c
@@ -0,0 +1,470 @@
+
+#include <stdio.h> /* stderr etc */
+#include <stdlib.h> /* malloc free */
+#include <string.h> /* strlen */
+#include <ctype.h> /* isalpha etc */
+#include "header.h"
+
+struct system_word {
+ int s_size; /* size of system word */
+ const byte * s; /* pointer to the system word */
+ int code; /* its internal code */
+};
+
+
+/* ASCII collating assumed in syswords.c */
+
+#include "syswords.h"
+
+static int smaller(int a, int b) { return a < b ? a : b; }
+
+extern symbol * get_input(symbol * p, char ** p_file) {
+
+ char * s = b_to_s(p);
+ {
+ FILE * input = fopen(s, "r");
+ if (input == 0) { free(s); return 0; }
+ *p_file = s;
+ {
+ symbol * u = create_b(STARTSIZE);
+ int size = 0;
+ repeat
+ { int ch = getc(input);
+ if (ch == EOF) break;
+ if (size >= CAPACITY(u)) u = increase_capacity(u, size/2);
+ u[size++] = ch;
+ }
+ fclose(input);
+ SIZE(u) = size; return u;
+ }
+ }
+}
+
+static void error(struct tokeniser * t, char * s1, int n, symbol * p, char * s2) {
+ if (t->error_count == 20) { fprintf(stderr, "... etc\n"); exit(1); }
+ fprintf(stderr, "%s:%d: ", t->file, t->line_number);
+ unless (s1 == 0) fprintf(stderr, "%s", s1);
+ unless (p == 0) {
+ int i;
+ for (i = 0; i < n; i++) fprintf(stderr, "%c", p[i]);
+ }
+ unless (s2 == 0) fprintf(stderr, "%s", s2);
+ fprintf(stderr, "\n");
+ t->error_count++;
+}
+
+static void error1(struct tokeniser * t, char * s) {
+ error(t, s, 0,0, 0);
+}
+
+static void error2(struct tokeniser * t, char * s) {
+ error(t, "unexpected end of text after ", 0,0, s);
+}
+
+static int compare_words(int m, symbol * p, int n, const byte * q) {
+ unless (m == n) return m - n;
+ {
+ int i; for (i = 0; i < n; i++) {
+ int diff = p[i] - q[i];
+ unless (diff == 0) return diff;
+ }
+ }
+ return 0;
+}
+
+static int find_word(int n, symbol * p) {
+ int i = 0; int j = vocab->code;
+ repeat {
+ int k = i + (j - i)/2;
+ const struct system_word * w = vocab + k;
+ int diff = compare_words(n, p, w->s_size, w->s);
+ if (diff == 0) return w->code;
+ if (diff < 0) j = k; else i = k;
+ if (j - i == 1) break;
+ }
+ return -1;
+}
+
+static int get_number(int n, symbol * p) {
+ int x = 0;
+ int i; for (i = 0; i < n; i++) x = 10*x + p[i] - '0';
+ return x;
+}
+
+static int eq_s(struct tokeniser * t, char * s) {
+ int l = strlen(s);
+ if (SIZE(t->p) - t->c < l) return false;
+ {
+ int i;
+ for (i = 0; i < l; i++) if (t->p[t->c + i] != s[i]) return false;
+ }
+ t->c += l; return true;
+}
+
+static int white_space(struct tokeniser * t, int ch) {
+ switch (ch) {
+ case '\n': t->line_number++;
+ case '\r':
+ case '\t':
+ case ' ': return true;
+ }
+ return false;
+}
+
+static symbol * find_in_m(struct tokeniser * t, int n, symbol * p) {
+ struct m_pair * q = t->m_pairs;
+ repeat {
+ if (q == 0) return 0;
+ {
+ symbol * name = q->name;
+ if (n == SIZE(name) && memcmp(name, p, n * sizeof(symbol)) == 0) return q->value;
+ }
+ q = q->next;
+ }
+}
+
+static int read_literal_string(struct tokeniser * t, int c) {
+ symbol * p = t->p;
+ int ch;
+ SIZE(t->b) = 0;
+ repeat {
+ if (c >= SIZE(p)) { error2(t, "'"); return c; }
+ ch = p[c];
+ if (ch == '\n') { error1(t, "string not terminated"); return c; }
+ c++;
+ if (ch == t->m_start) {
+ int c0 = c;
+ int newlines = false; /* no newlines as yet */
+ int black_found = false; /* no printing chars as yet */
+ repeat {
+ if (c >= SIZE(p)) { error2(t, "'"); return c; }
+ ch = p[c]; c++;
+ if (ch == t->m_end) break;
+ unless (white_space(t, ch)) black_found = true;
+ if (ch == '\n') newlines = true;
+ if (newlines && black_found) {
+ error1(t, "string not terminated");
+ return c;
+ }
+ }
+ unless (newlines) {
+ int n = c - c0 - 1; /* macro size */
+ int firstch = p[c0];
+ symbol * q = find_in_m(t, n, p + c0);
+ if (q == 0) {
+ if (n == 1 && (firstch == '\'' || firstch == t->m_start))
+ t->b = add_to_b(t->b, 1, p + c0);
+ else
+ error(t, "string macro '", n, p + c0, "' undeclared");
+ } else
+ t->b = add_to_b(t->b, SIZE(q), q);
+ }
+ } else {
+ if (ch == '\'') return c;
+ t->b = add_to_b(t->b, 1, p + c - 1);
+ }
+ }
+}
+
+static int next_token(struct tokeniser * t) {
+ symbol * p = t->p;
+ int c = t->c;
+ int ch;
+ int code = -1;
+ repeat {
+ if (c >= SIZE(p)) { t->c = c; return -1; }
+ ch = p[c];
+ if (white_space(t, ch)) { c++; continue; }
+ if (isalpha(ch)) {
+ int c0 = c;
+ while (c < SIZE(p) && (isalnum(p[c]) || p[c] == '_')) c++;
+ code = find_word(c - c0, p + c0);
+ if (code < 0) {
+ t->b = move_to_b(t->b, c - c0, p + c0);
+ code = c_name;
+ }
+ } else
+ if (isdigit(ch)) {
+ int c0 = c;
+ while (c < SIZE(p) && isdigit(p[c])) c++;
+ t->number = get_number(c - c0, p + c0);
+ code = c_number;
+ } else
+ if (ch == '\'') {
+ c = read_literal_string(t, c + 1);
+ code = c_literalstring;
+ } else
+ {
+ int lim = smaller(2, SIZE(p) - c);
+ int i;
+ for (i = lim; i > 0; i--) {
+ code = find_word(i, p + c);
+ if (code >= 0) { c += i; break; }
+ }
+ }
+ if (code >= 0) {
+ t->c = c;
+ return code;
+ }
+ error(t, "'", 1, p + c, "' unknown");
+ c++;
+ continue;
+ }
+}
+
+static int next_char(struct tokeniser * t) {
+ if (t->c >= SIZE(t->p)) return -1;
+ return t->p[t->c++];
+}
+
+static int next_real_char(struct tokeniser * t) {
+ repeat {
+ int ch = next_char(t);
+ if (white_space(t, ch)) continue;
+ return ch;
+ }
+}
+
+static void read_chars(struct tokeniser * t) {
+ int ch = next_real_char(t);
+ if (ch < 0) { error2(t, "stringdef"); return; }
+ {
+ int c0 = t->c-1;
+ repeat {
+ ch = next_char(t);
+ if (white_space(t, ch) || ch < 0) break;
+ }
+ t->b2 = move_to_b(t->b2, t->c - c0 - 1, t->p + c0);
+ }
+}
+
+static int decimal_to_num(int ch) {
+ if ('0' <= ch && ch <= '9') return ch - '0';
+ return -1;
+}
+
+static int hex_to_num(int ch) {
+ if ('0' <= ch && ch <= '9') return ch - '0';
+ if ('a' <= ch && ch <= 'f') return ch - 'a' + 10;
+ return -1;
+}
+
+static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) {
+ int c = 0; int d = 0;
+ repeat {
+ while (c < SIZE(p) && p[c] == ' ') c++;
+ if (c == SIZE(p)) break;
+ {
+ int number = 0;
+ repeat {
+ int ch = p[c];
+ if (c == SIZE(p) || ch == ' ') break;
+ if (base == 10) {
+ ch = decimal_to_num(ch);
+ if (ch < 0) {
+ error1(t, "decimal string contains non-digits");
+ return;
+ }
+ } else {
+ ch = hex_to_num(tolower(ch));
+ if (ch < 0) {
+ error1(t, "hex string contains non-hex characters");
+ return;
+ }
+ }
+ number = base * number + ch;
+ c++;
+ }
+ if (t->widechars || t->utf8) {
+ unless (0 <= number && number <= 0xffff) {
+ error1(t, "character values exceed 64K");
+ return;
+ }
+ } else {
+ unless (0 <= number && number <= 0xff) {
+ error1(t, "character values exceed 256");
+ return;
+ }
+ }
+ if (t->utf8)
+ d += put_utf8(number, p + d);
+ else
+ p[d++] = number;
+ }
+ }
+ SIZE(p) = d;
+}
+
+extern int read_token(struct tokeniser * t) {
+ symbol * p = t->p;
+ int held = t->token_held;
+ t->token_held = false;
+ if (held) return t->token;
+ repeat {
+ int code = next_token(t);
+ switch (code) {
+ case c_comment1: /* slash-slash comment */
+ while (t->c < SIZE(p) && p[t->c] != '\n') t->c++;
+ continue;
+ case c_comment2: /* slash-star comment */
+ repeat {
+ if (t->c >= SIZE(p)) {
+ error1(t, "/* comment not terminated");
+ t->token = -1;
+ return -1;
+ }
+ if (p[t->c] == '\n') t->line_number++;
+ if (eq_s(t, "*/")) break;
+ t->c++;
+ }
+ continue;
+ case c_stringescapes:
+ {
+ int ch1 = next_real_char(t);
+ int ch2 = next_real_char(t);
+ if (ch2 < 0)
+ { error2(t, "stringescapes"); continue; }
+ if (ch1 == '\'')
+ { error1(t, "first stringescape cannot be '"); continue; }
+ t->m_start = ch1;
+ t->m_end = ch2;
+ }
+ continue;
+ case c_stringdef:
+ {
+ int base = 0;
+ read_chars(t);
+ code = read_token(t);
+ if (code == c_hex) { base = 16; code = read_token(t); } else
+ if (code == c_decimal) { base = 10; code = read_token(t); }
+ unless (code == c_literalstring)
+ { error1(t, "string omitted after stringdef"); continue; }
+ if (base > 0) convert_numeric_string(t, t->b, base);
+ { NEW(m_pair, q);
+ q->next = t->m_pairs;
+ q->name = copy_b(t->b2);
+ q->value = copy_b(t->b);
+ t->m_pairs = q;
+ }
+ }
+ continue;
+ case c_get:
+ code = read_token(t);
+ unless (code == c_literalstring) {
+ error1(t, "string omitted after get"); continue;
+ }
+ t->get_depth++;
+ if (t->get_depth > 10) {
+ fprintf(stderr, "get directives go 10 deep. Looping?\n");
+ exit(1);
+ }
+ {
+ char * file;
+ NEW(input, q);
+ symbol * u = get_input(t->b, &file);
+ if (u == 0) {
+ struct include * r = t->includes;
+ until (r == 0) {
+ symbol * b = copy_b(r->b);
+ b = add_to_b(b, SIZE(t->b), t->b);
+ u = get_input(b, &file);
+ lose_b(b);
+ unless (u == 0) break;
+ r = r->next;
+ }
+ }
+ if (u == 0) {
+ error(t, "Can't get '", SIZE(t->b), t->b, "'");
+ exit(1);
+ }
+ memmove(q, t, sizeof(struct input));
+ t->next = q;
+ t->p = u;
+ t->c = 0;
+ t->file = file;
+ t->line_number = 1;
+ }
+ p = t->p;
+ continue;
+ case -1:
+ unless (t->next == 0) {
+ lose_b(p);
+ {
+ struct input * q = t->next;
+ memmove(t, q, sizeof(struct input)); p = t->p;
+ FREE(q);
+ }
+ t->get_depth--;
+ continue;
+ }
+ /* drop through */
+ default:
+ t->previous_token = t->token;
+ t->token = code;
+ return code;
+ }
+ }
+}
+
+extern const char * name_of_token(int code) {
+ int i;
+ for (i = 1; i < vocab->code; i++)
+ if ((vocab + i)->code == code) return (const char *)(vocab + i)->s;
+ switch (code) {
+ case c_mathassign: return "=";
+ case c_name: return "name";
+ case c_number: return "number";
+ case c_literalstring:return "literal";
+ case c_neg: return "neg";
+ case c_grouping: return "grouping";
+ case c_call: return "call";
+ case c_booltest: return "Boolean test";
+ case -2: return "start of text";
+ case -1: return "end of text";
+ default: return "?";
+ }
+}
+
+extern struct tokeniser * create_tokeniser(symbol * p, char * file) {
+ NEW(tokeniser, t);
+ t->next = 0;
+ t->p = p;
+ t->c = 0;
+ t->file = file;
+ t->line_number = 1;
+ t->b = create_b(0);
+ t->b2 = create_b(0);
+ t->m_start = -1;
+ t->m_pairs = 0;
+ t->get_depth = 0;
+ t->error_count = 0;
+ t->token_held = false;
+ t->token = -2;
+ t->previous_token = -2;
+ return t;
+}
+
+extern void close_tokeniser(struct tokeniser * t) {
+ lose_b(t->b);
+ lose_b(t->b2);
+ {
+ struct m_pair * q = t->m_pairs;
+ until (q == 0) {
+ struct m_pair * q_next = q->next;
+ lose_b(q->name);
+ lose_b(q->value);
+ FREE(q);
+ q = q_next;
+ }
+ }
+ {
+ struct input * q = t->next;
+ until (q == 0) {
+ struct input * q_next = q->next;
+ FREE(q);
+ q = q_next;
+ }
+ }
+ free(t->file);
+ FREE(t);
+}
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-21 19:04:56 +0000