diff options
Diffstat (limited to 'contrib/snowball/compiler')
| -rw-r--r-- | contrib/snowball/compiler/analyser.c | 866 | ||||
| -rw-r--r-- | contrib/snowball/compiler/driver.c | 466 | ||||
| -rw-r--r-- | contrib/snowball/compiler/generator.c | 1158 | ||||
| -rw-r--r-- | contrib/snowball/compiler/generator_java.c | 1452 | ||||
| -rw-r--r-- | contrib/snowball/compiler/header.h | 215 | ||||
| -rw-r--r-- | contrib/snowball/compiler/space.c | 72 | ||||
| -rw-r--r-- | contrib/snowball/compiler/syswords.h | 6 | ||||
| -rw-r--r-- | contrib/snowball/compiler/syswords2.h | 4 | ||||
| -rw-r--r-- | contrib/snowball/compiler/tokeniser.c | 407 |
9 files changed, 2201 insertions, 2445 deletions
diff --git a/contrib/snowball/compiler/analyser.c b/contrib/snowball/compiler/analyser.c index ed712f64f..dffa5552d 100644 --- a/contrib/snowball/compiler/analyser.c +++ b/contrib/snowball/compiler/analyser.c @@ -4,31 +4,53 @@ #include <string.h> /* memmove */ #include "header.h" +typedef enum { + e_token_omitted = 0, + e_unexpected_token = 1, + e_string_omitted = 2, + e_unexpected_token_in_among = 3, + /* For codes above here, report "after " t->previous_token after the error. */ + e_unresolved_substring = 14, + e_not_allowed_inside_reverse = 15, + e_empty_grouping = 16, + e_already_backwards = 17, + e_empty_among = 18, + e_adjacent_bracketed_in_among = 19, + e_substring_preceded_by_substring = 20, + /* For codes below here, tokeniser->b is printed before the error. */ + e_redeclared = 30, + e_undeclared = 31, + e_declared_as_different_mode = 32, + e_not_of_type_x = 33, + e_not_of_type_string_or_integer = 34, + e_misplaced = 35, + e_redefined = 36, + e_misused = 37 +} error_code; + /* recursive usage: */ static void read_program_(struct analyser * a, int terminator); static struct node * read_C(struct analyser * a); -static struct node * C_style(struct analyser * a, char * s, int token); - +static struct node * C_style(struct analyser * a, const char * s, int token); -static void fault(int n) { fprintf(stderr, "fault %d\n", n); exit(1); } static void print_node_(struct node * p, int n, const char * s) { int i; for (i = 0; i < n; i++) fputs(i == n - 1 ? s : " ", stdout); printf("%s ", name_of_token(p->type)); - unless (p->name == 0) report_b(stdout, p->name->b); - unless (p->literalstring == 0) { + if (p->name) report_b(stdout, p->name->b); + if (p->literalstring) { printf("'"); report_b(stdout, p->literalstring); printf("'"); } printf("\n"); - unless (p->AE == 0) print_node_(p->AE, n+1, "# "); - unless (p->left == 0) print_node_(p->left, n+1, " "); - unless (p->right == 0) print_node_(p->right, n, " "); - if (p->aux != 0) print_node_(p->aux, n+1, "@ "); + if (p->AE) print_node_(p->AE, n+1, "# "); + if (p->left) print_node_(p->left, n+1, " "); + if (p->aux) print_node_(p->aux, n+1, "@ "); + if (p->right) print_node_(p->right, n, " "); } extern void print_program(struct analyser * a) { @@ -52,22 +74,37 @@ static struct node * new_node(struct analyser * a, int type) { static const char * name_of_mode(int n) { switch (n) { - default: fault(0); - case m_backward: return "string backward"; - case m_forward: return "string forward"; - /* case m_integer: return "integer"; */ + case m_backward: return "string backward"; + case m_forward: return "string forward"; + /* case m_integer: return "integer"; */ } + fprintf(stderr, "Invalid mode %d in name_of_mode()\n", n); + exit(1); } static const char * name_of_type(int n) { switch (n) { - default: fault(1); - case 's': return "string"; - case 'i': return "integer"; - case 'r': return "routine"; - case 'R': return "routine or grouping"; - case 'g': return "grouping"; + case 's': return "string"; + case 'i': return "integer"; + case 'r': return "routine"; + case 'R': return "routine or grouping"; + case 'g': return "grouping"; + } + fprintf(stderr, "Invalid type %d in name_of_type()\n", n); + exit(1); +} + +static const char * name_of_name_type(int code) { + switch (code) { + case t_string: return "string"; + case t_boolean: return "boolean"; + case t_integer: return "integer"; + case t_routine: return "routine"; + case t_external: return "external"; + case t_grouping: return "grouping"; } + fprintf(stderr, "Invalid type code %d in name_of_name_type()\n", code); + exit(1); } static void count_error(struct analyser * a) { @@ -76,86 +113,78 @@ static void count_error(struct analyser * a) { t->error_count++; } -static void error2(struct analyser * a, int n, int x) { +static void error2(struct analyser * a, error_code n, int x) { struct tokeniser * t = a->tokeniser; count_error(a); fprintf(stderr, "%s:%d: ", t->file, t->line_number); - if (n >= 30) report_b(stderr, t->b); + if ((int)n >= (int)e_redeclared) report_b(stderr, t->b); switch (n) { - case 0: + case e_token_omitted: fprintf(stderr, "%s omitted", name_of_token(t->omission)); break; - case 3: + case e_unexpected_token_in_among: fprintf(stderr, "in among(...), "); - case 1: + /* fall through */ + case e_unexpected_token: fprintf(stderr, "unexpected %s", name_of_token(t->token)); if (t->token == c_number) fprintf(stderr, " %d", t->number); if (t->token == c_name) { fprintf(stderr, " "); report_b(stderr, t->b); } break; - case 2: + case e_string_omitted: fprintf(stderr, "string omitted"); break; - case 14: + case e_unresolved_substring: fprintf(stderr, "unresolved substring on line %d", x); break; - case 15: + case e_not_allowed_inside_reverse: fprintf(stderr, "%s not allowed inside reverse(...)", name_of_token(t->token)); break; - case 16: + case e_empty_grouping: fprintf(stderr, "empty grouping"); break; - case 17: + case e_already_backwards: fprintf(stderr, "backwards used when already in this mode"); break; - case 18: + case e_empty_among: fprintf(stderr, "empty among(...)"); break; - case 19: + case e_adjacent_bracketed_in_among: fprintf(stderr, "two adjacent bracketed expressions in among(...)"); break; - case 20: + case e_substring_preceded_by_substring: fprintf(stderr, "substring preceded by another substring on line %d", x); break; - case 30: + case e_redeclared: fprintf(stderr, " re-declared"); break; - case 31: + case e_undeclared: fprintf(stderr, " undeclared"); break; - case 32: + case e_declared_as_different_mode: fprintf(stderr, " declared as %s mode; used as %s mode", name_of_mode(a->mode), name_of_mode(x)); break; - case 33: + case e_not_of_type_x: fprintf(stderr, " not of type %s", name_of_type(x)); break; - case 34: + case e_not_of_type_string_or_integer: fprintf(stderr, " not of type string or integer"); break; - case 35: + case e_misplaced: fprintf(stderr, " misplaced"); break; - case 36: + case e_redefined: fprintf(stderr, " redefined"); break; - case 37: + case e_misused: fprintf(stderr, " mis-used as %s mode", name_of_mode(x)); break; - default: - fprintf(stderr, " error %d", n); break; - } - if (n <= 13 && t->previous_token > 0) + if ((int)n < (int)e_unresolved_substring && t->previous_token > 0) fprintf(stderr, " after %s", name_of_token(t->previous_token)); fprintf(stderr, "\n"); } -static void error(struct analyser * a, int n) { error2(a, n, 0); } - -static void error3(struct analyser * a, struct node * p, symbol * b) { - count_error(a); - fprintf(stderr, "%s:%d: among(...) has repeated string '", a->tokeniser->file, p->line_number); - report_b(stderr, b); - fprintf(stderr, "'\n"); -} +static void error(struct analyser * a, error_code n) { error2(a, n, 0); } static void error4(struct analyser * a, struct name * q) { count_error(a); + fprintf(stderr, "%s:%d: ", a->tokeniser->file, q->used->line_number); report_b(stderr, q->b); fprintf(stderr, " undefined\n"); } static void omission_error(struct analyser * a, int n) { a->tokeniser->omission = n; - error(a, 0); + error(a, e_token_omitted); } static int check_token(struct analyser * a, int code) { @@ -169,71 +198,120 @@ static int get_token(struct analyser * a, int code) { read_token(t); { int x = check_token(a, code); - unless (x) t->token_held = true; + if (!x) t->token_held = true; return x; } } static struct name * look_for_name(struct analyser * a) { - struct name * p = a->names; symbol * q = a->tokeniser->b; - repeat { - if (p == 0) return 0; - { symbol * b = p->b; - int n = SIZE(b); - if (n == SIZE(q) && memcmp(q, b, n * sizeof(symbol)) == 0) { - p->referenced = true; - return p; - } + struct name * p; + for (p = a->names; p; p = p->next) { + symbol * b = p->b; + int n = SIZE(b); + if (n == SIZE(q) && memcmp(q, b, n * sizeof(symbol)) == 0) { + p->referenced = true; + return p; } - p = p->next; } + return 0; } static struct name * find_name(struct analyser * a) { struct name * p = look_for_name(a); - if (p == 0) error(a, 31); + if (p == 0) error(a, e_undeclared); return p; } static void check_routine_mode(struct analyser * a, struct name * p, int mode) { if (p->mode < 0) p->mode = mode; else - unless (p->mode == mode) error2(a, 37, mode); + if (p->mode != mode) error2(a, e_misused, mode); } static void check_name_type(struct analyser * a, struct name * p, int type) { switch (type) { - case 's': if (p->type == t_string) return; break; - case 'i': if (p->type == t_integer) return; break; - case 'b': if (p->type == t_boolean) return; break; - case 'R': if (p->type == t_grouping) return; - case 'r': if (p->type == t_routine || p->type == t_external) return; break; - case 'g': if (p->type == t_grouping) return; break; + case 's': + if (p->type == t_string) return; + break; + case 'i': + if (p->type == t_integer) return; + break; + case 'b': + if (p->type == t_boolean) return; + break; + case 'R': + if (p->type == t_grouping) return; + /* FALLTHRU */ + case 'r': + if (p->type == t_routine || p->type == t_external) return; + break; + case 'g': + if (p->type == t_grouping) return; + break; } - error2(a, 33, type); + error2(a, e_not_of_type_x, type); } static void read_names(struct analyser * a, int type) { struct tokeniser * t = a->tokeniser; - unless (get_token(a, c_bra)) return; - repeat { - if (read_token(t) != c_name) break; - if (look_for_name(a) != 0) error(a, 30); else { - NEW(name, p); - p->b = copy_b(t->b); - p->type = type; - p->mode = -1; /* routines, externals */ - p->count = a->name_count[type]; - p->referenced = false; - p->used = false; - p->grouping = 0; - p->definition = 0; - a->name_count[type] ++; - p->next = a->names; - a->names = p; - } - } - unless (check_token(a, c_ket)) t->token_held = true; + if (!get_token(a, c_bra)) return; + while (true) { + int token = read_token(t); + switch (token) { + case c_len: { + /* Context-sensitive token - once declared as a name, it loses + * its special meaning, for compatibility with older versions + * of snowball. + */ + static const symbol c_len_lit[] = { + 'l', 'e', 'n' + }; + MOVE_TO_B(t->b, c_len_lit); + goto handle_as_name; + } + case c_lenof: { + /* Context-sensitive token - once declared as a name, it loses + * its special meaning, for compatibility with older versions + * of snowball. + */ + static const symbol c_lenof_lit[] = { + 'l', 'e', 'n', 'o', 'f' + }; + MOVE_TO_B(t->b, c_lenof_lit); + goto handle_as_name; + } + case c_name: +handle_as_name: + if (look_for_name(a) != 0) error(a, e_redeclared); else { + NEW(name, p); + p->b = copy_b(t->b); + p->type = type; + p->mode = -1; /* routines, externals */ + /* We defer assigning counts until after we've eliminated + * variables whose values are never used. */ + p->count = -1; + p->referenced = false; + p->used_in_among = false; + p->used = 0; + p->value_used = false; + p->initialised = false; + p->used_in_definition = false; + p->local_to = 0; + p->grouping = 0; + p->definition = 0; + p->declaration_line_number = t->line_number; + p->next = a->names; + a->names = p; + if (token != c_name) { + disable_token(t, token); + } + } + break; + default: + if (!check_token(a, c_ket)) t->token_held = true; + return; + } + } } static symbol * new_literalstring(struct analyser * a) { @@ -259,7 +337,7 @@ static int read_AE_test(struct analyser * a) { case c_ge: case c_ls: case c_le: return t->token; - default: error(a, 1); t->token_held = true; return c_eq; + default: error(a, e_unexpected_token); t->token_held = true; return c_eq; } } @@ -271,11 +349,23 @@ static int binding(int t) { } } +static void mark_used_in(struct analyser * a, struct name * q, struct node * p) { + if (!q->used) { + q->used = p; + q->local_to = a->program_end->name; + } else if (q->local_to) { + if (q->local_to != a->program_end->name) { + /* Used in more than one routine/external. */ + q->local_to = NULL; + } + } +} + static void name_to_node(struct analyser * a, struct node * p, int type) { struct name * q = find_name(a); - unless (q == 0) { + if (q) { check_name_type(a, q, type); - q->used = true; + mark_used_in(a, q, p); } p->name = q; } @@ -286,8 +376,18 @@ static struct node * read_AE(struct analyser * a, int B) { struct node * q; switch (read_token(t)) { case c_minus: /* monadic */ + q = read_AE(a, 100); + if (q->type == c_neg) { + /* Optimise away double negation, which avoids generators + * having to worry about generating "--" (decrement operator + * in many languages). + */ + p = q->right; + /* Don't free q, it's in the linked list a->nodes. */ + break; + } p = new_node(a, c_neg); - p->right = read_AE(a, 100); + p->right = q; break; case c_bra: p = read_AE(a, 0); @@ -296,11 +396,15 @@ static struct node * read_AE(struct analyser * a, int B) { case c_name: p = new_node(a, c_name); name_to_node(a, p, 'i'); + if (p->name) p->name->value_used = true; break; case c_maxint: case c_minint: + a->int_limits_used = true; + /* fall through */ case c_cursor: case c_limit: + case c_len: case c_size: p = new_node(a, t->token); break; @@ -308,18 +412,19 @@ static struct node * read_AE(struct analyser * a, int B) { p = new_node(a, c_number); p->number = t->number; break; + case c_lenof: case c_sizeof: - p = C_style(a, "s", c_sizeof); + p = C_style(a, "s", t->token); break; default: - error(a, 1); + error(a, e_unexpected_token); t->token_held = true; return 0; } - repeat { + while (true) { int token = read_token(t); int b = binding(token); - unless (binding(token) > B) { + if (binding(token) <= B) { t->token_held = true; return p; } @@ -335,14 +440,11 @@ static struct node * read_C_connection(struct analyser * a, struct node * q, int struct node * p = new_node(a, op); struct node * p_end = q; p->left = q; - repeat { + do { q = read_C(a); p_end->right = q; p_end = q; - if (read_token(t) != op) { - t->token_held = true; - break; - } - } + } while (read_token(t) == op); + t->token_held = true; return p; } @@ -350,14 +452,14 @@ static struct node * read_C_list(struct analyser * a) { struct tokeniser * t = a->tokeniser; struct node * p = new_node(a, c_bra); struct node * p_end = 0; - repeat { + while (true) { int token = read_token(t); if (token == c_ket) return p; if (token < 0) { omission_error(a, c_ket); return p; } t->token_held = true; { struct node * q = read_C(a); - repeat { + while (true) { token = read_token(t); if (token != c_and && token != c_or) { t->token_held = true; @@ -371,10 +473,10 @@ static struct node * read_C_list(struct analyser * a) { } } -static struct node * C_style(struct analyser * a, char * s, int token) { +static struct node * C_style(struct analyser * a, const char * s, int token) { int i; struct node * p = new_node(a, token); - for (i = 0; s[i] != 0; i++) switch(s[i]) { + for (i = 0; s[i] != 0; i++) switch (s[i]) { case 'C': p->left = read_C(a); continue; case 'D': @@ -388,7 +490,7 @@ static struct node * C_style(struct analyser * a, char * s, int token) { int str_token = read_token(a->tokeniser); if (str_token == c_name) name_to_node(a, p, 's'); else if (str_token == c_literalstring) p->literalstring = new_literalstring(a); - else error(a, 2); + else error(a, e_string_omitted); } continue; case 'b': @@ -408,7 +510,7 @@ static struct node * read_literalstring(struct analyser * a) { static void reverse_b(symbol * b) { int i = 0; int j = SIZE(b) - 1; - until (i >= j) { + while (i < j) { int ch1 = b[i]; int ch2 = b[j]; b[i++] = ch2; b[j--] = ch1; } @@ -423,7 +525,74 @@ static int compare_amongvec(const void *pv, const void *qv) { int i; for (i = 0; i < smaller_size; i++) if (b_p[i] != b_q[i]) return b_p[i] - b_q[i]; - return p_size - q_size; + if (p_size - q_size) + return p_size - q_size; + return p->line_number - q->line_number; +} + +#define PTR_NULL_CHECK(P, Q) do {\ + if ((Q) == NULL) {\ + if ((P) != NULL) return 1;\ + } else {\ + if ((P) == NULL) return -1;\ + }\ + } while (0) + +static int compare_node(const struct node *p, const struct node *q) { + PTR_NULL_CHECK(p, q); + if (q == NULL) { + /* p must be NULL too. */ + return 0; + } + + if (p->type != q->type) return p->type > q->type ? 1 : -1; + if (p->mode != q->mode) return p->mode > q->mode ? 1 : -1; + if (p->type == c_number) { + if (p->number != q->number) + return p->number > q->number ? 1 : -1; + } + + PTR_NULL_CHECK(p->left, q->left); + if (p->left) { + int r = compare_node(p->left, q->left); + if (r != 0) return r; + } + + PTR_NULL_CHECK(p->AE, q->AE); + if (p->AE) { + int r = compare_node(p->AE, q->AE); + if (r != 0) return r; + } + + PTR_NULL_CHECK(p->aux, q->aux); + if (p->aux) { + int r = compare_node(p->aux, q->aux); + if (r != 0) return r; + } + + PTR_NULL_CHECK(p->name, q->name); + if (p->name) { + int r; + if (SIZE(p->name->b) != SIZE(q->name->b)) { + return SIZE(p->name->b) - SIZE(q->name->b); + } + r = memcmp(p->name->b, q->name->b, + SIZE(p->name->b) * sizeof(symbol)); + if (r != 0) return r; + } + + PTR_NULL_CHECK(p->literalstring, q->literalstring); + if (p->literalstring) { + int r; + if (SIZE(p->literalstring) != SIZE(q->literalstring)) { + return SIZE(p->literalstring) - SIZE(q->literalstring); + } + r = memcmp(p->literalstring, q->literalstring, + SIZE(p->literalstring) * sizeof(symbol)); + if (r != 0) return r; + } + + return compare_node(p->right, q->right); } static void make_among(struct analyser * a, struct node * p, struct node * substring) { @@ -443,38 +612,84 @@ static void make_among(struct analyser * a, struct node * p, struct node * subst x->next = 0; x->b = v; x->number = a->among_count++; + x->function_count = 0; x->starter = 0; + x->nocommand_count = 0; + x->amongvar_needed = false; if (q->type == c_bra) { x->starter = q; q = q->right; } - until (q == 0) { + while (q) { if (q->type == c_literalstring) { symbol * b = q->literalstring; w1->b = b; /* pointer to case string */ - w1->p = 0; /* pointer to corresponding case expression */ + w1->action = NULL; /* action gets filled in below */ + w1->line_number = q->line_number; w1->size = SIZE(b); /* number of characters in string */ w1->i = -1; /* index of longest substring */ w1->result = -1; /* number of corresponding case expression */ - w1->function = q->left == 0 ? 0 : q->left->name; - unless (w1->function == 0) - check_routine_mode(a, w1->function, direction); + if (q->left) { + struct name * function = q->left->name; + w1->function = function; + function->used_in_among = true; + check_routine_mode(a, function, direction); + x->function_count++; + } else { + w1->function = 0; + } w1++; - } - else - if (q->left == 0) /* empty command: () */ + } else if (q->left == 0) { + /* empty command: () */ w0 = w1; - else { - until (w0 == w1) { - w0->p = q; - w0->result = result; + } else { + /* Check for previous action which is the same as this one and use + * the same action code if we find one. + */ + int among_result = -1; + struct amongvec * w; + for (w = v; w < w0; ++w) { + if (w->action && compare_node(w->action->left, q->left) == 0) { + if (w->result <= 0) { + printf("Among code %d isn't positive\n", w->result); + exit(1); + } + among_result = w->result; + break; + } + } + if (among_result < 0) { + among_result = result++; + } + + while (w0 != w1) { + w0->action = q; + w0->result = among_result; w0++; } - result++; } q = q->right; } - unless (w1-v == p->number) { fprintf(stderr, "oh! %d %d\n", (int)(w1-v), p->number); exit(1); } - if (backward) for (w0 = v; w0 < w1; w0++) reverse_b(w0->b); + if (w1-v != p->number) { fprintf(stderr, "oh! %d %d\n", (int)(w1-v), p->number); exit(1); } + x->command_count = result - 1; + { + NEWVEC(node*, commands, x->command_count); + memset(commands, 0, x->command_count * sizeof(struct node*)); + for (w0 = v; w0 < w1; w0++) { + if (w0->result > 0) { + /* result == -1 when there's no command. */ + if (w0->result > x->command_count) { + fprintf(stderr, "More among codes than expected\n"); + exit(1); + } + if (!commands[w0->result - 1]) + commands[w0->result - 1] = w0->action; + } else { + ++x->nocommand_count; + } + if (backward) reverse_b(w0->b); + } + x->commands = commands; + } qsort(v, w1 - v, sizeof(struct amongvec), compare_amongvec); /* the following loop is O(n squared) */ @@ -494,15 +709,30 @@ static void make_among(struct analyser * a, struct node * p, struct node * subst for (w0 = v; w0 < w1 - 1; w0++) if (w0->size == (w0 + 1)->size && - memcmp(w0->b, (w0 + 1)->b, w0->size * sizeof(symbol)) == 0) error3(a, p, w0->b); + memcmp(w0->b, (w0 + 1)->b, w0->size * sizeof(symbol)) == 0) { + count_error(a); + fprintf(stderr, "%s:%d: among(...) has repeated string '", + a->tokeniser->file, (w0 + 1)->line_number); + report_b(stderr, (w0 + 1)->b); + fprintf(stderr, "'\n"); + count_error(a); + fprintf(stderr, "%s:%d: previously seen here\n", + a->tokeniser->file, w0->line_number); + } x->literalstring_count = p->number; - x->command_count = result - 1; p->among = x; x->substring = substring; if (substring != 0) substring->among = x; - unless (x->command_count == 0 && x->starter == 0) a->amongvar_needed = true; + if (x->command_count > 1 || + (x->command_count == 1 && x->nocommand_count > 0) || + x->starter != 0) { + /* We need to set among_var rather than just checking if find_among*() + * returns zero or not. + */ + x->amongvar_needed = a->amongvar_needed = true; + } } static struct node * read_among(struct analyser * a) { @@ -514,8 +744,8 @@ static struct node * read_among(struct analyser * a) { a->substring = 0; p->number = 0; /* counts the number of literals */ - unless (get_token(a, c_bra)) return p; - repeat { + if (!get_token(a, c_bra)) return p; + while (true) { struct node * q; int token = read_token(t); switch (token) { @@ -529,12 +759,14 @@ static struct node * read_among(struct analyser * a) { else t->token_held = true; p->number++; break; case c_bra: - if (previous_token == c_bra) error(a, 19); + if (previous_token == c_bra) error(a, e_adjacent_bracketed_in_among); q = read_C_list(a); break; default: - error(a, 3); + error(a, e_unexpected_token_in_among); + previous_token = token; + continue; case c_ket: - if (p->number == 0) error(a, 18); + if (p->number == 0) error(a, e_empty_among); if (t->error_count == 0) make_among(a, p, substring); return p; } @@ -547,13 +779,13 @@ static struct node * read_among(struct analyser * a) { static struct node * read_substring(struct analyser * a) { struct node * p = new_node(a, c_substring); - if (a->substring != 0) error2(a, 20, a->substring->line_number); + if (a->substring != 0) error2(a, e_substring_preceded_by_substring, a->substring->line_number); a->substring = p; return p; } static void check_modifyable(struct analyser * a) { - unless (a->modifyable) error(a, 15); + if (!a->modifyable) error(a, e_not_allowed_inside_reverse); } static struct node * read_C(struct analyser * a) { @@ -565,7 +797,7 @@ static struct node * read_C(struct analyser * a) { case c_backwards: { int mode = a->mode; - if (a->mode == m_backward) error(a, 17); else a->mode = m_backward; + if (a->mode == m_backward) error(a, e_already_backwards); else a->mode = m_backward; { struct node * p = C_style(a, "C", token); a->mode = mode; return p; @@ -596,14 +828,18 @@ static struct node * read_C(struct analyser * a) { case c_loop: case c_atleast: return C_style(a, "AC", token); - case c_setmark: - return C_style(a, "i", token); + case c_setmark: { + struct node * n = C_style(a, "i", token); + if (n->name) n->name->initialised = true; + return n; + } case c_tomark: case c_atmark: case c_hop: return C_style(a, "A", token); case c_delete: check_modifyable(a); + /* fall through */ case c_next: case c_tolimit: case c_atlimit: @@ -612,60 +848,147 @@ static struct node * read_C(struct analyser * a) { case c_true: case c_false: case c_debug: - return C_style(a, "", token); + return new_node(a, token); case c_assignto: - case c_sliceto: + case c_sliceto: { + struct node *n; check_modifyable(a); - return C_style(a, "s", token); + n = C_style(a, "s", token); + if (n->name) n->name->initialised = true; + return n; + } case c_assign: case c_insert: case c_attach: - case c_slicefrom: + case c_slicefrom: { + struct node *n; check_modifyable(a); - return C_style(a, "S", token); + n = C_style(a, "S", token); + if (n->name) n->name->value_used = true; + return n; + } case c_setlimit: return C_style(a, "CfD", token); case c_set: - case c_unset: - return C_style(a, "b", token); - case c_dollar: - get_token(a, c_name); - { + case c_unset: { + struct node * n = C_style(a, "b", token); + if (n->name) n->name->initialised = true; + return n; + } + case c_dollar: { + struct tokeniser * t = a->tokeniser; + read_token(t); + if (t->token == c_bra) { + /* Handle newer $(AE REL_OP AE) syntax. */ + struct node * n = read_AE(a, 0); + read_token(t); + switch (t->token) { + case c_eq: + case c_ne: + case c_gr: + case c_ge: + case c_ls: + case c_le: { + struct node * lhs = n; + n = new_node(a, t->token); + n->left = lhs; + n->AE = read_AE(a, 0); + get_token(a, c_ket); + break; + } + default: + error(a, e_unexpected_token); + t->token_held = true; + break; + } + return n; + } + + if (t->token == c_name) { struct node * p; struct name * q = find_name(a); int mode = a->mode; int modifyable = a->modifyable; - switch (q ? q->type : t_string) - /* above line was: switch (q->type) - bug #1 fix 7/2/2003 */ - { - default: error(a, 34); - case t_string: - a->mode = m_forward; - a->modifyable = true; - p = new_node(a, c_dollar); - p->left = read_C(a); break; - case t_integer: - /* a->mode = m_integer; */ - p = new_node(a, read_AE_test(a)); - p->AE = read_AE(a, 0); break; + if (q && q->type == t_string) { + /* Assume for now that $ on string both initialises and + * uses the string variable. FIXME: Can we do better? + */ + q->initialised = true; + q->value_used = true; + a->mode = m_forward; + a->modifyable = true; + p = new_node(a, c_dollar); + p->left = read_C(a); + p->name = q; + } else { + if (q && q->type != t_integer) { + /* If $ is used on an unknown name or a name which + * isn't a string or an integer then we assume the + * unknown name is an integer as $ is used more often + * on integers than strings, so hopefully this it less + * likely to cause an error avalanche. + * + * For an unknown name, we'll already have reported an + * error. + */ + error(a, e_not_of_type_string_or_integer); + q = NULL; + } + p = new_node(a, read_AE_test(a)); + p->AE = read_AE(a, 0); + + if (q) { + switch (p->type) { + case c_mathassign: + q->initialised = true; + p->name = q; + break; + default: + /* +=, etc don't "initialise" as they only + * amend an existing value. Similarly, they + * don't count as using the value. + */ + p->name = q; + break; + case c_eq: + case c_ne: + case c_gr: + case c_ge: + case c_ls: + case c_le: + p->left = new_node(a, c_name); + p->left->name = q; + q->value_used = true; + break; + } + } } - p->name = q; + if (q) mark_used_in(a, q, p); a->mode = mode; a->modifyable = modifyable; return p; } + + error(a, e_unexpected_token); + t->token_held = true; + return new_node(a, c_dollar); + } case c_name: { struct name * q = find_name(a); struct node * p = new_node(a, c_name); - unless (q == 0) { - q->used = true; + if (q) { + mark_used_in(a, q, p); switch (q->type) { case t_boolean: - p->type = c_booltest; break; + p->type = c_booltest; + q->value_used = true; + break; case t_integer: - error(a, 35); /* integer name misplaced */ + error(a, e_misplaced); /* integer name misplaced */ + break; case t_string: + q->value_used = true; break; case t_routine: case t_external: @@ -684,7 +1007,7 @@ static struct node * read_C(struct analyser * a) { struct node * p = new_node(a, token); read_token(t); if (t->token == c_minus) read_token(t); - unless (check_token(a, c_name)) { omission_error(a, c_name); return p; } + if (!check_token(a, c_name)) { omission_error(a, c_name); return p; } name_to_node(a, p, 'g'); return p; } @@ -692,7 +1015,7 @@ static struct node * read_C(struct analyser * a) { return read_literalstring(a); case c_among: return read_among(a); case c_substring: return read_substring(a); - default: error(a, 1); return 0; + default: error(a, e_unexpected_token); return 0; } } @@ -739,26 +1062,27 @@ static void read_define_grouping(struct analyser * a, struct name * q) { NEW(grouping, p); if (a->groupings == 0) a->groupings = p; else a->groupings_end->next = p; a->groupings_end = p; - q->grouping = p; + if (q) q->grouping = p; p->next = 0; p->name = q; - p->number = q->count; + p->line_number = a->tokeniser->line_number; p->b = create_b(0); - repeat { + while (true) { switch (read_token(t)) { case c_name: { struct name * r = find_name(a); - unless (r == 0) { + if (r) { check_name_type(a, r, 'g'); p->b = alter_grouping(p->b, r->grouping->b, style, false); + r->used_in_definition = true; } } break; case c_literalstring: - p->b = alter_grouping(p->b, t->b, style, a->utf8); + p->b = alter_grouping(p->b, t->b, style, (a->encoding == ENC_UTF8)); break; - default: error(a, 1); return; + default: error(a, e_unexpected_token); return; } switch (read_token(t)) { case c_plus: @@ -777,7 +1101,7 @@ static void read_define_grouping(struct analyser * a, struct name * q) { } p->largest_ch = max; p->smallest_ch = min; - if (min == 1<<16) error(a, 16); + if (min == 1<<16) error(a, e_empty_grouping); } t->token_held = true; return; } @@ -786,32 +1110,49 @@ static void read_define_grouping(struct analyser * a, struct name * q) { static void read_define_routine(struct analyser * a, struct name * q) { struct node * p = new_node(a, c_define); a->amongvar_needed = false; - unless (q == 0) { + if (q) { check_name_type(a, q, 'R'); - if (q->definition != 0) error(a, 36); + if (q->definition != 0) error(a, e_redefined); if (q->mode < 0) q->mode = a->mode; else - if (q->mode != a->mode) error2(a, 32, q->mode); + if (q->mode != a->mode) error2(a, e_declared_as_different_mode, q->mode); } p->name = q; if (a->program == 0) a->program = p; else a->program_end->right = p; a->program_end = p; get_token(a, c_as); p->left = read_C(a); - unless (q == 0) q->definition = p->left; + if (q) q->definition = p->left; if (a->substring != 0) { - error2(a, 14, a->substring->line_number); - a->substring = 0; + error2(a, e_unresolved_substring, a->substring->line_number); + a->substring = 0; } p->amongvar_needed = a->amongvar_needed; } static void read_define(struct analyser * a) { - unless (get_token(a, c_name)) return; - { + if (get_token(a, c_name)) { struct name * q = find_name(a); - if (q != 0 && q->type == t_grouping) read_define_grouping(a, q); - else read_define_routine(a, q); + int type; + if (q) { + type = q->type; + } else { + /* No declaration, so sniff next token - if it is 'as' then parse + * as a routine, otherwise as a grouping. + */ + if (read_token(a->tokeniser) == c_as) { + type = t_routine; + } else { + type = t_grouping; + } + a->tokeniser->token_held = true; + } + + if (type == t_grouping) { + read_define_grouping(a, q); + } else { + read_define_routine(a, q); + } } } @@ -827,7 +1168,7 @@ static void read_backwardmode(struct analyser * a) { static void read_program_(struct analyser * a, int terminator) { struct tokeniser * t = a->tokeniser; - repeat { + while (true) { switch (read_token(t)) { case c_strings: read_names(a, t_string); break; case c_booleans: read_names(a, t_boolean); break; @@ -839,25 +1180,60 @@ static void read_program_(struct analyser * a, int terminator) { case c_backwardmode:read_backwardmode(a); break; case c_ket: if (terminator == c_ket) return; + /* fall through */ default: - error(a, 1); break; + error(a, e_unexpected_token); break; case -1: - unless (terminator < 0) omission_error(a, c_ket); + if (terminator >= 0) omission_error(a, c_ket); return; } } } +static void remove_dead_assignments(struct node * p, struct name * q) { + if (p->name == q) { + switch (p->type) { + case c_assignto: + case c_sliceto: + case c_mathassign: + case c_plusassign: + case c_minusassign: + case c_multiplyassign: + case c_divideassign: + case c_setmark: + case c_set: + case c_unset: + case c_dollar: + /* c_true is a no-op. */ + p->type = c_true; + break; + default: + /* There are no read accesses to this variable, so any + * references must be assignments. + */ + fprintf(stderr, "Unhandled type of dead assignment via %s\n", + name_of_token(p->type)); + exit(1); + } + } + if (p->AE) remove_dead_assignments(p->AE, q); + if (p->left) remove_dead_assignments(p->left, q); + if (p->aux) remove_dead_assignments(p->aux, q); + if (p->right) remove_dead_assignments(p->right, q); +} + extern void read_program(struct analyser * a) { read_program_(a, -1); { struct name * q = a->names; - until (q == 0) { - switch(q->type) { + while (q) { + switch (q->type) { case t_external: case t_routine: - if (q->used && q->definition == 0) { error4(a, q); } break; + if (q->used && q->definition == 0) error4(a, q); + break; case t_grouping: - if (q->used && q->grouping == 0) { error4(a, q); } break; + if (q->used && q->grouping == 0) error4(a, q); + break; } q = q->next; } @@ -865,33 +1241,77 @@ extern void read_program(struct analyser * a) { if (a->tokeniser->error_count == 0) { struct name * q = a->names; - int warned = false; - until (q == 0) { - unless (q->referenced) { - unless (warned) { - fprintf(stderr, "Declared but not used:"); - warned = true; + struct name ** ptr = &(a->names); + while (q) { + if (!q->referenced) { + fprintf(stderr, "%s:%d: warning: %s '", + a->tokeniser->file, + q->declaration_line_number, + name_of_name_type(q->type)); + report_b(stderr, q->b); + if (q->type == t_routine || + q->type == t_external || + q->type == t_grouping) { + fprintf(stderr, "' declared but not defined\n"); + } else { + fprintf(stderr, "' defined but not used\n"); + q = q->next; + *ptr = q; + continue; } - fprintf(stderr, " "); report_b(stderr, q->b); + } else if (q->type == t_routine || q->type == t_grouping) { + /* It's OK to define a grouping but only use it to define other + * groupings. + */ + if (!q->used && !q->used_in_definition) { + int line_num; + if (q->type == t_routine) { + line_num = q->definition->line_number; + } else { + line_num = q->grouping->line_number; + } + fprintf(stderr, "%s:%d: warning: %s '", + a->tokeniser->file, + line_num, + name_of_name_type(q->type)); + report_b(stderr, q->b); + fprintf(stderr, "' defined but not used\n"); + } + } else if (q->type == t_external) { + /* Unused is OK. */ + } else if (!q->initialised) { + fprintf(stderr, "%s:%d: warning: %s '", + a->tokeniser->file, + q->declaration_line_number, + name_of_name_type(q->type)); + report_b(stderr, q->b); + fprintf(stderr, "' is never initialised\n"); + } else if (!q->value_used) { + fprintf(stderr, "%s:%d: warning: %s '", + a->tokeniser->file, + q->declaration_line_number, + name_of_name_type(q->type)); + report_b(stderr, q->b); + fprintf(stderr, "' is set but never used\n"); + remove_dead_assignments(a->program, q); + q = q->next; + *ptr = q; + continue; } + ptr = &(q->next); q = q->next; } - if (warned) fprintf(stderr, "\n"); - q = a->names; - warned = false; - until (q == 0) { - if (! q->used && (q->type == t_routine || - q->type == t_grouping)) { - unless (warned) { - fprintf(stderr, "Declared and defined but not used:"); - warned = true; - } - fprintf(stderr, " "); report_b(stderr, q->b); + { + /* Now we've eliminated variables whose values are never used we + * can number the variables, which is used by some generators. + */ + int * name_count = a->name_count; + struct name * n; + for (n = a->names; n; n = n->next) { + n->count = name_count[n->type]++; } - q = q->next; } - if (warned) fprintf(stderr, "\n"); } } @@ -909,13 +1329,14 @@ extern struct analyser * create_analyser(struct tokeniser * t) { a->modifyable = true; { int i; for (i = 0; i < t_size; i++) a->name_count[i] = 0; } a->substring = 0; + a->int_limits_used = false; return a; } extern void close_analyser(struct analyser * a) { { struct node * q = a->nodes; - until (q == 0) { + while (q) { struct node * q_next = q->next; FREE(q); q = q_next; @@ -923,7 +1344,7 @@ extern void close_analyser(struct analyser * a) { } { struct name * q = a->names; - until (q == 0) { + while (q) { struct name * q_next = q->next; lose_b(q->b); FREE(q); q = q_next; @@ -931,7 +1352,7 @@ extern void close_analyser(struct analyser * a) { } { struct literalstring * q = a->literalstrings; - until (q == 0) { + while (q) { struct literalstring * q_next = q->next; lose_b(q->b); FREE(q); q = q_next; @@ -939,15 +1360,17 @@ extern void close_analyser(struct analyser * a) { } { struct among * q = a->amongs; - until (q == 0) { + while (q) { struct among * q_next = q->next; - FREE(q->b); FREE(q); + FREE(q->b); + FREE(q->commands); + FREE(q); q = q_next; } } { struct grouping * q = a->groupings; - until (q == 0) { + while (q) { struct grouping * q_next = q->next; lose_b(q->b); FREE(q); q = q_next; @@ -955,4 +1378,3 @@ extern void close_analyser(struct analyser * a) { } FREE(a); } - diff --git a/contrib/snowball/compiler/driver.c b/contrib/snowball/compiler/driver.c index fbb1e9cae..9cbe447a5 100644 --- a/contrib/snowball/compiler/driver.c +++ b/contrib/snowball/compiler/driver.c @@ -1,48 +1,86 @@ +#include <ctype.h> /* for toupper etc */ #include <stdio.h> /* for fprintf etc */ #include <stdlib.h> /* for free etc */ -#include <string.h> /* for strlen */ +#include <string.h> /* for strcmp */ #include "header.h" -#define DEFAULT_PACKAGE "org.tartarus.snowball.ext" -#define DEFAULT_BASE_CLASS "org.tartarus.snowball.SnowballProgram" -#define DEFAULT_AMONG_CLASS "org.tartarus.snowball.Among" -#define DEFAULT_STRING_CLASS "java.lang.StringBuilder" +#define DEFAULT_JAVA_PACKAGE "org.tartarus.snowball.ext" +#define DEFAULT_JAVA_BASE_CLASS "org.tartarus.snowball.SnowballProgram" +#define DEFAULT_JAVA_AMONG_CLASS "org.tartarus.snowball.Among" +#define DEFAULT_JAVA_STRING_CLASS "java.lang.StringBuilder" + +#define DEFAULT_GO_PACKAGE "snowball" +#define DEFAULT_GO_SNOWBALL_RUNTIME "github.com/snowballstem/snowball/go" + +#define DEFAULT_CS_NAMESPACE "Snowball" +#define DEFAULT_CS_BASE_CLASS "Stemmer" +#define DEFAULT_CS_AMONG_CLASS "Among" +#define DEFAULT_CS_STRING_CLASS "StringBuilder" + +#define DEFAULT_JS_BASE_CLASS "BaseStemmer" + +#define DEFAULT_PYTHON_BASE_CLASS "BaseStemmer" static int eq(const char * s1, const char * s2) { - int s1_len = strlen(s1); - int s2_len = strlen(s2); - return s1_len == s2_len && memcmp(s1, s2, s1_len) == 0; + return strcmp(s1, s2) == 0; } -static void print_arglist(void) { - fprintf(stderr, "Usage: snowball <file> [options]\n\n" - "options are: [-o[utput] file]\n" - " [-s[yntax]]\n" +static void print_arglist(int exit_code) { + FILE * f = exit_code ? stderr : stdout; + fprintf(f, "Usage: snowball SOURCE_FILE... [OPTIONS]\n\n" + "Supported options:\n" + " -o[utput] file\n" + " -s[yntax]\n" + " -comments\n" #ifndef DISABLE_JAVA - " [-j[ava]]\n" + " -j[ava]\n" #endif - " [-c++]\n" - " [-w[idechars]]\n" - " [-u[tf8]]\n" - " [-n[ame] class name]\n" - " [-ep[refix] string]\n" - " [-vp[refix] string]\n" - " [-i[nclude] directory]\n" - " [-r[untime] path to runtime headers]\n" -#ifndef DISABLE_JAVA - " [-p[arentclassname] fully qualified parent class name]\n" - " [-P[ackage] package name for stemmers]\n" - " [-S[tringclass] StringBuffer-compatible class]\n" - " [-a[mongclass] fully qualified name of the Among class]\n" +#ifndef DISABLE_CSHARP + " -cs[harp]\n" +#endif + " -c++\n" +#ifndef DISABLE_PASCAL + " -pascal\n" +#endif +#ifndef DISABLE_PYTHON + " -py[thon]\n" #endif +#ifndef DISABLE_JS + " -js\n" +#endif +#ifndef DISABLE_RUST + " -rust\n" +#endif +#ifndef DISABLE_GO + " -go\n" +#endif + " -w[idechars]\n" + " -u[tf8]\n" + " -n[ame] class name\n" + " -ep[refix] string\n" + " -vp[refix] string\n" + " -i[nclude] directory\n" + " -r[untime] path to runtime headers\n" + " -p[arentclassname] fully qualified parent class name\n" +#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP) + " -P[ackage] package name for stemmers\n" + " -S[tringclass] StringBuffer-compatible class\n" + " -a[mongclass] fully qualified name of the Among class\n" +#endif +#ifndef DISABLE_GO + " -gop[ackage] Go package name for stemmers\n" + " -gor[untime] Go snowball runtime package\n" +#endif + " --help display this help and exit\n" + " --version output version information and exit\n" ); - exit(1); + exit(exit_code); } static void check_lim(int i, int argc) { if (i >= argc) { fprintf(stderr, "argument list is one short\n"); - print_arglist(); + print_arglist(1); } } @@ -57,35 +95,47 @@ static FILE * get_output(symbol * b) { return output; } -static void read_options(struct options * o, int argc, char * argv[]) { +static int read_options(struct options * o, int argc, char * argv[]) { char * s; - int i = 2; + int i = 1; + int new_argc = 1; + /* Note down the last option used to specify an explicit encoding so + * we can warn we ignored it for languages with a fixed encoding. + */ + const char * encoding_opt = NULL; /* set defaults: */ o->output_file = 0; o->syntax_tree = false; - o->externals_prefix = ""; + o->comments = false; + o->externals_prefix = NULL; o->variables_prefix = 0; o->runtime_path = 0; - o->parent_class_name = DEFAULT_BASE_CLASS; - o->string_class = DEFAULT_STRING_CLASS; - o->among_class = DEFAULT_AMONG_CLASS; - o->package = DEFAULT_PACKAGE; - o->name = ""; + o->parent_class_name = NULL; + o->string_class = NULL; + o->among_class = NULL; + o->package = NULL; + o->go_snowball_runtime = DEFAULT_GO_SNOWBALL_RUNTIME; + o->name = NULL; o->make_lang = LANG_C; - o->widechars = false; o->includes = 0; o->includes_end = 0; - o->utf8 = false; + o->encoding = ENC_SINGLEBYTE; /* read options: */ - repeat { - if (i >= argc) break; + while (i < argc) { s = argv[i++]; - { if (eq(s, "-o") || eq(s, "-output")) { - check_lim(i, argc); + if (s[0] != '-') { + /* Non-option argument - shuffle down. */ + argv[new_argc++] = s; + continue; + } + + { + if (eq(s, "-o") || eq(s, "-output")) { + check_lim(i, argc); o->output_file = argv[i++]; continue; } @@ -94,10 +144,33 @@ static void read_options(struct options * o, int argc, char * argv[]) { o->name = argv[i++]; continue; } +#ifndef DISABLE_JS + if (eq(s, "-js")) { + o->make_lang = LANG_JAVASCRIPT; + continue; + } +#endif +#ifndef DISABLE_RUST + if (eq(s, "-rust")) { + o->make_lang = LANG_RUST; + continue; + } +#endif +#ifndef DISABLE_GO + if (eq(s, "-go")) { + o->make_lang = LANG_GO; + continue; + } +#endif #ifndef DISABLE_JAVA if (eq(s, "-j") || eq(s, "-java")) { o->make_lang = LANG_JAVA; - o->widechars = true; + continue; + } +#endif +#ifndef DISABLE_CSHARP + if (eq(s, "-cs") || eq(s, "-csharp")) { + o->make_lang = LANG_CSHARP; continue; } #endif @@ -105,15 +178,31 @@ static void read_options(struct options * o, int argc, char * argv[]) { o->make_lang = LANG_CPLUSPLUS; continue; } +#ifndef DISABLE_PASCAL + if (eq(s, "-pascal")) { + o->make_lang = LANG_PASCAL; + continue; + } +#endif +#ifndef DISABLE_PYTHON + if (eq(s, "-py") || eq(s, "-python")) { + o->make_lang = LANG_PYTHON; + continue; + } +#endif if (eq(s, "-w") || eq(s, "-widechars")) { - o->widechars = true; - o->utf8 = false; + encoding_opt = s; + o->encoding = ENC_WIDECHARS; continue; } if (eq(s, "-s") || eq(s, "-syntax")) { o->syntax_tree = true; continue; } + if (eq(s, "-comments")) { + o->comments = true; + continue; + } if (eq(s, "-ep") || eq(s, "-eprefix")) { check_lim(i, argc); o->externals_prefix = argv[i++]; @@ -145,16 +234,16 @@ static void read_options(struct options * o, int argc, char * argv[]) { continue; } if (eq(s, "-u") || eq(s, "-utf8")) { - o->utf8 = true; - o->widechars = false; + encoding_opt = s; + o->encoding = ENC_UTF8; continue; } -#ifndef DISABLE_JAVA if (eq(s, "-p") || eq(s, "-parentclassname")) { check_lim(i, argc); o->parent_class_name = argv[i++]; continue; } +#if !defined(DISABLE_JAVA) || !defined(DISABLE_CSHARP) if (eq(s, "-P") || eq(s, "-Package")) { check_lim(i, argc); o->package = argv[i++]; @@ -171,44 +260,216 @@ static void read_options(struct options * o, int argc, char * argv[]) { continue; } #endif +#ifndef DISABLE_GO + if (eq(s, "-gop") || eq(s, "-gopackage")) { + check_lim(i, argc); + o->package = argv[i++]; + continue; + } + if (eq(s, "-gor") || eq(s, "-goruntime")) { + check_lim(i, argc); + o->go_snowball_runtime = argv[i++]; + continue; + } +#endif + if (eq(s, "--help")) { + print_arglist(0); + } + + if (eq(s, "--version")) { + printf("Snowball compiler version " SNOWBALL_VERSION "\n"); + exit(0); + } + fprintf(stderr, "'%s' misplaced\n", s); - print_arglist(); + print_arglist(1); } } + if (new_argc == 1) { + fprintf(stderr, "no source files specified\n"); + print_arglist(1); + } + argv[new_argc] = NULL; + + /* Set language-dependent defaults. */ + switch (o->make_lang) { + case LANG_C: + case LANG_CPLUSPLUS: + encoding_opt = NULL; + break; + case LANG_CSHARP: + o->encoding = ENC_WIDECHARS; + if (!o->parent_class_name) + o->parent_class_name = DEFAULT_CS_BASE_CLASS; + if (!o->string_class) + o->string_class = DEFAULT_CS_STRING_CLASS; + if (!o->among_class) + o->among_class = DEFAULT_CS_AMONG_CLASS; + if (!o->package) + o->package = DEFAULT_CS_NAMESPACE; + break; + case LANG_GO: + o->encoding = ENC_UTF8; + if (!o->package) + o->package = DEFAULT_GO_PACKAGE; + break; + case LANG_JAVA: + o->encoding = ENC_WIDECHARS; + if (!o->parent_class_name) + o->parent_class_name = DEFAULT_JAVA_BASE_CLASS; + if (!o->string_class) + o->string_class = DEFAULT_JAVA_STRING_CLASS; + if (!o->among_class) + o->among_class = DEFAULT_JAVA_AMONG_CLASS; + if (!o->package) + o->package = DEFAULT_JAVA_PACKAGE; + break; + case LANG_JAVASCRIPT: + o->encoding = ENC_WIDECHARS; + if (!o->parent_class_name) + o->parent_class_name = DEFAULT_JS_BASE_CLASS; + break; + case LANG_PYTHON: + o->encoding = ENC_WIDECHARS; + if (!o->parent_class_name) + o->parent_class_name = DEFAULT_PYTHON_BASE_CLASS; + break; + case LANG_RUST: + o->encoding = ENC_UTF8; + break; + default: + break; + } + + if (encoding_opt) { + fprintf(stderr, "warning: %s only meaningful for C and C++\n", + encoding_opt); + } + + if (o->make_lang != LANG_C && o->make_lang != LANG_CPLUSPLUS) { + if (o->runtime_path) { + fprintf(stderr, "warning: -r/-runtime only meaningful for C and C++\n"); + } + if (o->externals_prefix) { + fprintf(stderr, "warning: -ep/-eprefix only meaningful for C and C++\n"); + } + } + if (!o->externals_prefix) o->externals_prefix = ""; + + if (!o->name && o->output_file) { + /* Default class name to basename of output_file - this is the standard + * convention for at least Java and C#. + */ + const char * slash = strrchr(o->output_file, '/'); + size_t len; + const char * leaf = (slash == NULL) ? o->output_file : slash + 1; + + slash = strrchr(leaf, '\\'); + if (slash != NULL) leaf = slash + 1; + + { + const char * dot = strchr(leaf, '.'); + len = (dot == NULL) ? strlen(leaf) : (size_t)(dot - leaf); + } + + { + char * new_name = malloc(len + 1); + switch (o->make_lang) { + case LANG_CSHARP: + case LANG_PASCAL: + /* Upper case initial letter. */ + memcpy(new_name, leaf, len); + new_name[0] = toupper(new_name[0]); + break; + case LANG_JAVASCRIPT: + case LANG_PYTHON: { + /* Upper case initial letter and change each + * underscore+letter or hyphen+letter to an upper case + * letter. + */ + size_t i, j = 0; + int uc_next = true; + for (i = 0; i != len; ++i) { + unsigned char ch = leaf[i]; + if (ch == '_' || ch == '-') { + uc_next = true; + } else { + if (uc_next) { + new_name[j] = toupper(ch); + uc_next = false; + } else { + new_name[j] = ch; + } + ++j; + } + } + len = j; + break; + } + default: + /* Just copy. */ + memcpy(new_name, leaf, len); + break; + } + new_name[len] = '\0'; + o->name = new_name; + } + } + + return new_argc; } extern int main(int argc, char * argv[]) { + int i; NEW(options, o); - if (argc == 1) print_arglist(); - read_options(o, argc, argv); + argc = read_options(o, argc, argv); { - symbol * filename = add_s_to_b(0, argv[1]); - char * file; - symbol * u = get_input(filename, &file); + char * file = argv[1]; + symbol * u = get_input(file); if (u == 0) { - fprintf(stderr, "Can't open input %s\n", argv[1]); + fprintf(stderr, "Can't open input %s\n", file); exit(1); } { struct tokeniser * t = create_tokeniser(u, file); struct analyser * a = create_analyser(t); - t->widechars = o->widechars; + struct input ** next_input_ptr = &(t->next); + a->encoding = t->encoding = o->encoding; t->includes = o->includes; - a->utf8 = t->utf8 = o->utf8; + /* If multiple source files are specified, set up the others to be + * read after the first in order, using the same mechanism as + * 'get' uses. */ + for (i = 2; i != argc; ++i) { + NEW(input, q); + file = argv[i]; + u = get_input(file); + if (u == 0) { + fprintf(stderr, "Can't open input %s\n", file); + exit(1); + } + q->p = u; + q->c = 0; + q->file = file; + q->file_needs_freeing = false; + q->line_number = 1; + *next_input_ptr = q; + next_input_ptr = &(q->next); + } + *next_input_ptr = NULL; read_program(a); if (t->error_count > 0) exit(1); if (o->syntax_tree) print_program(a); close_tokeniser(t); - unless (o->syntax_tree) { + if (!o->syntax_tree) { struct generator * g; - char * s = o->output_file; - unless (s) { + const char * s = o->output_file; + if (!s) { fprintf(stderr, "Please include the -o option\n"); - print_arglist(); - exit(1); + print_arglist(1); } + g = create_generator(a, o); if (o->make_lang == LANG_C || o->make_lang == LANG_CPLUSPLUS) { symbol * b = add_s_to_b(0, s); b = add_s_to_b(b, ".h"); @@ -217,41 +478,96 @@ extern int main(int argc, char * argv[]) { if (o->make_lang == LANG_CPLUSPLUS) { b = add_s_to_b(b, "c"); } - o->output_c = get_output(b); + o->output_src = get_output(b); lose_b(b); - g = create_generator_c(a, o); generate_program_c(g); - close_generator_c(g); - fclose(o->output_c); + fclose(o->output_src); fclose(o->output_h); } #ifndef DISABLE_JAVA if (o->make_lang == LANG_JAVA) { symbol * b = add_s_to_b(0, s); b = add_s_to_b(b, ".java"); - o->output_java = get_output(b); + o->output_src = get_output(b); lose_b(b); - g = create_generator_java(a, o); generate_program_java(g); - close_generator_java(g); - fclose(o->output_java); + fclose(o->output_src); } #endif +#ifndef DISABLE_PASCAL + if (o->make_lang == LANG_PASCAL) { + symbol *b = add_s_to_b(0, s); + b = add_s_to_b(b, ".pas"); + o->output_src = get_output(b); + lose_b(b); + generate_program_pascal(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_PYTHON + if (o->make_lang == LANG_PYTHON) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".py"); + o->output_src = get_output(b); + lose_b(b); + generate_program_python(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_JS + if (o->make_lang == LANG_JAVASCRIPT) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".js"); + o->output_src = get_output(b); + lose_b(b); + generate_program_js(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_CSHARP + if (o->make_lang == LANG_CSHARP) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".cs"); + o->output_src = get_output(b); + lose_b(b); + generate_program_csharp(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_RUST + if (o->make_lang == LANG_RUST) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".rs"); + o->output_src = get_output(b); + lose_b(b); + generate_program_rust(g); + fclose(o->output_src); + } +#endif +#ifndef DISABLE_GO + if (o->make_lang == LANG_GO) { + symbol * b = add_s_to_b(0, s); + b = add_s_to_b(b, ".go"); + o->output_src = get_output(b); + lose_b(b); + generate_program_go(g); + fclose(o->output_src); + } +#endif + close_generator(g); } close_analyser(a); } lose_b(u); - lose_b(filename); } { struct include * p = o->includes; - until (p == 0) - { struct include * q = p->next; + while (p) { + struct include * q = p->next; lose_b(p->b); FREE(p); p = q; } } FREE(o); - unless (space_count == 0) fprintf(stderr, "%d blocks unfreed\n", space_count); + if (space_count) fprintf(stderr, "%d blocks unfreed\n", space_count); return 0; } - diff --git a/contrib/snowball/compiler/generator.c b/contrib/snowball/compiler/generator.c index 7294b0471..eed86c117 100644 --- a/contrib/snowball/compiler/generator.c +++ b/contrib/snowball/compiler/generator.c @@ -12,286 +12,349 @@ static void generate(struct generator * g, struct node * p); -enum special_labels { - - x_return = -1 - -}; - static int new_label(struct generator * g) { return g->next_label++; } -/* Output routines */ -static void output_str(FILE * outfile, struct str * str) { +/* Write routines for simple entities */ - char * s = b_to_s(str_data(str)); - fprintf(outfile, "%s", s); - free(s); +/* Write a space if the preceding character was not whitespace */ +static void ws_opt_space(struct generator * g, const char * s) { + int ch = str_back(g->outbuf); + if (ch != ' ' && ch != '\n' && ch != '\t' && ch != -1) + write_char(g, ' '); + write_string(g, s); } -static void wch(struct generator * g, int ch) { - str_append_ch(g->outbuf, ch); /* character */ +static void wi3(struct generator * g, int i) { + if (i < 100) write_char(g, ' '); + if (i < 10) write_char(g, ' '); + write_int(g, i); /* integer (width 3) */ } -static void wnl(struct generator * g) { - str_append_ch(g->outbuf, '\n'); /* newline */ - g->line_count++; -} -static void ws(struct generator * g, const char * s) { - str_append_string(g->outbuf, s); /* string */ -} +/* Write routines for items from the syntax tree */ -static void wi(struct generator * g, int i) { - str_append_int(g->outbuf, i); /* integer */ -} +static void write_varname(struct generator * g, struct name * p) { -static void wh_ch(struct generator * g, int i) { - str_append_ch(g->outbuf, "0123456789ABCDEF"[i & 0xF]); /* hexchar */ + int ch = "SIIrxg"[p->type]; + switch (p->type) { + case t_external: + write_string(g, g->options->externals_prefix); break; + case t_string: + case t_boolean: + case t_integer: { + int count = p->count; + if (count < 0) { + fprintf(stderr, "Reference to optimised out variable "); + report_b(stderr, p->b); + fprintf(stderr, " attempted\n"); + exit(1); + } + if (p->type == t_boolean) { + /* We use a single array for booleans and integers, with the + * integers first. + */ + count += g->analyser->name_count[t_integer]; + } + write_char(g, ch); + write_char(g, '['); + write_int(g, count); + write_char(g, ']'); + return; + } + default: + write_char(g, ch); write_char(g, '_'); + } + write_b(g, p->b); } -static void wh(struct generator * g, int i) { - if (i >> 4) wh(g, i >> 4); - wh_ch(g, i); /* hex integer */ +static void write_varref(struct generator * g, struct name * p) { /* reference to variable */ + if (p->type < t_routine) write_string(g, "z->"); + write_varname(g, p); } -static void wi3(struct generator * g, int i) { - if (i < 100) wch(g, ' '); - if (i < 10) wch(g, ' '); - wi(g, i); /* integer (width 3) */ +static void write_hexdigit(struct generator * g, int i) { + str_append_ch(g->outbuf, "0123456789ABCDEF"[i & 0xF]); /* hexchar */ } -static void wvn(struct generator * g, struct name * p) { /* variable name */ - - int ch = "SBIrxg"[p->type]; - switch (p->type) { - case t_string: - case t_boolean: - case t_integer: - wch(g, ch); wch(g, '['); wi(g, p->count); wch(g, ']'); return; - case t_external: - ws(g, g->options->externals_prefix); break; - default: - wch(g, ch); wch(g, '_'); - } - str_append_b(g->outbuf, p->b); +static void write_hex(struct generator * g, int i) { + if (i >> 4) write_hex(g, i >> 4); + write_hexdigit(g, i); /* hex integer */ } -static void wv(struct generator * g, struct name * p) { /* reference to variable */ - if (p->type < t_routine) ws(g, "z->"); - wvn(g, p); +/* write character literal */ +static void wlitch(struct generator * g, int ch) { + if (32 <= ch && ch < 127) { + write_char(g, '\''); + if (ch == '\'' || ch == '\\') { + write_char(g, '\\'); + } + write_char(g, ch); + write_char(g, '\''); + } else { + write_string(g, "0x"); write_hex(g, ch); + } } static void wlitarray(struct generator * g, symbol * p) { /* write literal array */ - ws(g, "{ "); + write_string(g, "{ "); { int i; for (i = 0; i < SIZE(p); i++) { - int ch = p[i]; - if (32 <= ch && ch < 127) { - wch(g, '\''); - switch (ch) { - case '\'': - case '\\': wch(g, '\\'); - default: wch(g, ch); - } - wch(g, '\''); - } else { - wch(g, '0'); wch(g, 'x'); wh(g, ch); - } - if (i < SIZE(p) - 1) ws(g, ", "); + wlitch(g, p[i]); + if (i < SIZE(p) - 1) write_string(g, ", "); } } - ws(g, " }"); + write_string(g, " }"); } static void wlitref(struct generator * g, symbol * p) { /* write ref to literal array */ - if (SIZE(p) == 0) ws(g, "0"); else { + if (SIZE(p) == 0) { + write_char(g, '0'); + } else { struct str * s = g->outbuf; g->outbuf = g->declarations; - ws(g, "static const symbol s_"); wi(g, g->literalstring_count); ws(g, "[] = "); + write_string(g, "static const symbol s_"); write_int(g, g->literalstring_count); write_string(g, "[] = "); wlitarray(g, p); - ws(g, ";\n"); + write_string(g, ";\n"); g->outbuf = s; - ws(g, "s_"); wi(g, g->literalstring_count); + write_string(g, "s_"); write_int(g, g->literalstring_count); g->literalstring_count++; } } - -static void wm(struct generator * g) { /* margin */ +static void write_margin(struct generator * g) { int i; - for (i = 0; i < g->margin; i++) ws(g, " "); + for (i = 0; i < g->margin; i++) write_string(g, " "); } -static void wc(struct generator * g, struct node * p) { /* comment */ - - ws(g, " /* "); +void write_comment_content(struct generator * g, struct node * p) { switch (p->type) { case c_mathassign: case c_plusassign: case c_minusassign: case c_multiplyassign: case c_divideassign: + if (p->name) { + write_char(g, '$'); + write_b(g, p->name->b); + write_char(g, ' '); + } + write_string(g, name_of_token(p->type)); + write_string(g, " <integer expression>"); + break; case c_eq: case c_ne: case c_gr: case c_ge: case c_ls: case c_le: - if (p->name) { - wch(g, '$'); - str_append_b(g->outbuf, p->name->b); - wch(g, ' '); - } - ws(g, name_of_token(p->type)); - ws(g, " <integer expression>"); + write_string(g, "$(<integer expression> "); + write_string(g, name_of_token(p->type)); + write_string(g, " <integer expression>)"); break; default: - ws(g, name_of_token(p->type)); + write_string(g, name_of_token(p->type)); if (p->name) { - wch(g, ' '); - str_append_b(g->outbuf, p->name->b); + write_char(g, ' '); + write_b(g, p->name->b); } } - ws(g, ", line "); wi(g, p->line_number); ws(g, " */"); - wnl(g); + write_string(g, ", line "); + write_int(g, p->line_number); +} + +static void write_comment(struct generator * g, struct node * p) { + if (g->options->comments) { + ws_opt_space(g, "/* "); + write_comment_content(g, p); + write_string(g, " */"); + } + write_newline(g); } static void wms(struct generator * g, const char * s) { - wm(g); ws(g, s); } /* margin + string */ + write_margin(g); write_string(g, s); } /* margin + string */ -static void wbs(struct generator * g) { /* block start */ +static void write_block_start(struct generator * g) { /* block start */ wms(g, "{ "); g->margin++; } -static void wbe(struct generator * g) { /* block end */ +static void write_block_end(struct generator * g) { /* block end */ - if (g->line_labelled == g->line_count) { wms(g, ";"); wnl(g); } + if (g->line_labelled == g->line_count) { wms(g, ";"); write_newline(g); } g->margin--; - wms(g, "}"); wnl(g); + wms(g, "}"); write_newline(g); } -static void wk(struct generator * g, struct node * p) { /* keep c */ +static void w(struct generator * g, const char * s); + +/* keep c */ +static void wk(struct generator * g, struct node * p, int keep_limit) { ++g->keep_count; if (p->mode == m_forward) { - ws(g, "int c"); wi(g, g->keep_count); ws(g, " = z->c;"); + write_string(g, "int c"); + write_int(g, g->keep_count); + write_string(g, " = z->c"); + if (keep_limit) { + write_string(g, ", mlimit"); + write_int(g, g->keep_count); + } + write_char(g, ';'); } else { - ws(g, "int m"); wi(g, g->keep_count); ws(g, " = z->l - z->c; (void)m"); - wi(g, g->keep_count); ws(g, ";"); + write_string(g, "int m"); + write_int(g, g->keep_count); + write_string(g, " = z->l - z->c"); + if (keep_limit) { + write_string(g, ", mlimit"); + write_int(g, g->keep_count); + } + write_string(g, "; (void)m"); + write_int(g, g->keep_count); + write_char(g, ';'); } } static void wrestore(struct generator * g, struct node * p, int keep_token) { /* restore c */ if (p->mode == m_forward) { - ws(g, "z->c = c"); + write_string(g, "z->c = c"); } else { - ws(g, "z->c = z->l - m"); + write_string(g, "z->c = z->l - m"); } - wi(g, keep_token); ws(g, ";"); + write_int(g, keep_token); write_char(g, ';'); +} + +static void wrestorelimit(struct generator * g, struct node * p, int keep_token) { /* restore limit */ + if (p->mode == m_forward) { + w(g, "z->l += mlimit"); + } else { + w(g, "z->lb = mlimit"); + } + write_int(g, keep_token); write_string(g, ";"); } static void winc(struct generator * g, struct node * p) { /* increment c */ - ws(g, p->mode == m_forward ? "z->c++;" : + write_string(g, p->mode == m_forward ? "z->c++;" : "z->c--;"); } static void wsetl(struct generator * g, int n) { g->margin--; - wms(g, "lab"); wi(g, n); wch(g, ':'); wnl(g); + wms(g, "lab"); write_int(g, n); write_char(g, ':'); write_newline(g); g->line_labelled = g->line_count; g->margin++; } static void wgotol(struct generator * g, int n) { - wms(g, "goto lab"); wi(g, n); wch(g, ';'); wnl(g); + wms(g, "goto lab"); write_int(g, n); write_char(g, ';'); write_newline(g); } -static void wf(struct generator * g) { /* fail */ - if (g->failure_string != 0) { ws(g, "{ "); ws(g, g->failure_string); wch(g, ' '); } - switch (g->failure_label) - { +static void write_failure(struct generator * g, struct node * p) { /* fail */ + if (g->failure_keep_count != 0) { + write_string(g, "{ "); + if (g->failure_keep_count > 0) { + wrestore(g, p, g->failure_keep_count); + } else { + wrestorelimit(g, p, -g->failure_keep_count); + } + write_char(g, ' '); + } + switch (g->failure_label) { case x_return: - ws(g, "return 0;"); - break; + write_string(g, "return 0;"); + break; default: - ws(g, "goto lab"); - wi(g, g->failure_label); - wch(g, ';'); - g->label_used = 1; + write_string(g, "goto lab"); + write_int(g, g->failure_label); + write_char(g, ';'); + g->label_used = 1; } - if (g->failure_string != 0) ws(g, " }"); + if (g->failure_keep_count != 0) write_string(g, " }"); } -static void wlim(struct generator * g, struct node * p) { /* if at limit fail */ - ws(g, p->mode == m_forward ? "if (z->c >= z->l) " : +/* if at limit fail */ +static void write_check_limit(struct generator * g, struct node * p) { + + write_string(g, p->mode == m_forward ? "if (z->c >= z->l) " : "if (z->c <= z->lb) "); - wf(g); + write_failure(g, p); } -static void wp(struct generator * g, const char * s, struct node * p) { /* formatted write */ +static void write_data_address(struct generator * g, struct node * p) { + symbol * b = p->literalstring; + if (b != 0) { + write_int(g, SIZE(b)); w(g, ", "); + wlitref(g, b); + } else { + write_varref(g, p->name); + } +} + +/* Formatted write. */ +static void writef(struct generator * g, const char * input, struct node * p) { int i = 0; - int l = strlen(s); - until (i >= l) { - int ch = s[i++]; - if (ch != '~') wch(g, ch); else - switch(s[i++]) { - default: wch(g, s[i - 1]); continue; - case 'C': wc(g, p); continue; - case 'k': wk(g, p); continue; - case 'K': /* keep for c_test */ - ws(g, p->mode == m_forward ? "int c_test = z->c;" : - "int m_test = z->l - z->c;"); - continue; - case 'R': /* restore for c_test */ - ws(g, p->mode == m_forward ? "z->c = c_test;" : - "z->c = z->l - m_test;"); - continue; + int l = strlen(input); + + while (i < l) { + int ch = input[i++]; + if (ch != '~') { + write_char(g, ch); + continue; + } + switch (input[i++]) { + default: write_char(g, input[i - 1]); continue; + case 'C': write_comment(g, p); continue; + case 'k': wk(g, p, false); continue; + case 'K': wk(g, p, true); continue; case 'i': winc(g, p); continue; - case 'l': wlim(g, p); continue; - case 'f': wf(g); continue; - case 'M': wm(g); continue; - case 'N': wnl(g); continue; - case '{': wbs(g); continue; - case '}': wbe(g); continue; - case 'S': ws(g, g->S[s[i++] - '0']); continue; - case 'I': wi(g, g->I[s[i++] - '0']); continue; - case 'J': wi3(g, g->I[s[i++] - '0']); continue; - case 'V': wv(g, g->V[s[i++] - '0']); continue; - case 'W': wvn(g, g->V[s[i++] - '0']); continue; - case 'L': wlitref(g, g->L[s[i++] - '0']); continue; - case 'A': wlitarray(g, g->L[s[i++] - '0']); continue; + case 'l': write_check_limit(g, p); continue; + case 'f': write_failure(g, p); continue; + case 'M': write_margin(g); continue; + case 'N': write_newline(g); continue; + case '{': write_block_start(g); continue; + case '}': write_block_end(g); continue; + case 'S': write_string(g, g->S[input[i++] - '0']); continue; + case 'I': write_int(g, g->I[input[i++] - '0']); continue; + case 'J': wi3(g, g->I[input[i++] - '0']); continue; + case 'V': write_varref(g, g->V[input[i++] - '0']); continue; + case 'W': write_varname(g, g->V[input[i++] - '0']); continue; + case 'L': wlitref(g, g->L[input[i++] - '0']); continue; + case 'A': wlitarray(g, g->L[input[i++] - '0']); continue; + case 'c': wlitch(g, g->I[input[i++] - '0']); continue; + case 'a': write_data_address(g, p); continue; case '+': g->margin++; continue; case '-': g->margin--; continue; case '$': /* insert_s, insert_v etc */ - wch(g, p->literalstring == 0 ? 'v' : 's'); + write_char(g, p->literalstring == 0 ? 'v' : 's'); continue; - case 'p': ws(g, g->options->externals_prefix); continue; + case 'p': write_string(g, g->options->externals_prefix); continue; } } } -static void w(struct generator * g, const char * s) { wp(g, s, 0); } +static void w(struct generator * g, const char * s) { + writef(g, s, 0); +} static void generate_AE(struct generator * g, struct node * p) { - char * s; + const char * s; switch (p->type) { case c_name: - wv(g, p->name); break; + write_varref(g, p->name); break; case c_number: - wi(g, p->number); break; + write_int(g, p->number); break; case c_maxint: - ws(g, "MAXINT"); break; + write_string(g, "MAXINT"); break; case c_minint: - ws(g, "MININT"); break; + write_string(g, "MININT"); break; case c_neg: - wch(g, '-'); generate_AE(g, p->right); break; + write_char(g, '-'); generate_AE(g, p->right); break; case c_multiply: s = " * "; goto label0; case c_plus: @@ -301,28 +364,45 @@ static void generate_AE(struct generator * g, struct node * p) { case c_divide: s = " / "; label0: - wch(g, '('); generate_AE(g, p->left); - ws(g, s); generate_AE(g, p->right); wch(g, ')'); break; - case c_sizeof: - g->V[0] = p->name; - w(g, "SIZE(~V0)"); break; + write_char(g, '('); generate_AE(g, p->left); + write_string(g, s); generate_AE(g, p->right); write_char(g, ')'); break; case c_cursor: w(g, "z->c"); break; case c_limit: w(g, p->mode == m_forward ? "z->l" : "z->lb"); break; + case c_len: + if (g->options->encoding == ENC_UTF8) { + w(g, "len_utf8(z->p)"); + break; + } + /* FALLTHRU */ case c_size: - w(g, "SIZE(z->p)"); break; + w(g, "SIZE(z->p)"); + break; + case c_lenof: + if (g->options->encoding == ENC_UTF8) { + g->V[0] = p->name; + w(g, "len_utf8(~V0)"); + break; + } + /* FALLTHRU */ + case c_sizeof: + g->V[0] = p->name; + w(g, "SIZE(~V0)"); + break; } } /* K_needed() tests to see if we really need to keep c. Not true when the - the command does not touch the cursor. This and repeat_score() could be + command does not touch the cursor. This and repeat_score() could be elaborated almost indefinitely. */ -static int K_needed(struct generator * g, struct node * p) { - until (p == 0) { +static int K_needed_(struct generator * g, struct node * p, int call_depth) { + while (p) { switch (p->type) { + case c_atlimit: + case c_do: case c_dollar: case c_leftslice: case c_rightslice: @@ -338,17 +418,28 @@ static int K_needed(struct generator * g, struct node * p) { case c_ls: case c_le: case c_sliceto: + case c_booltest: + case c_set: + case c_unset: case c_true: case c_false: case c_debug: break; case c_call: - if (K_needed(g, p->name->definition)) return true; + /* Recursive functions aren't typical in snowball programs, so + * make the pessimistic assumption that keep is needed if we + * hit a generous limit on recursion. It's not likely to make + * a difference to any real world program, but means we won't + * recurse until we run out of stack for pathological cases. + */ + if (call_depth >= 100) return true; + if (K_needed_(g, p->name->definition, call_depth + 1)) + return true; break; case c_bra: - if (K_needed(g, p->left)) return true; + if (K_needed_(g, p->left, call_depth)) return true; break; default: return true; @@ -358,10 +449,13 @@ static int K_needed(struct generator * g, struct node * p) { return false; } -static int repeat_score(struct generator * g, struct node * p) { +extern int K_needed(struct generator * g, struct node * p) { + return K_needed_(g, p, 0); +} + +static int repeat_score(struct generator * g, struct node * p, int call_depth) { int score = 0; - until (p == 0) - { + while (p) { switch (p->type) { case c_dollar: case c_leftslice: @@ -382,11 +476,25 @@ static int repeat_score(struct generator * g, struct node * p) { break; case c_call: - score += repeat_score(g, p->name->definition); + /* Recursive functions aren't typical in snowball programs, so + * make the pessimistic assumption that repeat requires cursor + * reinstatement if we hit a generous limit on recursion. It's + * not likely to make a difference to any real world program, + * but means we won't recurse until we run out of stack for + * pathological cases. + */ + if (call_depth >= 100) { + return 2; + } + score += repeat_score(g, p->name->definition, call_depth + 1); + if (score >= 2) + return score; break; case c_bra: - score += repeat_score(g, p->left); + score += repeat_score(g, p->left, call_depth); + if (score >= 2) + return score; break; case c_name: @@ -395,9 +503,12 @@ static int repeat_score(struct generator * g, struct node * p) { case c_grouping: case c_non: case c_hop: - score = score + 1; break; + if (++score >= 2) + return score; + break; - default: score = 2; break; + default: + return 2; } p = p->right; } @@ -406,25 +517,28 @@ static int repeat_score(struct generator * g, struct node * p) { /* tests if an expression requires cursor reinstatement in a repeat */ -static int repeat_restore(struct generator * g, struct node * p) { - return repeat_score(g, p) >= 2; +extern int repeat_restore(struct generator * g, struct node * p) { + return repeat_score(g, p, 0) >= 2; } static void generate_bra(struct generator * g, struct node * p) { p = p->left; - until (p == 0) { generate(g, p); p = p->right; } + while (p) { + generate(g, p); + p = p->right; + } } static void generate_and(struct generator * g, struct node * p) { int keep_c = 0; if (K_needed(g, p->left)) { - wp(g, "~{~k~C", p); + writef(g, "~{~k~C", p); keep_c = g->keep_count; } else { - wp(g, "~M~C", p); + writef(g, "~M~C", p); } p = p->left; - until (p == 0) { + while (p) { generate(g, p); if (keep_c && p->right != 0) { w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N"); @@ -439,19 +553,19 @@ static void generate_or(struct generator * g, struct node * p) { int used = g->label_used; int a0 = g->failure_label; - const char * a1 = g->failure_string; + int a1 = g->failure_keep_count; int out_lab = new_label(g); if (K_needed(g, p->left)) { - wp(g, "~{~k~C", p); + writef(g, "~{~k~C", p); keep_c = g->keep_count; } else { - wp(g, "~M~C", p); + writef(g, "~M~C", p); } p = p->left; - g->failure_string = 0; - until (p->right == 0) { + g->failure_keep_count = 0; + while (p->right) { g->failure_label = new_label(g); g->label_used = 0; generate(g, p); @@ -465,7 +579,7 @@ static void generate_or(struct generator * g, struct node * p) { } g->label_used = used; g->failure_label = a0; - g->failure_string = a1; + g->failure_keep_count = a1; generate(g, p); if (keep_c) w(g, "~}"); @@ -474,7 +588,7 @@ static void generate_or(struct generator * g, struct node * p) { static void generate_backwards(struct generator * g, struct node * p) { - wp(g,"~Mz->lb = z->c; z->c = z->l;~C~N", p); + writef(g, "~Mz->lb = z->c; z->c = z->l;~C~N", p); generate(g, p->left); w(g, "~Mz->c = z->lb;~N"); } @@ -485,18 +599,18 @@ static void generate_not(struct generator * g, struct node * p) { int used = g->label_used; int a0 = g->failure_label; - const char * a1 = g->failure_string; + int a1 = g->failure_keep_count; if (K_needed(g, p->left)) { - wp(g, "~{~k~C", p); + writef(g, "~{~k~C", p); keep_c = g->keep_count; } else { - wp(g, "~M~C", p); + writef(g, "~M~C", p); } g->failure_label = new_label(g); g->label_used = 0; - g->failure_string = 0; + g->failure_keep_count = 0; generate(g, p->left); { @@ -505,9 +619,9 @@ static void generate_not(struct generator * g, struct node * p) { g->label_used = used; g->failure_label = a0; - g->failure_string = a1; + g->failure_keep_count = a1; - w(g, "~M~f~N"); + writef(g, "~M~f~N", p); if (u) wsetl(g, l); } @@ -518,20 +632,14 @@ static void generate_not(struct generator * g, struct node * p) { static void generate_try(struct generator * g, struct node * p) { - int keep_c = K_needed(g, p->left); - - if (keep_c) { - if (p->mode == m_forward) { - wp(g, "~{int c_keep = z->c;~C", p); - g->failure_string = "z->c = c_keep;"; - } else { - wp(g, "~{int m_keep = z->l - z->c;/* (void) m_keep;*/~C", p); - g->failure_string = "z->c = z->l - m_keep;"; - } + int keep_c = 0; + if (K_needed(g, p->left)) { + writef(g, "~{~k~C", p); + keep_c = g->keep_count; } else { - wp(g, "~M~C", p); - g->failure_string = 0; + writef(g, "~M~C", p); } + g->failure_keep_count = keep_c; g->failure_label = new_label(g); g->label_used = 0; @@ -544,47 +652,65 @@ static void generate_try(struct generator * g, struct node * p) { } static void generate_set(struct generator * g, struct node * p) { - g->V[0] = p->name; wp(g, "~M~V0 = 1;~C", p); + g->V[0] = p->name; writef(g, "~M~V0 = 1;~C", p); } static void generate_unset(struct generator * g, struct node * p) { - g->V[0] = p->name; wp(g, "~M~V0 = 0;~C", p); + g->V[0] = p->name; writef(g, "~M~V0 = 0;~C", p); } static void generate_fail(struct generator * g, struct node * p) { generate(g, p->left); - wp(g, "~M~f~C", p); + writef(g, "~M~f~C", p); } /* generate_test() also implements 'reverse' */ static void generate_test(struct generator * g, struct node * p) { - int keep_c = K_needed(g, p->left); - if (keep_c) wp(g, "~{~K~C", p); - else wp(g, "~M~C", p); + int keep_c = 0; + if (K_needed(g, p->left)) { + keep_c = ++g->keep_count; + w(g, p->mode == m_forward ? "~{int c_test" : + "~{int m_test"); + write_int(g, keep_c); + w(g, p->mode == m_forward ? " = z->c;" : + " = z->l - z->c;"); + writef(g, "~C", p); + } else writef(g, "~M~C", p); generate(g, p->left); - if (keep_c) wp(g, "~M~R~N" - "~}", p); + if (keep_c) { + w(g, p->mode == m_forward ? "~Mz->c = c_test" : + "~Mz->c = z->l - m_test"); + write_int(g, keep_c); + writef(g, ";~N~}", p); + } } static void generate_do(struct generator * g, struct node * p) { int keep_c = 0; if (K_needed(g, p->left)) { - wp(g, "~{~k~C", p); + writef(g, "~{~k~C", p); keep_c = g->keep_count; } else { - wp(g, "~M~C", p); + writef(g, "~M~C", p); } - g->failure_label = new_label(g); - g->label_used = 0; - g->failure_string = 0; - generate(g, p->left); + if (p->left->type == c_call) { + /* Optimise do <call> */ + g->V[0] = p->left->name; + writef(g, "~{int ret = ~V0(z);~C", p->left); + w(g, "~Mif (ret < 0) return ret;~N~}"); + } else { + g->failure_label = new_label(g); + g->label_used = 0; + g->failure_keep_count = 0; + generate(g, p->left); - if (g->label_used) - wsetl(g, g->failure_label); + if (g->label_used) + wsetl(g, g->failure_label); + } if (keep_c) { w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N~}"); @@ -592,17 +718,17 @@ static void generate_do(struct generator * g, struct node * p) { } static void generate_next(struct generator * g, struct node * p) { - if (g->options->utf8) { + if (g->options->encoding == ENC_UTF8) { if (p->mode == m_forward) w(g, "~{int ret = skip_utf8(z->p, z->c, 0, z->l, 1"); else w(g, "~{int ret = skip_utf8(z->p, z->c, z->lb, 0, -1"); - wp(g, ");~N" + writef(g, ");~N" "~Mif (ret < 0) ~f~N" "~Mz->c = ret;~C" "~}", p); } else - wp(g, "~M~l~N" + writef(g, "~M~l~N" "~M~i~C", p); } @@ -611,21 +737,21 @@ static void generate_GO_grouping(struct generator * g, struct node * p, int is_g struct grouping * q = p->name->grouping; g->S[0] = p->mode == m_forward ? "" : "_b"; g->S[1] = complement ? "in" : "out"; - g->S[2] = g->options->utf8 ? "_U" : ""; + g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : ""; g->V[0] = p->name; g->I[0] = q->smallest_ch; g->I[1] = q->largest_ch; if (is_goto) { - wp(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1) < 0) ~f /* goto */~C", p); + writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1) < 0) ~f~C", p); } else { - wp(g, "~{ /* gopast */~C" - "~Mint ret = ~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1);~N" - "~Mif (ret < 0) ~f~N", p); - if (p->mode == m_forward) - w(g, "~Mz->c += ret;~N"); - else - w(g, "~Mz->c -= ret;~N"); - w(g, "~}"); + writef(g, "~{~C" + "~Mint ret = ~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 1);~N" + "~Mif (ret < 0) ~f~N", p); + if (p->mode == m_forward) + w(g, "~Mz->c += ret;~N"); + else + w(g, "~Mz->c -= ret;~N"); + w(g, "~}"); } } @@ -634,23 +760,26 @@ static void generate_GO(struct generator * g, struct node * p, int style) { int used = g->label_used; int a0 = g->failure_label; - const char * a1 = g->failure_string; + int a1 = g->failure_keep_count; if (p->left->type == c_grouping || p->left->type == c_non) { - /* Special case for "goto" or "gopast" when used on a grouping or an - * inverted grouping - the movement of c by the matching action is - * exactly what we want! */ + /* Special case for "goto" or "gopast" when used on a grouping or an + * inverted grouping - the movement of c by the matching action is + * exactly what we want! */ #ifdef OPTIMISATION_WARNINGS - printf("Optimising %s %s\n", style ? "goto" : "gopast", p->left->type == c_non ? "non" : "grouping"); + printf("Optimising %s %s\n", style ? "goto" : "gopast", p->left->type == c_non ? "non" : "grouping"); #endif - generate_GO_grouping(g, p->left, style, p->left->type == c_non); - return; + if (g->options->comments) { + writef(g, "~M~C", p); + } + generate_GO_grouping(g, p->left, style, p->left->type == c_non); + return; } - w(g, "~Mwhile(1) {"); wp(g, "~C~+", p); + w(g, "~Mwhile(1) {"); writef(g, "~C~+", p); if (style == 1 || repeat_restore(g, p->left)) { - wp(g, "~M~k~N", p); + writef(g, "~M~k~N", p); keep_c = g->keep_count; } @@ -671,16 +800,16 @@ static void generate_GO(struct generator * g, struct node * p, int style) { g->label_used = used; g->failure_label = a0; - g->failure_string = a1; + g->failure_keep_count = a1; -/* wp(g, "~M~l~N" +/* writef(g, "~M~l~N" "~M~i~N", p); */ generate_next(g, p); w(g, "~}"); } static void generate_loop(struct generator * g, struct node * p) { - w(g, "~{int i; for (i = "); generate_AE(g, p->AE); wp(g, "; i > 0; i--)~C" + w(g, "~{int i; for (i = "); generate_AE(g, p->AE); writef(g, "; i > 0; i--)~C" "~{", p); generate(g, p->left); @@ -689,18 +818,22 @@ static void generate_loop(struct generator * g, struct node * p) { "~}"); } -static void generate_repeat(struct generator * g, struct node * p, int atleast_case) { +static void generate_repeat_or_atleast(struct generator * g, struct node * p, int atleast_case) { int keep_c = 0; - wp(g, "~Mwhile(1) {~C~+", p); + if (atleast_case) { + writef(g, "~Mwhile(1) {~+~N", p); + } else { + writef(g, "~Mwhile(1) {~+~C", p); + } if (repeat_restore(g, p->left)) { - wp(g, "~M~k~N", p); + writef(g, "~M~k~N", p); keep_c = g->keep_count; } g->failure_label = new_label(g); g->label_used = 0; - g->failure_string = 0; + g->failure_keep_count = 0; generate(g, p->left); if (atleast_case) w(g, "~Mi--;~N"); @@ -717,130 +850,120 @@ static void generate_repeat(struct generator * g, struct node * p, int atleast_c "~}"); } +static void generate_repeat(struct generator * g, struct node * p) { + generate_repeat_or_atleast(g, p, false); +} + static void generate_atleast(struct generator * g, struct node * p) { - w(g, "~{int i = "); generate_AE(g, p->AE); w(g, ";~N"); + w(g, "~{int i = "); generate_AE(g, p->AE); w(g, ";~C"); { int used = g->label_used; int a0 = g->failure_label; - const char * a1 = g->failure_string; + int a1 = g->failure_keep_count; - generate_repeat(g, p, true); + generate_repeat_or_atleast(g, p, true); g->label_used = used; g->failure_label = a0; - g->failure_string = a1; + g->failure_keep_count = a1; } - w(g, "~Mif (i > 0) ~f~N" - "~}"); + writef(g, "~Mif (i > 0) ~f~N" + "~}", p); } static void generate_setmark(struct generator * g, struct node * p) { g->V[0] = p->name; - wp(g, "~M~V0 = z->c;~C", p); + writef(g, "~M~V0 = z->c;~C", p); } static void generate_tomark(struct generator * g, struct node * p) { g->S[0] = p->mode == m_forward ? ">" : "<"; - w(g, "~Mif (z->c ~S0 "); generate_AE(g, p->AE); w(g, ") ~f~N"); - w(g, "~Mz->c = "); generate_AE(g, p->AE); wp(g, ";~C", p); + w(g, "~Mif (z->c ~S0 "); generate_AE(g, p->AE); writef(g, ") ~f~N", p); + w(g, "~Mz->c = "); generate_AE(g, p->AE); writef(g, ";~C", p); } static void generate_atmark(struct generator * g, struct node * p) { - w(g, "~Mif (z->c != "); generate_AE(g, p->AE); wp(g, ") ~f~C", p); + w(g, "~Mif (z->c != "); generate_AE(g, p->AE); writef(g, ") ~f~C", p); } static void generate_hop(struct generator * g, struct node * p) { g->S[0] = p->mode == m_forward ? "+" : "-"; g->S[1] = p->mode == m_forward ? "0" : "z->lb"; - if (g->options->utf8) { + if (g->options->encoding == ENC_UTF8) { w(g, "~{int ret = skip_utf8(z->p, z->c, ~S1, z->l, ~S0 "); - generate_AE(g, p->AE); wp(g, ");~C", p); - w(g, "~Mif (ret < 0) ~f~N"); + generate_AE(g, p->AE); writef(g, ");~C", p); + writef(g, "~Mif (ret < 0) ~f~N", p); } else { w(g, "~{int ret = z->c ~S0 "); - generate_AE(g, p->AE); wp(g, ";~C", p); - w(g, "~Mif (~S1 > ret || ret > z->l) ~f~N"); + generate_AE(g, p->AE); writef(g, ";~C", p); + writef(g, "~Mif (~S1 > ret || ret > z->l) ~f~N", p); } - wp(g, "~Mz->c = ret;~C" + writef(g, "~Mz->c = ret;~N" "~}", p); } static void generate_delete(struct generator * g, struct node * p) { - wp(g, "~{int ret = slice_del(z);~C", p); - wp(g, "~Mif (ret < 0) return ret;~N" + writef(g, "~{int ret = slice_del(z);~C", p); + writef(g, "~Mif (ret < 0) return ret;~N" "~}", p); } static void generate_tolimit(struct generator * g, struct node * p) { g->S[0] = p->mode == m_forward ? "" : "b"; - wp(g, "~Mz->c = z->l~S0;~C", p); + writef(g, "~Mz->c = z->l~S0;~C", p); } static void generate_atlimit(struct generator * g, struct node * p) { g->S[0] = p->mode == m_forward ? "" : "b"; g->S[1] = p->mode == m_forward ? "<" : ">"; - wp(g, "~Mif (z->c ~S1 z->l~S0) ~f~C", p); + writef(g, "~Mif (z->c ~S1 z->l~S0) ~f~C", p); } static void generate_leftslice(struct generator * g, struct node * p) { g->S[0] = p->mode == m_forward ? "bra" : "ket"; - wp(g, "~Mz->~S0 = z->c;~C", p); + writef(g, "~Mz->~S0 = z->c;~C", p); } static void generate_rightslice(struct generator * g, struct node * p) { g->S[0] = p->mode == m_forward ? "ket" : "bra"; - wp(g, "~Mz->~S0 = z->c;~C", p); + writef(g, "~Mz->~S0 = z->c;~C", p); } static void generate_assignto(struct generator * g, struct node * p) { g->V[0] = p->name; - wp(g, "~M~V0 = assign_to(z, ~V0);~C" + writef(g, "~M~V0 = assign_to(z, ~V0);~C" "~Mif (~V0 == 0) return -1;~C", p); } static void generate_sliceto(struct generator * g, struct node * p) { g->V[0] = p->name; - wp(g, "~M~V0 = slice_to(z, ~V0);~C" - "~Mif (~V0 == 0) return -1;~C", p); -} - -static void generate_data_address(struct generator * g, struct node * p) { - - symbol * b = p->literalstring; - if (b != 0) { - wi(g, SIZE(b)); w(g, ", "); - wlitref(g, b); - } else - wv(g, p->name); + writef(g, "~M~V0 = slice_to(z, ~V0);~C" + "~Mif (~V0 == 0) return -1;~N", p); } static void generate_insert(struct generator * g, struct node * p, int style) { int keep_c = style == c_attach; if (p->mode == m_backward) keep_c = !keep_c; - wp(g, "~{", p); - if (keep_c) w(g, "int c_keep = z->c;~N~M"); - wp(g, "int ret = insert_~$(z, z->c, z->c, ", p); - generate_data_address(g, p); - wp(g, ");~C", p); - if (keep_c) w(g, "~Mz->c = c_keep;~N"); - wp(g, "~Mif (ret < 0) return ret;~N" + writef(g, "~{int ret;~N", p); + if (keep_c) w(g, "~{int saved_c = z->c;~N"); + writef(g, "~Mret = insert_~$(z, z->c, z->c, ~a);~C", p); + if (keep_c) w(g, "~Mz->c = saved_c;~N~}"); + writef(g, "~Mif (ret < 0) return ret;~N" "~}", p); } static void generate_assignfrom(struct generator * g, struct node * p) { int keep_c = p->mode == m_forward; /* like 'attach' */ - wp(g, "~{", p); - if (keep_c) wp(g, "int c_keep = z->c;~N" - "~Mret = insert_~$(z, z->c, z->l, ", p); - else wp(g, "ret = insert_~$(z, z->lb, z->c, ", p); - generate_data_address(g, p); - wp(g, ");~C", p); - if (keep_c) w(g, "~Mz->c = c_keep;~N"); - wp(g, "~Mif (ret < 0) return ret;~N" + writef(g, "~{int ret;~N", p); + if (keep_c) writef(g, "~{int saved_c = z->c;~N", p); + w(g, "~Mret = "); + writef(g, keep_c ? "insert_~$(z, z->c, z->l, ~a);~C" : "insert_~$(z, z->lb, z->c, ~a);~C", p); + if (keep_c) w(g, "~Mz->c = saved_c;~N~}"); + writef(g, "~Mif (ret < 0) return ret;~N" "~}", p); } @@ -849,81 +972,132 @@ static void generate_assignfrom(struct generator * g, struct node * p) { static void generate_slicefrom(struct generator * g, struct node * p) { /* w(g, "~Mslice_from_s(z, "); <============= bug! should be: */ - wp(g, "~{int ret = slice_from_~$(z, ", p); - generate_data_address(g, p); - wp(g, ");~C", p); - wp(g, "~Mif (ret < 0) return ret;~N" + writef(g, "~{int ret = slice_from_~$(z, ~a);~C", p); + writef(g, "~Mif (ret < 0) return ret;~N" "~}", p); } static void generate_setlimit(struct generator * g, struct node * p) { int keep_c; - wp(g, "~{int mlimit;~C" - "~M~k~N" - , p); - keep_c = g->keep_count; - generate(g, p->left); - if (p->mode == m_forward) w(g, "~Mmlimit = z->l - z->c; z->l = z->c;~N"); - else w(g, "~Mmlimit = z->lb; z->lb = z->c;~N"); - w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N"); - g->failure_string = p->mode == m_forward ? "z->l += mlimit;" : - "z->lb = mlimit;"; + if (p->left && p->left->type == c_tomark) { + /* Special case for: + * + * setlimit tomark AE for C + * + * All uses of setlimit in the current stemmers we ship follow this + * pattern, and by special-casing we can avoid having to save and + * restore c. + */ + struct node * q = p->left; + + ++g->keep_count; + writef(g, "~N~{int mlimit", p); + write_int(g, g->keep_count); + writef(g, ";~C", p); + keep_c = g->keep_count; + + g->S[0] = q->mode == m_forward ? ">" : "<"; + + w(g, "~Mif (z->c ~S0 "); generate_AE(g, q->AE); writef(g, ") ~f~N", q); + w(g, "~Mmlimit"); + write_int(g, keep_c); + if (p->mode == m_forward) { + w(g, " = z->l - z->c; z->l = "); + } else { + w(g, " = z->lb; z->lb = "); + } + generate_AE(g, q->AE); + w(g, ";~N"); + } else { + writef(g, "~{~K~C", p); + keep_c = g->keep_count; + generate(g, p->left); + + w(g, "~Mmlimit"); + write_int(g, keep_c); + if (p->mode == m_forward) + w(g, " = z->l - z->c; z->l = z->c;~N"); + else + w(g, " = z->lb; z->lb = z->c;~N"); + w(g, "~M"); wrestore(g, p, keep_c); w(g, "~N"); + } + + g->failure_keep_count = -keep_c; generate(g, p->aux); - wms(g, g->failure_string); + w(g, "~M"); + wrestorelimit(g, p, -g->failure_keep_count); w(g, "~N" "~}"); } +/* dollar sets snowball up to operate on a string variable as if it were the + * current string */ static void generate_dollar(struct generator * g, struct node * p) { int used = g->label_used; int a0 = g->failure_label; - const char * a1 = g->failure_string; + int a1 = g->failure_keep_count; + int keep_token; g->failure_label = new_label(g); g->label_used = 0; - g->failure_string = 0; + g->failure_keep_count = 0; + keep_token = ++g->keep_count; + g->I[0] = keep_token; + writef(g, "~{struct SN_env env~I0 = * z;~C", p); g->V[0] = p->name; - wp(g, "~{struct SN_env env = * z;~C" - "~Mint failure = 1; /* assume failure */~N" - "~Mz->p = ~V0;~N" - "~Mz->lb = z->c = 0;~N" - "~Mz->l = SIZE(z->p);~N", p); + /* Assume failure. */ + writef(g, "~Mint failure = 1;~N" + "~Mz->p = ~V0;~N" + "~Mz->lb = z->c = 0;~N" + "~Mz->l = SIZE(z->p);~N", p); generate(g, p->left); - w(g, "~Mfailure = 0; /* mark success */~N"); + /* Mark success. */ + w(g, "~Mfailure = 0;~N"); if (g->label_used) wsetl(g, g->failure_label); g->V[0] = p->name; /* necessary */ g->label_used = used; g->failure_label = a0; - g->failure_string = a1; + g->failure_keep_count = a1; - w(g, "~M~V0 = z->p;~N" - "~M* z = env;~N" - "~Mif (failure) ~f~N~}"); + g->I[0] = keep_token; + writef(g, "~M~V0 = z->p;~N" + "~M* z = env~I0;~N" + "~Mif (failure) ~f~N~}", p); } static void generate_integer_assign(struct generator * g, struct node * p, char * s) { g->V[0] = p->name; g->S[0] = s; - w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); wp(g, ";~C", p); + w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); writef(g, ";~C", p); } static void generate_integer_test(struct generator * g, struct node * p, char * s) { - g->V[0] = p->name; - g->S[0] = s; - w(g, "~Mif (!(~V0 ~S0 "); generate_AE(g, p->AE); wp(g, ")) ~f~C", p); + w(g, "~Mif (!("); + generate_AE(g, p->left); + write_char(g, ' '); + write_string(g, s); + write_char(g, ' '); + generate_AE(g, p->AE); + writef(g, ")) ~f~C", p); } static void generate_call(struct generator * g, struct node * p) { g->V[0] = p->name; - wp(g, "~{int ret = ~V0(z);~C" - "~Mif (ret == 0) ~f~N" - "~Mif (ret < 0) return ret;~N~}", p); + writef(g, "~{int ret = ~V0(z);~C", p); + if (g->failure_keep_count == 0 && g->failure_label == x_return) { + /* Combine the two tests in this special case for better optimisation + * and clearer generated code. */ + writef(g, "~Mif (ret <= 0) return ret;~N~}", p); + } else { + writef(g, "~Mif (ret == 0) ~f~N" + "~Mif (ret < 0) return ret;~N~}", p); + } } static void generate_grouping(struct generator * g, struct node * p, int complement) { @@ -931,27 +1105,47 @@ static void generate_grouping(struct generator * g, struct node * p, int complem struct grouping * q = p->name->grouping; g->S[0] = p->mode == m_forward ? "" : "_b"; g->S[1] = complement ? "out" : "in"; - g->S[2] = g->options->utf8 ? "_U" : ""; + g->S[2] = g->options->encoding == ENC_UTF8 ? "_U" : ""; g->V[0] = p->name; g->I[0] = q->smallest_ch; g->I[1] = q->largest_ch; - wp(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 0)) ~f~C", p); + writef(g, "~Mif (~S1_grouping~S0~S2(z, ~V0, ~I0, ~I1, 0)) ~f~C", p); } static void generate_namedstring(struct generator * g, struct node * p) { g->S[0] = p->mode == m_forward ? "" : "_b"; g->V[0] = p->name; - wp(g, "~Mif (!(eq_v~S0(z, ~V0))) ~f~C", p); + writef(g, "~Mif (!(eq_v~S0(z, ~V0))) ~f~C", p); } static void generate_literalstring(struct generator * g, struct node * p) { symbol * b = p->literalstring; - g->S[0] = p->mode == m_forward ? "" : "_b"; - g->I[0] = SIZE(b); - g->L[0] = b; + if (SIZE(b) == 1) { + /* It's quite common to compare with a single character literal string, + * so just inline the simpler code for this case rather than making a + * function call. In UTF-8 mode, only do this for the ASCII subset, + * since multi-byte characters are more complex to test against. + */ + if (g->options->encoding == ENC_UTF8 && *b >= 128) { + printf("single byte %d\n", *b); + exit(1); + } + g->I[0] = *b; + if (p->mode == m_forward) { + writef(g, "~Mif (z->c == z->l || z->p[z->c] != ~c0) ~f~C" + "~Mz->c++;~N", p); + } else { + writef(g, "~Mif (z->c <= z->lb || z->p[z->c - 1] != ~c0) ~f~C" + "~Mz->c--;~N", p); + } + } else { + g->S[0] = p->mode == m_forward ? "" : "_b"; + g->I[0] = SIZE(b); + g->L[0] = b; - wp(g, "~Mif (!(eq_s~S0(z, ~I0, ~L0))) ~f~C", p); + writef(g, "~Mif (!(eq_s~S0(z, ~I0, ~L0))) ~f~C", p); + } } static void generate_define(struct generator * g, struct node * p) { @@ -961,9 +1155,13 @@ static void generate_define(struct generator * g, struct node * p) { g->S[0] = q->type == t_routine ? "static" : "extern"; g->V[0] = q; - w(g, "~N~S0 int ~V0(struct SN_env * z) {~N~+"); + w(g, "~N~S0 int ~V0(struct SN_env * z) {"); + if (g->options->comments) { + write_string(g, p->mode == m_forward ? " /* forwardmode */" : " /* backwardmode */"); + } + w(g, "~N~+"); if (p->amongvar_needed) w(g, "~Mint among_var;~N"); - g->failure_string = 0; + g->failure_keep_count = 0; g->failure_label = x_return; g->label_used = 0; g->keep_count = 0; @@ -982,6 +1180,7 @@ static void generate_substring(struct generator * g, struct node * p) { int n_cases = 0; symbol cases[2]; int shortest_size = INT_MAX; + int shown_comment = 0; g->S[0] = p->mode == m_forward ? "" : "_b"; g->I[0] = x->number; @@ -1019,19 +1218,19 @@ static void generate_substring(struct generator * g, struct node * p) { if (n_cases > 2) break; } if (block == -1) { - if (ch == cases[0]) continue; + if (n_cases > 0 && ch == cases[0]) continue; if (n_cases < 2) { - cases[n_cases++] = ch; + cases[n_cases++] = ch; } else if (ch != cases[1]) { - ++n_cases; - break; + ++n_cases; + break; } } else { if ((bitmap & (1u << (ch & 0x1f))) == 0) { - bitmap |= 1u << (ch & 0x1f); - if (n_cases < 2) - cases[n_cases] = ch; - ++n_cases; + bitmap |= 1u << (ch & 0x1f); + if (n_cases < 2) + cases[n_cases] = ch; + ++n_cases; } } } @@ -1045,16 +1244,16 @@ static void generate_substring(struct generator * g, struct node * p) { sprintf(buf, "z->p[z->c + %d]", shortest_size - 1); g->S[1] = buf; if (shortest_size == 1) { - wp(g, "~Mif (z->c >= z->l || ", p); + writef(g, "~Mif (z->c >= z->l", p); } else { - wp(g, "~Mif (z->c + ~I4 >= z->l || ", p); + writef(g, "~Mif (z->c + ~I4 >= z->l", p); } } else { g->S[1] = "z->p[z->c - 1]"; if (shortest_size == 1) { - wp(g, "~Mif (z->c <= z->lb || ", p); + writef(g, "~Mif (z->c <= z->lb", p); } else { - wp(g, "~Mif (z->c - ~I4 <= z->lb || ", p); + writef(g, "~Mif (z->c - ~I4 <= z->lb", p); } } if (n_cases == 0) { @@ -1062,81 +1261,83 @@ static void generate_substring(struct generator * g, struct node * p) { * This doesn't seem to be a useful construct, but it is * syntactically valid. */ - wp(g, "0", p); } else if (n_cases == 1) { g->I[4] = cases[0]; - wp(g, "~S1 != ~I4", p); + writef(g, " || ~S1 != ~I4", p); } else if (n_cases == 2) { g->I[4] = cases[0]; g->I[5] = cases[1]; - wp(g, "(~S1 != ~I4 && ~S1 != ~I5)", p); + writef(g, " || (~S1 != ~I4 && ~S1 != ~I5)", p); } else { - wp(g, "~S1 >> 5 != ~I2 || !((~I3 >> (~S1 & 0x1f)) & 1)", p); + writef(g, " || ~S1 >> 5 != ~I2 || !((~I3 >> (~S1 & 0x1f)) & 1)", p); } - ws(g, ") "); + write_string(g, ") "); if (empty_case != -1) { /* If the among includes the empty string, it can never fail * so not matching the bitmap means we match the empty string. */ g->I[4] = among_cases[empty_case].result; - wp(g, "among_var = ~I4; else~C", p); + writef(g, "among_var = ~I4; else~C", p); } else { - wp(g, "~f~C", p); + writef(g, "~f~C", p); } + shown_comment = 1; } else { #ifdef OPTIMISATION_WARNINGS printf("Couldn't shortcut among %d\n", x->number); #endif } - if (x->command_count == 0 && x->starter == 0) - wp(g, "~Mif (!(find_among~S0(z, a_~I0, ~I1))) ~f~C", p); - else - wp(g, "~Mamong_var = find_among~S0(z, a_~I0, ~I1);~C" - "~Mif (!(among_var)) ~f~N", p); + if (!x->amongvar_needed) { + writef(g, "~Mif (!(find_among~S0(z, a_~I0, ~I1))) ~f", p); + writef(g, shown_comment ? "~N" : "~C", p); + } else { + writef(g, "~Mamong_var = find_among~S0(z, a_~I0, ~I1);", p); + writef(g, shown_comment ? "~N" : "~C", p); + writef(g, "~Mif (!(among_var)) ~f~N", p); + } } static void generate_among(struct generator * g, struct node * p) { struct among * x = p->among; - int case_number = 1; if (x->substring == 0) generate_substring(g, p); - if (x->command_count == 0 && x->starter == 0) return; - unless (x->starter == 0) generate(g, x->starter); + if (x->starter != 0) generate(g, x->starter); - p = p->left; - if (p != 0 && p->type != c_literalstring) p = p->right; - w(g, "~Mswitch(among_var) {~N~+" - "~Mcase 0: ~f~N"); - - until (p == 0) { - if (p->type == c_bra && p->left != 0) { - g->I[0] = case_number++; - w(g, "~Mcase ~I0:~N~+"); generate(g, p); w(g, "~Mbreak;~N~-"); - } - p = p->right; + if (x->command_count == 1 && x->nocommand_count == 0) { + /* Only one outcome ("no match" already handled). */ + generate(g, x->commands[0]); + } else if (x->command_count > 0) { + int i; + writef(g, "~Mswitch (among_var) {~C~+", p); + for (i = 1; i <= x->command_count; i++) { + g->I[0] = i; + w(g, "~Mcase ~I0:~N~+"); + generate(g, x->commands[i - 1]); + w(g, "~Mbreak;~N~-"); + } + w(g, "~}"); } - w(g, "~}"); } static void generate_booltest(struct generator * g, struct node * p) { g->V[0] = p->name; - wp(g, "~Mif (!(~V0)) ~f~C", p); + writef(g, "~Mif (!(~V0)) ~f~C", p); } static void generate_false(struct generator * g, struct node * p) { - wp(g, "~M~f~C", p); + writef(g, "~M~f~C", p); } static void generate_debug(struct generator * g, struct node * p) { g->I[0] = g->debug_count++; g->I[1] = p->line_number; - wp(g, "~Mdebug(z, ~I0, ~I1);~C", p); + writef(g, "~Mdebug(z, ~I0, ~I1);~C", p); } @@ -1144,10 +1345,9 @@ static void generate(struct generator * g, struct node * p) { int used = g->label_used; int a0 = g->failure_label; - const char * a1 = g->failure_string; + int a1 = g->failure_keep_count; - switch (p->type) - { + switch (p->type) { case c_define: generate_define(g, p); break; case c_bra: generate_bra(g, p); break; case c_and: generate_and(g, p); break; @@ -1163,7 +1363,7 @@ static void generate(struct generator * g, struct node * p) { case c_do: generate_do(g, p); break; case c_goto: generate_GO(g, p, 1); break; case c_gopast: generate_GO(g, p, 0); break; - case c_repeat: generate_repeat(g, p, false); break; + case c_repeat: generate_repeat(g, p); break; case c_loop: generate_loop(g, p); break; case c_atleast: generate_atleast(g, p); break; case c_setmark: generate_setmark(g, p); break; @@ -1213,30 +1413,40 @@ static void generate(struct generator * g, struct node * p) { if (g->failure_label != a0) g->label_used = used; g->failure_label = a0; - g->failure_string = a1; + g->failure_keep_count = a1; } -static void generate_start_comment(struct generator * g) { +void write_generated_comment_content(struct generator * g) { + w(g, "Generated by Snowball " SNOWBALL_VERSION + " - https://snowballstem.org/"); +} - w(g, "~N/* This file was generated automatically by the Snowball to ANSI C compiler */~N"); +void write_start_comment(struct generator * g, + const char * comment_start, + const char * comment_end) { + write_margin(g); + w(g, comment_start); + write_generated_comment_content(g); + if (comment_end) { + w(g, comment_end); + } + w(g, "~N~N"); } static void generate_head(struct generator * g) { - if (g->options->runtime_path == 0) { - w(g, "~N#include \"header.h\"~N~N"); - } else { - w(g, "~N#include \""); - ws(g, g->options->runtime_path); + w(g, "#include \""); + if (g->options->runtime_path) { + write_string(g, g->options->runtime_path); if (g->options->runtime_path[strlen(g->options->runtime_path) - 1] != '/') - wch(g, '/'); - w(g, "header.h\"~N~N"); + write_char(g, '/'); } + w(g, "header.h\"~N~N"); } static void generate_routine_headers(struct generator * g) { - struct name * q = g->analyser->names; - until (q == 0) { + struct name * q; + for (q = g->analyser->names; q; q = q->next) { g->V[0] = q; switch (q->type) { case t_routine: @@ -1254,7 +1464,6 @@ static void generate_routine_headers(struct generator * g) { ); break; } - q = q->next; } } @@ -1265,12 +1474,11 @@ static void generate_among_table(struct generator * g, struct among * x) { g->I[0] = x->number; { int i; - for (i = 0; i < x->literalstring_count; i++) - { + for (i = 0; i < x->literalstring_count; i++) { g->I[1] = i; g->I[2] = v->size; g->L[0] = v->b; - unless (v->size == 0) + if (v->size) w(g, "static const symbol s_~I0_~I1[~I2] = ~A0;~N"); v++; } @@ -1289,12 +1497,21 @@ static void generate_among_table(struct generator * g, struct among * x) { g->I[4] = v->result; g->S[0] = i < x->literalstring_count - 1 ? "," : ""; - w(g, "/*~J1 */ { ~I2, "); - if (v->size == 0) w(g, "0,"); - else w(g, "s_~I0_~I1,"); + if (g->options->comments) { + w(g, "/*~J1 */ "); + } + w(g, "{ ~I2, "); + if (v->size == 0) { + w(g, "0,"); + } else { + w(g, "s_~I0_~I1,"); + } w(g, " ~I3, ~I4, "); - if (v->function == 0) w(g, "0"); else - wvn(g, v->function); + if (v->function == 0) { + write_char(g, '0'); + } else { + write_varname(g, v->function); + } w(g, "}~S0~N"); v++; } @@ -1303,10 +1520,9 @@ static void generate_among_table(struct generator * g, struct among * x) { } static void generate_amongs(struct generator * g) { - struct among * x = g->analyser->amongs; - until (x == 0) { + struct among * x; + for (x = g->analyser->amongs; x; x = x->next) { generate_among_table(g, x); - x = x->next; } } @@ -1323,24 +1539,22 @@ static void generate_grouping_table(struct generator * g, struct grouping * q) { for (i = 0; i < SIZE(b); i++) set_bit(map, b[i] - q->smallest_ch); - { - g->V[0] = q->name; + g->V[0] = q->name; - w(g, "static const unsigned char ~V0[] = { "); - for (i = 0; i < size; i++) { - wi(g, map[i]); - if (i < size - 1) w(g, ", "); - } - w(g, " };~N~N"); + w(g, "static const unsigned char ~V0[] = { "); + for (i = 0; i < size; i++) { + write_int(g, map[i]); + if (i < size - 1) w(g, ", "); } + w(g, " };~N~N"); lose_b(map); } static void generate_groupings(struct generator * g) { - struct grouping * q = g->analyser->groupings; - until (q == 0) { - generate_grouping_table(g, q); - q = q->next; + struct grouping * q; + for (q = g->analyser->groupings; q; q = q->next) { + if (q->name->used) + generate_grouping_table(g, q); } } @@ -1348,10 +1562,9 @@ static void generate_create(struct generator * g) { int * p = g->analyser->name_count; g->I[0] = p[t_string]; - g->I[1] = p[t_integer]; - g->I[2] = p[t_boolean]; + g->I[1] = p[t_integer] + p[t_boolean]; w(g, "~N" - "extern struct SN_env * ~pcreate_env(void) { return SN_create_env(~I0, ~I1, ~I2); }" + "extern struct SN_env * ~pcreate_env(void) { return SN_create_env(~I0, ~I1); }" "~N"); } @@ -1371,36 +1584,47 @@ static void generate_create_and_close_templates(struct generator * g) { static void generate_header_file(struct generator * g) { - struct name * q = g->analyser->names; - char * vp = g->options->variables_prefix; + struct name * q; + const char * vp = g->options->variables_prefix; g->S[0] = vp; - w(g, "~N" - "#ifdef __cplusplus~N" + w(g, "#ifdef __cplusplus~N" "extern \"C\" {~N" "#endif~N"); /* for C++ */ generate_create_and_close_templates(g); - until (q == 0) { + for (q = g->analyser->names; q; q = q->next) { g->V[0] = q; - switch (q->type) - { + switch (q->type) { case t_external: w(g, "extern int ~W0(struct SN_env * z);~N"); break; - case t_string: g->S[1] = "S"; goto label0; - case t_integer: g->S[1] = "I"; goto label0; - case t_boolean: g->S[1] = "B"; - label0: + case t_string: + case t_integer: + case t_boolean: if (vp) { - g->I[0] = q->count; + int count = q->count; + if (count < 0) { + /* Unused variables should get removed from `names`. */ + fprintf(stderr, "Optimised out variable "); + report_b(stderr, q->b); + fprintf(stderr, " still in names list\n"); + exit(1); + } + if (q->type == t_boolean) { + /* We use a single array for booleans and integers, + * with the integers first. + */ + count += g->analyser->name_count[t_integer]; + } + g->I[0] = count; + g->I[1] = "SIIrxg"[q->type]; w(g, "#define ~S0"); - str_append_b(g->outbuf, q->b); - w(g, " (~S1[~I0])~N"); + write_b(g, q->b); + w(g, " (~c1[~I0])~N"); } break; } - q = q->next; } w(g, "~N" @@ -1414,7 +1638,7 @@ static void generate_header_file(struct generator * g) { extern void generate_program_c(struct generator * g) { g->outbuf = str_new(); - generate_start_comment(g); + write_start_comment(g, "/* ", " */"); generate_head(g); generate_routine_headers(g); w(g, "#ifdef __cplusplus~N" @@ -1433,33 +1657,69 @@ extern void generate_program_c(struct generator * g) { g->literalstring_count = 0; { struct node * p = g->analyser->program; - until (p == 0) { generate(g, p); p = p->right; } + while (p) { generate(g, p); p = p->right; } } generate_create(g); generate_close(g); - output_str(g->options->output_c, g->declarations); + output_str(g->options->output_src, g->declarations); str_delete(g->declarations); - output_str(g->options->output_c, g->outbuf); + output_str(g->options->output_src, g->outbuf); str_clear(g->outbuf); - generate_start_comment(g); + write_start_comment(g, "/* ", " */"); generate_header_file(g); output_str(g->options->output_h, g->outbuf); str_delete(g->outbuf); } -extern struct generator * create_generator_c(struct analyser * a, struct options * o) { +/* Generator functions common to multiple languages. */ + +extern struct generator * create_generator(struct analyser * a, struct options * o) { NEW(generator, g); g->analyser = a; g->options = o; g->margin = 0; g->debug_count = 0; + g->copy_from_count = 0; g->line_count = 0; + g->line_labelled = 0; + g->failure_label = -1; + g->unreachable = false; +#ifndef DISABLE_PYTHON + g->max_label = 0; +#endif return g; } -extern void close_generator_c(struct generator * g) { - +extern void close_generator(struct generator * g) { FREE(g); } +/* Write routines for simple entities */ + +extern void write_char(struct generator * g, int ch) { + str_append_ch(g->outbuf, ch); /* character */ +} + +extern void write_newline(struct generator * g) { + str_append_ch(g->outbuf, '\n'); /* newline */ + g->line_count++; +} + +extern void write_string(struct generator * g, const char * s) { + str_append_string(g->outbuf, s); +} + +extern void write_int(struct generator * g, int i) { + str_append_int(g->outbuf, i); +} + +extern void write_b(struct generator * g, symbol * b) { + + str_append_b(g->outbuf, b); +} + +extern void write_str(struct generator * g, struct str * str) { + + str_append(g->outbuf, str); +} diff --git a/contrib/snowball/compiler/generator_java.c b/contrib/snowball/compiler/generator_java.c deleted file mode 100644 index 07a4b8e2e..000000000 --- a/contrib/snowball/compiler/generator_java.c +++ /dev/null @@ -1,1452 +0,0 @@ - -#include <stdlib.h> /* for exit */ -#include <string.h> /* for strlen */ -#include <stdio.h> /* for fprintf etc */ -#include "header.h" - -/* prototypes */ - -static void generate(struct generator * g, struct node * p); -static void w(struct generator * g, const char * s); -static void writef(struct generator * g, const char * s, struct node * p); - - -enum special_labels { - x_return = -1 -}; - -static int new_label(struct generator * g) { - - return g->next_label++; -} - -static struct str * vars_newname(struct generator * g) { - - struct str * output; - g->var_number ++; - output = str_new(); - str_append_string(output, "v_"); - str_append_int(output, g->var_number); - return output; -} - -/* Output routines */ -static void output_str(FILE * outfile, struct str * str) { - - char * s = b_to_s(str_data(str)); - fprintf(outfile, "%s", s); - free(s); -} - -/* Write routines for simple entities */ - -static void write_char(struct generator * g, int ch) { - - str_append_ch(g->outbuf, ch); -} - -static void write_newline(struct generator * g) { - - str_append_string(g->outbuf, "\n"); -} - -static void write_string(struct generator * g, const char * s) { - - str_append_string(g->outbuf, s); -} - -static void write_b(struct generator * g, symbol * b) { - - str_append_b(g->outbuf, b); -} - -static void write_str(struct generator * g, struct str * str) { - - str_append(g->outbuf, str); -} - -static void write_int(struct generator * g, int i) { - - str_append_int(g->outbuf, i); -} - - -/* Write routines for items from the syntax tree */ - -static void write_varname(struct generator * g, struct name * p) { - - int ch = "SBIrxg"[p->type]; - if (p->type != t_external) - { - write_char(g, ch); - write_char(g, '_'); - } - str_append_b(g->outbuf, p->b); -} - -static void write_varref(struct generator * g, struct name * p) { - - /* In java, references look just the same */ - write_varname(g, p); -} - -static void write_hexdigit(struct generator * g, int n) { - - write_char(g, n < 10 ? n + '0' : n - 10 + 'A'); -} - -static void write_hex(struct generator * g, int ch) { - - write_string(g, "\\u"); - { - int i; - for (i = 12; i >= 0; i -= 4) write_hexdigit(g, ch >> i & 0xf); - } -} - -static void write_literal_string(struct generator * g, symbol * p) { - - int i; - write_string(g, "\""); - for (i = 0; i < SIZE(p); i++) { - int ch = p[i]; - if (32 <= ch && ch <= 127) { - if (ch == '\"' || ch == '\\') write_string(g, "\\"); - write_char(g, ch); - } else { - write_hex(g, ch); - } - } - write_string(g, "\""); -} - -static void write_margin(struct generator * g) { - - int i; - for (i = 0; i < g->margin; i++) write_string(g, " "); -} - -/* Write a variable declaration. */ -static void write_declare(struct generator * g, - char * declaration, - struct node * p) { - - struct str * temp = g->outbuf; - g->outbuf = g->declarations; - write_string(g, " "); - writef(g, declaration, p); - write_string(g, ";"); - write_newline(g); - g->outbuf = temp; -} - -static void write_comment(struct generator * g, struct node * p) { - - write_margin(g); - write_string(g, "// "); - write_string(g, name_of_token(p->type)); - if (p->name != 0) { - write_string(g, " "); - str_append_b(g->outbuf, p->name->b); - } - write_string(g, ", line "); - write_int(g, p->line_number); - write_newline(g); -} - -static void write_block_start(struct generator * g) { - - w(g, "~M{~+~N"); -} - -static void write_block_end(struct generator * g) /* block end */ { - - w(g, "~-~M}~N"); -} - -static void write_savecursor(struct generator * g, struct node * p, - struct str * savevar) { - - g->B[0] = str_data(savevar); - g->S[1] = ""; - if (p->mode != m_forward) g->S[1] = "limit - "; - write_declare(g, "int ~B0", p); - writef(g, "~M~B0 = ~S1cursor;~N" , p); -} - -static void restore_string(struct node * p, struct str * out, struct str * savevar) { - - str_clear(out); - str_append_string(out, "cursor = "); - if (p->mode != m_forward) str_append_string(out, "limit - "); - str_append(out, savevar); - str_append_string(out, ";"); -} - -static void write_restorecursor(struct generator * g, struct node * p, - struct str * savevar) { - - struct str * temp = str_new(); - write_margin(g); - restore_string(p, temp, savevar); - write_str(g, temp); - write_newline(g); - str_delete(temp); -} - -static void write_inc_cursor(struct generator * g, struct node * p) { - - write_margin(g); - write_string(g, p->mode == m_forward ? "cursor++;" : "cursor--;"); - write_newline(g); -} - -static void wsetlab_begin(struct generator * g, int n) { - - w(g, "~Mlab"); - write_int(g, n); - w(g, ": do {~+~N"); -} - -static void wsetlab_end(struct generator * g) { - - w(g, "~-~M} while (false);~N"); -} - -static void wgotol(struct generator * g, int n) { - - write_margin(g); - write_string(g, "break lab"); - write_int(g, n); - write_string(g, ";"); - write_newline(g); -} - -static void write_failure(struct generator * g) { - - if (str_len(g->failure_str) != 0) { - write_margin(g); - write_str(g, g->failure_str); - write_newline(g); - } - write_margin(g); - switch (g->failure_label) - { - case x_return: - write_string(g, "return false;"); - break; - default: - write_string(g, "break lab"); - write_int(g, g->failure_label); - write_string(g, ";"); - g->unreachable = true; - } - write_newline(g); -} - -static void write_failure_if(struct generator * g, char * s, struct node * p) { - - writef(g, "~Mif (", p); - writef(g, s, p); - writef(g, ")~N", p); - write_block_start(g); - write_failure(g); - write_block_end(g); - g->unreachable = false; -} - -/* if at limit fail */ -static void write_check_limit(struct generator * g, struct node * p) { - - if (p->mode == m_forward) { - write_failure_if(g, "cursor >= limit", p); - } else { - write_failure_if(g, "cursor <= limit_backward", p); - } -} - -/* Formatted write. */ -static void writef(struct generator * g, const char * input, struct node * p) { - - int i = 0; - int l = strlen(input); - - while (i < l) { - int ch = input[i++]; - if (ch == '~') { - switch(input[i++]) { - default: write_char(g, input[i - 1]); continue; - case 'C': write_comment(g, p); continue; - case 'f': write_block_start(g); - write_failure(g); - g->unreachable = false; - write_block_end(g); - continue; - case 'M': write_margin(g); continue; - case 'N': write_newline(g); continue; - case '{': write_block_start(g); continue; - case '}': write_block_end(g); continue; - case 'S': write_string(g, g->S[input[i++] - '0']); continue; - case 'B': write_b(g, g->B[input[i++] - '0']); continue; - case 'I': write_int(g, g->I[input[i++] - '0']); continue; - case 'V': write_varref(g, g->V[input[i++] - '0']); continue; - case 'W': write_varname(g, g->V[input[i++] - '0']); continue; - case 'L': write_literal_string(g, g->L[input[i++] - '0']); continue; - case '+': g->margin++; continue; - case '-': g->margin--; continue; - case 'n': write_string(g, g->options->name); continue; - } - } else { - write_char(g, ch); - } - } -} - -static void w(struct generator * g, const char * s) { - writef(g, s, 0); -} - -static void generate_AE(struct generator * g, struct node * p) { - char * s; - switch (p->type) { - case c_name: - write_varref(g, p->name); break; - case c_number: - write_int(g, p->number); break; - case c_maxint: - write_string(g, "MAXINT"); break; - case c_minint: - write_string(g, "MININT"); break; - case c_neg: - write_string(g, "-"); generate_AE(g, p->right); break; - case c_multiply: - s = " * "; goto label0; - case c_plus: - s = " + "; goto label0; - case c_minus: - s = " - "; goto label0; - case c_divide: - s = " / "; - label0: - write_string(g, "("); generate_AE(g, p->left); - write_string(g, s); generate_AE(g, p->right); write_string(g, ")"); break; - case c_sizeof: - g->V[0] = p->name; - w(g, "(~V0.length())"); break; - case c_cursor: - w(g, "cursor"); break; - case c_limit: - w(g, p->mode == m_forward ? "limit" : "limit_backward"); break; - case c_size: - w(g, "(current.length())"); break; - } -} - -/* K_needed() tests to see if we really need to keep c. Not true when the - the command does not touch the cursor. This and repeat_score() could be - elaborated almost indefinitely. -*/ - -static int K_needed(struct generator * g, struct node * p) { - - while (p != 0) { - switch (p->type) { - case c_dollar: - case c_leftslice: - case c_rightslice: - case c_mathassign: - case c_plusassign: - case c_minusassign: - case c_multiplyassign: - case c_divideassign: - case c_eq: - case c_ne: - case c_gr: - case c_ge: - case c_ls: - case c_le: - case c_sliceto: - case c_booltest: - case c_true: - case c_false: - case c_debug: - break; - - case c_call: - if (K_needed(g, p->name->definition)) return true; - break; - - case c_bra: - if (K_needed(g, p->left)) return true; - break; - - default: return true; - } - p = p->right; - } - return false; -} - -static int repeat_score(struct generator * g, struct node * p) { - - int score = 0; - while (p != 0) { - switch (p->type) { - case c_dollar: - case c_leftslice: - case c_rightslice: - case c_mathassign: - case c_plusassign: - case c_minusassign: - case c_multiplyassign: - case c_divideassign: - case c_eq: - case c_ne: - case c_gr: - case c_ge: - case c_ls: - case c_le: - case c_sliceto: /* case c_not: must not be included here! */ - case c_debug: - break; - - case c_call: - score += repeat_score(g, p->name->definition); - break; - - case c_bra: - score += repeat_score(g, p->left); - break; - - case c_name: - case c_literalstring: - case c_next: - case c_grouping: - case c_non: - case c_hop: - score = score + 1; - break; - - default: - score = 2; - break; - } - p = p->right; - } - return score; -} - -/* tests if an expression requires cursor reinstatement in a repeat */ - -static int repeat_restore(struct generator * g, struct node * p) { - - return repeat_score(g, p) >= 2; -} - -static void generate_bra(struct generator * g, struct node * p) { - - write_comment(g, p); - p = p->left; - while (p != 0) { - generate(g, p); - p = p->right; - } -} - -static void generate_and(struct generator * g, struct node * p) { - - struct str * savevar = vars_newname(g); - int keep_c = K_needed(g, p->left); - - write_comment(g, p); - - if (keep_c) write_savecursor(g, p, savevar); - - p = p->left; - while (p != 0) { - generate(g, p); - if (g->unreachable) break; - if (keep_c && p->right != 0) write_restorecursor(g, p, savevar); - p = p->right; - } - str_delete(savevar); -} - -static void generate_or(struct generator * g, struct node * p) { - - struct str * savevar = vars_newname(g); - int keep_c = K_needed(g, p->left); - - int a0 = g->failure_label; - struct str * a1 = str_copy(g->failure_str); - - int out_lab = new_label(g); - write_comment(g, p); - wsetlab_begin(g, out_lab); - - if (keep_c) write_savecursor(g, p, savevar); - - p = p->left; - str_clear(g->failure_str); - - if (p == 0) { - /* p should never be 0 after an or: there should be at least two - * sub nodes. */ - fprintf(stderr, "Error: \"or\" node without children nodes."); - exit (1); - } - while (p->right != 0) { - g->failure_label = new_label(g); - wsetlab_begin(g, g->failure_label); - generate(g, p); - if (!g->unreachable) wgotol(g, out_lab); - wsetlab_end(g); - g->unreachable = false; - if (keep_c) write_restorecursor(g, p, savevar); - p = p->right; - } - - g->failure_label = a0; - str_delete(g->failure_str); - g->failure_str = a1; - - generate(g, p); - wsetlab_end(g); - str_delete(savevar); -} - -static void generate_backwards(struct generator * g, struct node * p) { - - write_comment(g, p); - writef(g,"~Mlimit_backward = cursor; cursor = limit;~N", p); - generate(g, p->left); - w(g, "~Mcursor = limit_backward;"); -} - - -static void generate_not(struct generator * g, struct node * p) { - - struct str * savevar = vars_newname(g); - int keep_c = K_needed(g, p->left); - - int a0 = g->failure_label; - struct str * a1 = str_copy(g->failure_str); - - write_comment(g, p); - if (keep_c) { - write_block_start(g); - write_savecursor(g, p, savevar); - } - - g->failure_label = new_label(g); - str_clear(g->failure_str); - - wsetlab_begin(g, g->failure_label); - - generate(g, p->left); - - g->failure_label = a0; - str_delete(g->failure_str); - g->failure_str = a1; - - if (!g->unreachable) write_failure(g); - - wsetlab_end(g); - g->unreachable = false; - - if (keep_c) write_restorecursor(g, p, savevar); - if (keep_c) write_block_end(g); - str_delete(savevar); -} - - -static void generate_try(struct generator * g, struct node * p) { - - struct str * savevar = vars_newname(g); - int keep_c = K_needed(g, p->left); - - write_comment(g, p); - if (keep_c) write_savecursor(g, p, savevar); - - g->failure_label = new_label(g); - if (keep_c) restore_string(p, g->failure_str, savevar); - - wsetlab_begin(g, g->failure_label); - generate(g, p->left); - wsetlab_end(g); - g->unreachable = false; - - str_delete(savevar); -} - -static void generate_set(struct generator * g, struct node * p) { - - write_comment(g, p); - g->V[0] = p->name; - writef(g, "~M~V0 = true;~N", p); -} - -static void generate_unset(struct generator * g, struct node * p) { - - write_comment(g, p); - g->V[0] = p->name; - writef(g, "~M~V0 = false;~N", p); -} - -static void generate_fail(struct generator * g, struct node * p) { - - write_comment(g, p); - generate(g, p->left); - if (!g->unreachable) write_failure(g); -} - -/* generate_test() also implements 'reverse' */ - -static void generate_test(struct generator * g, struct node * p) { - - struct str * savevar = vars_newname(g); - int keep_c = K_needed(g, p->left); - - write_comment(g, p); - - if (keep_c) { - write_savecursor(g, p, savevar); - } - - generate(g, p->left); - - if (!g->unreachable) { - if (keep_c) { - write_restorecursor(g, p, savevar); - } - } - str_delete(savevar); -} - -static void generate_do(struct generator * g, struct node * p) { - - struct str * savevar = vars_newname(g); - int keep_c = K_needed(g, p->left); - write_comment(g, p); - if (keep_c) write_savecursor(g, p, savevar); - - g->failure_label = new_label(g); - str_clear(g->failure_str); - - wsetlab_begin(g, g->failure_label); - generate(g, p->left); - wsetlab_end(g); - g->unreachable = false; - - if (keep_c) write_restorecursor(g, p, savevar); - str_delete(savevar); -} - -static void generate_GO(struct generator * g, struct node * p, int style) { - - int end_unreachable = false; - struct str * savevar = vars_newname(g); - int keep_c = style == 1 || repeat_restore(g, p->left); - - int a0 = g->failure_label; - struct str * a1 = str_copy(g->failure_str); - - int golab = new_label(g); - g->I[0] = golab; - write_comment(g, p); - w(g, "~Mgolab~I0: while(true)~N"); - w(g, "~{"); - - if (keep_c) write_savecursor(g, p, savevar); - - g->failure_label = new_label(g); - wsetlab_begin(g, g->failure_label); - generate(g, p->left); - - if (g->unreachable) { - /* Cannot break out of this loop: therefore the code after the - * end of the loop is unreachable.*/ - end_unreachable = true; - } else { - /* include for goto; omit for gopast */ - if (style == 1) write_restorecursor(g, p, savevar); - g->I[0] = golab; - w(g, "~Mbreak golab~I0;~N"); - } - g->unreachable = false; - wsetlab_end(g); - if (keep_c) write_restorecursor(g, p, savevar); - - g->failure_label = a0; - str_delete(g->failure_str); - g->failure_str = a1; - - write_check_limit(g, p); - write_inc_cursor(g, p); - write_block_end(g); - str_delete(savevar); - g->unreachable = end_unreachable; -} - -static void generate_loop(struct generator * g, struct node * p) { - - struct str * loopvar = vars_newname(g); - write_comment(g, p); - g->B[0] = str_data(loopvar); - write_declare(g, "int ~B0", p); - w(g, "~Mfor (~B0 = "); - generate_AE(g, p->AE); - g->B[0] = str_data(loopvar); - writef(g, "; ~B0 > 0; ~B0--)~N", p); - writef(g, "~{", p); - - generate(g, p->left); - - w(g, "~}"); - str_delete(loopvar); - g->unreachable = false; -} - -static void generate_repeat(struct generator * g, struct node * p, struct str * loopvar) { - - struct str * savevar = vars_newname(g); - int keep_c = repeat_restore(g, p->left); - int replab = new_label(g); - g->I[0] = replab; - write_comment(g, p); - writef(g, "~Mreplab~I0: while(true)~N~{", p); - - if (keep_c) write_savecursor(g, p, savevar); - - g->failure_label = new_label(g); - str_clear(g->failure_str); - wsetlab_begin(g, g->failure_label); - generate(g, p->left); - - if (!g->unreachable) { - if (loopvar != 0) { - g->B[0] = str_data(loopvar); - w(g, "~M~B0--;~N"); - } - - g->I[0] = replab; - w(g, "~Mcontinue replab~I0;~N"); - } - - wsetlab_end(g); - g->unreachable = false; - - if (keep_c) write_restorecursor(g, p, savevar); - - g->I[0] = replab; - w(g, "~Mbreak replab~I0;~N~}"); - str_delete(savevar); -} - -static void generate_atleast(struct generator * g, struct node * p) { - - struct str * loopvar = vars_newname(g); - write_comment(g, p); - w(g, "~{"); - g->B[0] = str_data(loopvar); - w(g, "~Mint ~B0 = "); - generate_AE(g, p->AE); - w(g, ";~N"); - { - int a0 = g->failure_label; - struct str * a1 = str_copy(g->failure_str); - - generate_repeat(g, p, loopvar); - - g->failure_label = a0; - str_delete(g->failure_str); - g->failure_str = a1; - } - g->B[0] = str_data(loopvar); - write_failure_if(g, "~B0 > 0", p); - w(g, "~}"); - str_delete(loopvar); -} - -static void generate_setmark(struct generator * g, struct node * p) { - - write_comment(g, p); - g->V[0] = p->name; - writef(g, "~M~V0 = cursor;~N", p); -} - -static void generate_tomark(struct generator * g, struct node * p) { - - write_comment(g, p); - g->S[0] = p->mode == m_forward ? ">" : "<"; - - w(g, "~Mif (cursor ~S0 "); generate_AE(g, p->AE); w(g, ")~N"); - write_block_start(g); - write_failure(g); - write_block_end(g); - g->unreachable = false; - w(g, "~Mcursor = "); generate_AE(g, p->AE); writef(g, ";~N", p); -} - -static void generate_atmark(struct generator * g, struct node * p) { - - write_comment(g, p); - w(g, "~Mif (cursor != "); generate_AE(g, p->AE); writef(g, ")~N", p); - write_block_start(g); - write_failure(g); - write_block_end(g); - g->unreachable = false; -} - - -static void generate_hop(struct generator * g, struct node * p) { - - write_comment(g, p); - g->S[0] = p->mode == m_forward ? "+" : "-"; - - w(g, "~{~Mint c = cursor ~S0 "); - generate_AE(g, p->AE); - w(g, ";~N"); - - g->S[0] = p->mode == m_forward ? "0" : "limit_backward"; - - write_failure_if(g, "~S0 > c || c > limit", p); - writef(g, "~Mcursor = c;~N", p); - writef(g, "~}", p); -} - -static void generate_delete(struct generator * g, struct node * p) { - - write_comment(g, p); - writef(g, "~Mslice_del();~N", p); -} - - -static void generate_next(struct generator * g, struct node * p) { - - write_comment(g, p); - write_check_limit(g, p); - write_inc_cursor(g, p); -} - -static void generate_tolimit(struct generator * g, struct node * p) { - - write_comment(g, p); - g->S[0] = p->mode == m_forward ? "limit" : "limit_backward"; - writef(g, "~Mcursor = ~S0;~N", p); -} - -static void generate_atlimit(struct generator * g, struct node * p) { - - write_comment(g, p); - g->S[0] = p->mode == m_forward ? "limit" : "limit_backward"; - g->S[1] = p->mode == m_forward ? "<" : ">"; - write_failure_if(g, "cursor ~S1 ~S0", p); -} - -static void generate_leftslice(struct generator * g, struct node * p) { - - write_comment(g, p); - g->S[0] = p->mode == m_forward ? "bra" : "ket"; - writef(g, "~M~S0 = cursor;~N", p); -} - -static void generate_rightslice(struct generator * g, struct node * p) { - - write_comment(g, p); - g->S[0] = p->mode == m_forward ? "ket" : "bra"; - writef(g, "~M~S0 = cursor;~N", p); -} - -static void generate_assignto(struct generator * g, struct node * p) { - - write_comment(g, p); - g->V[0] = p->name; - writef(g, "~M~V0 = assign_to(~V0);~N", p); -} - -static void generate_sliceto(struct generator * g, struct node * p) { - - write_comment(g, p); - g->V[0] = p->name; - writef(g, "~M~V0 = slice_to(~V0);~N", p); -} - -static void generate_address(struct generator * g, struct node * p) { - - symbol * b = p->literalstring; - if (b != 0) { - write_literal_string(g, b); - } else { - write_varref(g, p->name); - } -} - -static void generate_insert(struct generator * g, struct node * p, int style) { - - int keep_c = style == c_attach; - write_comment(g, p); - if (p->mode == m_backward) keep_c = !keep_c; - if (keep_c) w(g, "~{~Mint c = cursor;~N"); - writef(g, "~Minsert(cursor, cursor, ", p); - generate_address(g, p); - writef(g, ");~N", p); - if (keep_c) w(g, "~Mcursor = c;~N~}"); -} - -static void generate_assignfrom(struct generator * g, struct node * p) { - - int keep_c = p->mode == m_forward; /* like 'attach' */ - - write_comment(g, p); - if (keep_c) writef(g, "~{~Mint c = cursor;~N", p); - if (p->mode == m_forward) { - writef(g, "~Minsert(cursor, limit, ", p); - } else { - writef(g, "~Minsert(limit_backward, cursor, ", p); - } - generate_address(g, p); - writef(g, ");~N", p); - if (keep_c) w(g, "~Mcursor = c;~N~}"); -} - - -static void generate_slicefrom(struct generator * g, struct node * p) { - - write_comment(g, p); - w(g, "~Mslice_from("); - generate_address(g, p); - writef(g, ");~N", p); -} - -static void generate_setlimit(struct generator * g, struct node * p) { - - struct str * savevar = vars_newname(g); - struct str * varname = vars_newname(g); - write_comment(g, p); - write_savecursor(g, p, savevar); - generate(g, p->left); - - if (!g->unreachable) { - g->B[0] = str_data(varname); - write_declare(g, "int ~B0", p); - if (p->mode == m_forward) { - w(g, "~M~B0 = limit - cursor;~N"); - w(g, "~Mlimit = cursor;~N"); - } else { - w(g, "~M~B0 = limit_backward;~N"); - w(g, "~Mlimit_backward = cursor;~N"); - } - write_restorecursor(g, p, savevar); - - if (p->mode == m_forward) { - str_assign(g->failure_str, "limit += "); - str_append(g->failure_str, varname); - str_append_ch(g->failure_str, ';'); - } else { - str_assign(g->failure_str, "limit_backward = "); - str_append(g->failure_str, varname); - str_append_ch(g->failure_str, ';'); - } - generate(g, p->aux); - - if (!g->unreachable) { - write_margin(g); - write_str(g, g->failure_str); - write_newline(g); - } - } - str_delete(varname); - str_delete(savevar); -} - -/* dollar sets snowball up to operate on a string variable as if it were the - * current string */ -static void generate_dollar(struct generator * g, struct node * p) { - - struct str * savevar = vars_newname(g); - write_comment(g, p); - g->V[0] = p->name; - - str_assign(g->failure_str, "copy_from("); - str_append(g->failure_str, savevar); - str_append_string(g->failure_str, ");"); - g->B[0] = str_data(savevar); - writef(g, "~{~M~n ~B0 = this;~N" - "~Mcurrent = new StringBuffer(~V0.toString());~N" - "~Mcursor = 0;~N" - "~Mlimit = (current.length());~N", p); - generate(g, p->left); - if (!g->unreachable) { - write_margin(g); - write_str(g, g->failure_str); - write_newline(g); - } - w(g, "~}"); - str_delete(savevar); -} - -static void generate_integer_assign(struct generator * g, struct node * p, char * s) { - - g->V[0] = p->name; - g->S[0] = s; - w(g, "~M~V0 ~S0 "); generate_AE(g, p->AE); w(g, ";~N"); -} - -static void generate_integer_test(struct generator * g, struct node * p, char * s) { - - g->V[0] = p->name; - g->S[0] = s; - w(g, "~Mif (!(~V0 ~S0 "); generate_AE(g, p->AE); w(g, "))~N"); - write_block_start(g); - write_failure(g); - write_block_end(g); - g->unreachable = false; -} - -static void generate_call(struct generator * g, struct node * p) { - - write_comment(g, p); - g->V[0] = p->name; - write_failure_if(g, "!~V0()", p); -} - -static void generate_grouping(struct generator * g, struct node * p, int complement) { - - struct grouping * q = p->name->grouping; - g->S[0] = p->mode == m_forward ? "" : "_b"; - g->S[1] = complement ? "out" : "in"; - g->V[0] = p->name; - g->I[0] = q->smallest_ch; - g->I[1] = q->largest_ch; - if (q->no_gaps) - write_failure_if(g, "!(~S1_range~S0(~I0, ~I1))", p); - else - write_failure_if(g, "!(~S1_grouping~S0(~V0, ~I0, ~I1))", p); -} - -static void generate_namedstring(struct generator * g, struct node * p) { - - write_comment(g, p); - g->S[0] = p->mode == m_forward ? "" : "_b"; - g->V[0] = p->name; - write_failure_if(g, "!(eq_v~S0(~V0))", p); -} - -static void generate_literalstring(struct generator * g, struct node * p) { - - symbol * b = p->literalstring; - write_comment(g, p); - g->S[0] = p->mode == m_forward ? "" : "_b"; - g->I[0] = SIZE(b); - g->L[0] = b; - write_failure_if(g, "!(eq_s~S0(~I0, ~L0))", p); -} - -static void generate_define(struct generator * g, struct node * p) { - - struct name * q = p->name; - - struct str * saved_output = g->outbuf; - struct str * saved_declarations = g->declarations; - - g->S[0] = q->type == t_routine ? "private" : "public"; - g->V[0] = q; - w(g, "~+~+~N~M~S0 boolean ~V0() {~+~N"); - - g->outbuf = str_new(); - g->declarations = str_new(); - - g->next_label = 0; - g->var_number = 0; - - if (p->amongvar_needed) write_declare(g, "int among_var", p); - str_clear(g->failure_str); - g->failure_label = x_return; - g->unreachable = false; - generate(g, p->left); - if (!g->unreachable) w(g, "~Mreturn true;~N"); - w(g, "~}~-~-"); - - str_append(saved_output, g->declarations); - str_append(saved_output, g->outbuf); - str_delete(g->declarations); - str_delete(g->outbuf); - g->declarations = saved_declarations; - g->outbuf = saved_output; -} - -static void generate_substring(struct generator * g, struct node * p) { - - struct among * x = p->among; - - write_comment(g, p); - - g->S[0] = p->mode == m_forward ? "" : "_b"; - g->I[0] = x->number; - g->I[1] = x->literalstring_count; - - if (x->command_count == 0 && x->starter == 0) { - write_failure_if(g, "find_among~S0(a_~I0, ~I1) == 0", p); - } else { - writef(g, "~Mamong_var = find_among~S0(a_~I0, ~I1);~N", p); - write_failure_if(g, "among_var == 0", p); - } -} - -static void generate_among(struct generator * g, struct node * p) { - - struct among * x = p->among; - int case_number = 1; - - if (x->substring == 0) generate_substring(g, p); - if (x->command_count == 0 && x->starter == 0) return; - - if (x->starter != 0) generate(g, x->starter); - - p = p->left; - if (p != 0 && p->type != c_literalstring) p = p->right; - w(g, "~Mswitch(among_var) {~N~+"); - w(g, "~Mcase 0:~N~+"); - write_failure(g); - g->unreachable = false; - w(g, "~-"); - - while (p != 0) { - if (p->type == c_bra && p->left != 0) { - g->I[0] = case_number++; - w(g, "~Mcase ~I0:~N~+"); - generate(g, p); - if (!g->unreachable) w(g, "~Mbreak;~N"); - w(g, "~-"); - g->unreachable = false; - } - p = p->right; - } - write_block_end(g); -} - -static void generate_booltest(struct generator * g, struct node * p) { - - write_comment(g, p); - g->V[0] = p->name; - write_failure_if(g, "!(~V0)", p); -} - -static void generate_false(struct generator * g, struct node * p) { - - write_comment(g, p); - write_failure(g); -} - -static void generate_debug(struct generator * g, struct node * p) { - - write_comment(g, p); - g->I[0] = g->debug_count++; - g->I[1] = p->line_number; - writef(g, "~Mdebug(~I0, ~I1);~N", p); -} - -static void generate(struct generator * g, struct node * p) { - - int a0; - struct str * a1; - - if (g->unreachable) return; - - a0 = g->failure_label; - a1 = str_copy(g->failure_str); - - switch (p->type) - { - case c_define: generate_define(g, p); break; - case c_bra: generate_bra(g, p); break; - case c_and: generate_and(g, p); break; - case c_or: generate_or(g, p); break; - case c_backwards: generate_backwards(g, p); break; - case c_not: generate_not(g, p); break; - case c_set: generate_set(g, p); break; - case c_unset: generate_unset(g, p); break; - case c_try: generate_try(g, p); break; - case c_fail: generate_fail(g, p); break; - case c_reverse: - case c_test: generate_test(g, p); break; - case c_do: generate_do(g, p); break; - case c_goto: generate_GO(g, p, 1); break; - case c_gopast: generate_GO(g, p, 0); break; - case c_repeat: generate_repeat(g, p, 0); break; - case c_loop: generate_loop(g, p); break; - case c_atleast: generate_atleast(g, p); break; - case c_setmark: generate_setmark(g, p); break; - case c_tomark: generate_tomark(g, p); break; - case c_atmark: generate_atmark(g, p); break; - case c_hop: generate_hop(g, p); break; - case c_delete: generate_delete(g, p); break; - case c_next: generate_next(g, p); break; - case c_tolimit: generate_tolimit(g, p); break; - case c_atlimit: generate_atlimit(g, p); break; - case c_leftslice: generate_leftslice(g, p); break; - case c_rightslice: generate_rightslice(g, p); break; - case c_assignto: generate_assignto(g, p); break; - case c_sliceto: generate_sliceto(g, p); break; - case c_assign: generate_assignfrom(g, p); break; - case c_insert: - case c_attach: generate_insert(g, p, p->type); break; - case c_slicefrom: generate_slicefrom(g, p); break; - case c_setlimit: generate_setlimit(g, p); break; - case c_dollar: generate_dollar(g, p); break; - case c_mathassign: generate_integer_assign(g, p, "="); break; - case c_plusassign: generate_integer_assign(g, p, "+="); break; - case c_minusassign: generate_integer_assign(g, p, "-="); break; - case c_multiplyassign:generate_integer_assign(g, p, "*="); break; - case c_divideassign: generate_integer_assign(g, p, "/="); break; - case c_eq: generate_integer_test(g, p, "=="); break; - case c_ne: generate_integer_test(g, p, "!="); break; - case c_gr: generate_integer_test(g, p, ">"); break; - case c_ge: generate_integer_test(g, p, ">="); break; - case c_ls: generate_integer_test(g, p, "<"); break; - case c_le: generate_integer_test(g, p, "<="); break; - case c_call: generate_call(g, p); break; - case c_grouping: generate_grouping(g, p, false); break; - case c_non: generate_grouping(g, p, true); break; - case c_name: generate_namedstring(g, p); break; - case c_literalstring: generate_literalstring(g, p); break; - case c_among: generate_among(g, p); break; - case c_substring: generate_substring(g, p); break; - case c_booltest: generate_booltest(g, p); break; - case c_false: generate_false(g, p); break; - case c_true: break; - case c_debug: generate_debug(g, p); break; - default: fprintf(stderr, "%d encountered\n", p->type); - exit(1); - } - - g->failure_label = a0; - str_delete(g->failure_str); - g->failure_str = a1; -} - -static void generate_start_comment(struct generator * g) { - - w(g, "// This file was generated automatically by the Snowball to Java compiler~N"); - w(g, "~N"); -} - -static void generate_class_begin(struct generator * g) { - - w(g, "package " ); - w(g, g->options->package); - w(g, ";~N~N" ); - - w(g, "import "); - w(g, g->options->among_class ); - w(g, ";~N" - "~N" - " /**~N" - " * This class was automatically generated by a Snowball to Java compiler ~N" - " * It implements the stemming algorithm defined by a snowball script.~N" - " */~N" - "~N" - "public class ~n extends "); - - w(g, g->options->parent_class_name); - w(g, " {~N" - "~N" - "private static final long serialVersionUID = 1L;~N" - "~N" - "~+~+~Mprivate final static ~n methodObject = new ~n ();~N" - "~N"); -} - -static void generate_class_end(struct generator * g) { - - w(g, "~N}"); - w(g, "~N~N"); -} - -static void generate_equals(struct generator * g) { - - w(g, "~N" - "~Mpublic boolean equals( Object o ) {~N" - "~+~Mreturn o instanceof "); - w(g, g->options->name); - w(g, ";~N~-~M}~N" - "~N" - "~Mpublic int hashCode() {~N" - "~+~Mreturn "); - w(g, g->options->name); - w(g, ".class.getName().hashCode();~N" - "~-~M}~N"); - w(g, "~N~N"); -} - -static void generate_among_table(struct generator * g, struct among * x) { - - struct amongvec * v = x->b; - - g->I[0] = x->number; - g->I[1] = x->literalstring_count; - - w(g, "~+~+~Mprivate final static Among a_~I0[] = {~N~+"); - { - int i; - for (i = 0; i < x->literalstring_count; i++) { - g->I[0] = i; - g->I[1] = v->i; - g->I[2] = v->result; - g->L[0] = v->b; - g->S[0] = i < x->literalstring_count - 1 ? "," : ""; - - w(g, "~Mnew Among ( ~L0, ~I1, ~I2, \""); - if (v->function != 0) { - write_varname(g, v->function); - } - w(g, "\", methodObject )~S0~N"); - v++; - } - } - w(g, "~-~M};~-~-~N~N"); -} - -static void generate_amongs(struct generator * g) { - - struct among * x = g->analyser->amongs; - while (x != 0) { - generate_among_table(g, x); - x = x->next; - } -} - -static void set_bit(symbol * b, int i) { b[i/8] |= 1 << i%8; } - -static int bit_is_set(symbol * b, int i) { return b[i/8] & 1 << i%8; } - -static void generate_grouping_table(struct generator * g, struct grouping * q) { - - int range = q->largest_ch - q->smallest_ch + 1; - int size = (range + 7)/ 8; /* assume 8 bits per symbol */ - symbol * b = q->b; - symbol * map = create_b(size); - int i; - for (i = 0; i < size; i++) map[i] = 0; - - /* Using unicode would require revision here */ - - for (i = 0; i < SIZE(b); i++) set_bit(map, b[i] - q->smallest_ch); - - q->no_gaps = true; - for (i = 0; i < range; i++) unless (bit_is_set(map, i)) q->no_gaps = false; - - unless (q->no_gaps) { - g->V[0] = q->name; - - w(g, "~+~+~Mprivate static final char ~V0[] = {"); - for (i = 0; i < size; i++) { - write_int(g, map[i]); - if (i < size - 1) w(g, ", "); - } - w(g, " };~N~-~-~N"); - } - lose_b(map); -} - -static void generate_groupings(struct generator * g) { - struct grouping * q = g->analyser->groupings; - until (q == 0) { - generate_grouping_table(g, q); - q = q->next; - } -} - -static void generate_members(struct generator * g) { - - struct name * q = g->analyser->names; - until (q == 0) { - g->V[0] = q; - switch (q->type) { - case t_string: - w(g, " private "); - w(g, g->options->string_class ); - w(g, " ~W0 = new "); - w(g, g->options->string_class); - w(g, "();~N"); - break; - case t_integer: - w(g, " private int ~W0;~N"); - break; - case t_boolean: - w(g, " private boolean ~W0;~N"); - break; - } - q = q->next; - } - w(g, "~N"); -} - -static void generate_copyfrom(struct generator * g) { - - struct name * q; - w(g, "~+~+~Mprivate void copy_from(~n other) {~+~N"); - for (q = g->analyser->names; q != 0; q = q->next) { - g->V[0] = q; - switch (q->type) { - case t_string: - case t_integer: - case t_boolean: - w(g, "~M~W0 = other.~W0;~N"); - break; - } - } - w(g, "~Msuper.copy_from(other);~N"); - w(g, "~-~M}~-~-~N"); -} - -static void generate_methods(struct generator * g) { - - struct node * p = g->analyser->program; - while (p != 0) { - generate(g, p); - g->unreachable = false; - p = p->right; - } -} - -extern void generate_program_java(struct generator * g) { - - g->outbuf = str_new(); - g->failure_str = str_new(); - - generate_start_comment(g); - generate_class_begin(g); - - generate_amongs(g); - generate_groupings(g); - - generate_members(g); - generate_copyfrom(g); - generate_methods(g); - generate_equals(g); - - generate_class_end(g); - - output_str(g->options->output_java, g->outbuf); - str_delete(g->failure_str); - str_delete(g->outbuf); -} - -extern struct generator * create_generator_java(struct analyser * a, struct options * o) { - - NEW(generator, g); - g->analyser = a; - g->options = o; - g->margin = 0; - g->debug_count = 0; - g->unreachable = false; - return g; -} - -extern void close_generator_java(struct generator * g) { - - FREE(g); -} - diff --git a/contrib/snowball/compiler/header.h b/contrib/snowball/compiler/header.h index 9baf1d917..dadbee36c 100644 --- a/contrib/snowball/compiler/header.h +++ b/contrib/snowball/compiler/header.h @@ -1,49 +1,56 @@ +#include <stdio.h> + +#define SNOWBALL_VERSION "2.0.0" typedef unsigned char byte; typedef unsigned short symbol; #define true 1 #define false 0 -#define repeat while(true) -#define unless(C) if(!(C)) -#define until(C) while(!(C)) #define MALLOC check_malloc #define FREE check_free #define NEW(type, p) struct type * p = (struct type *) MALLOC(sizeof(struct type)) -#define NEWVEC(type, p, n) struct type * p = (struct type *) MALLOC(sizeof(struct type) * n) +#define NEWVEC(type, p, n) struct type * p = (struct type *) MALLOC(sizeof(struct type) * (n)) -#define STARTSIZE 10 #define SIZE(p) ((int *)(p))[-1] #define CAPACITY(p) ((int *)(p))[-2] extern symbol * create_b(int n); -extern void report_b(FILE * out, symbol * p); +extern void report_b(FILE * out, const symbol * p); extern void lose_b(symbol * p); extern symbol * increase_capacity(symbol * p, int n); -extern symbol * move_to_b(symbol * p, int n, symbol * q); -extern symbol * add_to_b(symbol * p, int n, symbol * q); -extern symbol * copy_b(symbol * p); -extern char * b_to_s(symbol * p); +extern symbol * move_to_b(symbol * p, int n, const symbol * q); +extern symbol * add_to_b(symbol * p, int n, const symbol * q); +extern symbol * copy_b(const symbol * p); +extern char * b_to_s(const symbol * p); extern symbol * add_s_to_b(symbol * p, const char * s); +#define MOVE_TO_B(B, LIT) \ + move_to_b(B, sizeof(LIT) / sizeof(LIT[0]), LIT) + struct str; /* defined in space.c */ extern struct str * str_new(void); extern void str_delete(struct str * str); -extern void str_append(struct str * str, struct str * add); +extern void str_append(struct str * str, const struct str * add); extern void str_append_ch(struct str * str, char add); -extern void str_append_b(struct str * str, symbol * q); +extern void str_append_b(struct str * str, const symbol * q); +extern void str_append_b_tail(struct str * str, const symbol * q, int skip); extern void str_append_string(struct str * str, const char * s); extern void str_append_int(struct str * str, int i); extern void str_clear(struct str * str); -extern void str_assign(struct str * str, char * s); -extern struct str * str_copy(struct str * old); -extern symbol * str_data(struct str * str); -extern int str_len(struct str * str); +extern void str_assign(struct str * str, const char * s); +extern struct str * str_copy(const struct str * old); +extern symbol * str_data(const struct str * str); +extern int str_len(const struct str * str); +extern int str_back(const struct str *str); extern int get_utf8(const symbol * p, int * slot); extern int put_utf8(int ch, symbol * p); +extern void output_str(FILE * outfile, struct str * str); + +typedef enum { ENC_SINGLEBYTE, ENC_UTF8, ENC_WIDECHARS } enc; struct m_pair { @@ -60,6 +67,7 @@ struct input { symbol * p; int c; char * file; + int file_needs_freeing; int line_number; }; @@ -71,6 +79,28 @@ struct include { }; +enum token_codes { + +#include "syswords2.h" + + c_mathassign, + c_name, + c_number, + c_literalstring, + c_neg, + c_call, + c_grouping, + c_booltest, + + NUM_TOKEN_CODES +}; + +enum uplus_modes { + UPLUS_NONE, + UPLUS_DEFINED, + UPLUS_UNICODE +}; + /* struct input must be a prefix of struct tokeniser. */ struct tokeniser { @@ -78,6 +108,7 @@ struct tokeniser { symbol * p; int c; char * file; + int file_needs_freeing; int line_number; symbol * b; symbol * b2; @@ -90,34 +121,28 @@ struct tokeniser { int token; int previous_token; byte token_held; - byte widechars; - byte utf8; + enc encoding; int omission; struct include * includes; + /* Mode in which U+ has been used: + * UPLUS_NONE - not used yet + * UPLUS_DEFINED - stringdef U+xxxx .... + * UPLUS_UNICODE - {U+xxxx} used with implicit meaning + */ + int uplusmode; + + char token_disabled[NUM_TOKEN_CODES]; }; -extern symbol * get_input(symbol * p, char ** p_file); +extern symbol * get_input(const char * filename); extern struct tokeniser * create_tokeniser(symbol * b, char * file); extern int read_token(struct tokeniser * t); extern const char * name_of_token(int code); +extern void disable_token(struct tokeniser * t, int code); extern void close_tokeniser(struct tokeniser * t); -enum token_codes { - -#include "syswords2.h" - - c_mathassign, - c_name, - c_number, - c_literalstring, - c_neg, - c_call, - c_grouping, - c_booltest -}; - extern int space_count; extern void * check_malloc(int n); extern void check_free(void * p); @@ -134,7 +159,13 @@ struct name { int count; /* 0, 1, 2 for each type */ struct grouping * grouping; /* for grouping names */ byte referenced; - byte used; + byte used_in_among; /* Function used in among? */ + byte value_used; /* (For variables) is its value ever used? */ + byte initialised; /* (For variables) is it ever initialised? */ + byte used_in_definition; /* (grouping) used in grouping definition? */ + struct node * used; /* First use, or NULL if not used */ + struct name * local_to; /* Local to one routine/external */ + int declaration_line_number;/* Line number of declaration */ }; @@ -149,9 +180,10 @@ struct amongvec { symbol * b; /* the string giving the case */ int size; /* - and its size */ - struct node * p; /* the corresponding command */ + struct node * action; /* the corresponding action */ int i; /* the amongvec index of the longest substring of b */ int result; /* the numeric result for the case */ + int line_number; /* for diagnostics and stable sorting */ struct name * function; }; @@ -163,19 +195,22 @@ struct among { int number; /* amongs are numbered 0, 1, 2 ... */ int literalstring_count; /* in this among */ int command_count; /* in this among */ + int nocommand_count; /* number of "no command" entries in this among */ + int function_count; /* in this among */ + int amongvar_needed; /* do we need to set among_var? */ struct node * starter; /* i.e. among( (starter) 'string' ... ) */ struct node * substring; /* i.e. substring ... among ( ... ) */ + struct node ** commands; /* array with command_count entries */ }; struct grouping { struct grouping * next; - int number; /* groupings are numbered 0, 1, 2 ... */ symbol * b; /* the characters of this group */ int largest_ch; /* character with max code */ int smallest_ch; /* character with min code */ - byte no_gaps; /* not used in generator.c after 11/5/05 */ struct name * name; /* so g->name->grouping == g */ + int line_number; }; struct node { @@ -234,7 +269,8 @@ struct analyser { struct grouping * groupings; struct grouping * groupings_end; struct node * substring; /* pending 'substring' in current routine definition */ - byte utf8; + enc encoding; + byte int_limits_used; /* are maxint or minint used? */ }; enum analyser_modes { @@ -259,16 +295,23 @@ struct generator { struct str * outbuf; /* temporary str to store output */ struct str * declarations; /* str storing variable declarations */ int next_label; +#ifndef DISABLE_PYTHON + int max_label; +#endif int margin; - const char * failure_string; /* String to output in case of a failure. */ -#ifndef DISABLE_JAVA - struct str * failure_str; /* This is used by the java generator instead of failure_string */ + /* if > 0, keep_count to restore in case of a failure; + * if < 0, the negated keep_count for the limit to restore in case of + * failure. */ + int failure_keep_count; +#if !defined(DISABLE_JAVA) && !defined(DISABLE_JS) && !defined(DISABLE_PYTHON) && !defined(DISABLE_CSHARP) + struct str * failure_str; /* This is used by some generators instead of failure_keep_count */ #endif int label_used; /* Keep track of whether the failure label is used. */ int failure_label; int debug_count; + int copy_from_count; /* count of calls to copy_from() */ const char * S[10]; /* strings */ symbol * B[10]; /* blocks */ @@ -277,48 +320,92 @@ struct generator { symbol * L[5]; /* literals, used in formatted write */ int line_count; /* counts number of lines output */ - int line_labelled; /* in ANSI C, will need extra ';' if it is a block end */ + int line_labelled; /* in ISO C, will need extra ';' if it is a block end */ int literalstring_count; int keep_count; /* used to number keep/restore pairs to avoid compiler warnings about shadowed variables */ }; +/* Special values for failure_label in struct generator. */ +enum special_labels { + x_return = -1 +}; + struct options { /* for the command line: */ - char * output_file; - char * name; - FILE * output_c; + const char * output_file; + const char * name; + FILE * output_src; FILE * output_h; -#ifndef DISABLE_JAVA - FILE * output_java; -#endif byte syntax_tree; - byte widechars; - enum { LANG_JAVA, LANG_C, LANG_CPLUSPLUS } make_lang; - char * externals_prefix; - char * variables_prefix; - char * runtime_path; - char * parent_class_name; - char * package; - char * string_class; - char * among_class; + byte comments; + enc encoding; + enum { LANG_JAVA, LANG_C, LANG_CPLUSPLUS, LANG_CSHARP, LANG_PASCAL, LANG_PYTHON, LANG_JAVASCRIPT, LANG_RUST, LANG_GO } make_lang; + const char * externals_prefix; + const char * variables_prefix; + const char * runtime_path; + const char * parent_class_name; + const char * package; + const char * go_snowball_runtime; + const char * string_class; + const char * among_class; struct include * includes; struct include * includes_end; - byte utf8; }; -/* Generator for C code. */ -extern struct generator * create_generator_c(struct analyser * a, struct options * o); -extern void close_generator_c(struct generator * g); +/* Generator functions common to several backends. */ + +extern struct generator * create_generator(struct analyser * a, struct options * o); +extern void close_generator(struct generator * g); + +extern void write_char(struct generator * g, int ch); +extern void write_newline(struct generator * g); +extern void write_string(struct generator * g, const char * s); +extern void write_int(struct generator * g, int i); +extern void write_b(struct generator * g, symbol * b); +extern void write_str(struct generator * g, struct str * str); + +extern void write_comment_content(struct generator * g, struct node * p); +extern void write_generated_comment_content(struct generator * g); +extern void write_start_comment(struct generator * g, + const char * comment_start, + const char * comment_end); + +extern int K_needed(struct generator * g, struct node * p); +extern int repeat_restore(struct generator * g, struct node * p); +/* Generator for C code. */ extern void generate_program_c(struct generator * g); #ifndef DISABLE_JAVA /* Generator for Java code. */ -extern struct generator * create_generator_java(struct analyser * a, struct options * o); -extern void close_generator_java(struct generator * g); - extern void generate_program_java(struct generator * g); #endif + +#ifndef DISABLE_CSHARP +/* Generator for C# code. */ +extern void generate_program_csharp(struct generator * g); +#endif + +#ifndef DISABLE_PASCAL +extern void generate_program_pascal(struct generator * g); +#endif + +#ifndef DISABLE_PYTHON +/* Generator for Python code. */ +extern void generate_program_python(struct generator * g); +#endif + +#ifndef DISABLE_JS +extern void generate_program_js(struct generator * g); +#endif + +#ifndef DISABLE_RUST +extern void generate_program_rust(struct generator * g); +#endif + +#ifndef DISABLE_GO +extern void generate_program_go(struct generator * g); +#endif diff --git a/contrib/snowball/compiler/space.c b/contrib/snowball/compiler/space.c index 310024e76..5b058763a 100644 --- a/contrib/snowball/compiler/space.c +++ b/contrib/snowball/compiler/space.c @@ -57,9 +57,19 @@ extern symbol * create_b(int n) { return p; } -extern void report_b(FILE * out, symbol * p) { +extern void report_b(FILE * out, const symbol * p) { int i; - for (i = 0; i < SIZE(p); i++) fprintf(out, "%c", p[i]); + for (i = 0; i < SIZE(p); i++) { + if (p[i] > 255) { + printf("In report_b, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]); + exit(1); + } + putc(p[i], out); + } +} + +extern void output_str(FILE * outfile, struct str * str) { + report_b(outfile, str_data(str)); } extern void lose_b(symbol * p) { @@ -74,19 +84,19 @@ extern symbol * increase_capacity(symbol * p, int n) { lose_b(p); return q; } -extern symbol * move_to_b(symbol * p, int n, symbol * q) { +extern symbol * move_to_b(symbol * p, int n, const symbol * q) { int x = n - CAPACITY(p); if (x > 0) p = increase_capacity(p, x); memmove(p, q, n * sizeof(symbol)); SIZE(p) = n; return p; } -extern symbol * add_to_b(symbol * p, int n, symbol * q) { +extern symbol * add_to_b(symbol * p, int n, const symbol * q) { int x = SIZE(p) + n - CAPACITY(p); if (x > 0) p = increase_capacity(p, x); memmove(p + SIZE(p), q, n * sizeof(symbol)); SIZE(p) += n; return p; } -extern symbol * copy_b(symbol * p) { +extern symbol * copy_b(const symbol * p) { int n = SIZE(p); symbol * q = create_b(n); move_to_b(q, n, p); @@ -97,7 +107,7 @@ int space_count = 0; extern void * check_malloc(int n) { space_count++; - return calloc(1, n); + return malloc(n); } extern void check_free(void * p) { @@ -107,18 +117,18 @@ extern void check_free(void * p) { /* To convert a block to a zero terminated string: */ -extern char * b_to_s(symbol * p) { +extern char * b_to_s(const symbol * p) { int n = SIZE(p); - char * s = (char *)calloc(1, n + 1); + char * s = (char *)malloc(n + 1); { int i; for (i = 0; i < n; i++) { - if (p[i] > 255) { - printf("In b_to_s, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]); - exit(1); - } - s[i] = (char)p[i]; - } + if (p[i] > 255) { + printf("In b_to_s, can't convert p[%d] to char because it's 0x%02x\n", i, (int)p[i]); + exit(1); + } + s[i] = (char)p[i]; + } } s[n] = 0; return s; @@ -153,9 +163,9 @@ struct str { }; /* Create a new string. */ -extern struct str * str_new() { +extern struct str * str_new(void) { - struct str * output = (struct str *) calloc(1, sizeof(struct str)); + struct str * output = (struct str *) malloc(sizeof(struct str)); output->data = create_b(0); return output; } @@ -168,7 +178,7 @@ extern void str_delete(struct str * str) { } /* Append a str to this str. */ -extern void str_append(struct str * str, struct str * add) { +extern void str_append(struct str * str, const struct str * add) { symbol * q = add->data; str->data = add_to_b(str->data, SIZE(q), q); @@ -183,12 +193,19 @@ extern void str_append_ch(struct str * str, char add) { } /* Append a low level block to a str. */ -extern void str_append_b(struct str * str, symbol * q) { +extern void str_append_b(struct str * str, const symbol * q) { str->data = add_to_b(str->data, SIZE(q), q); } -/* Append a (char *, null teminated) string to a str. */ +/* Append the tail of a low level block to a str. */ +extern void str_append_b_tail(struct str * str, const symbol * q, int skip) { + if (skip < 0 || skip >= SIZE(q)) return; + + str->data = add_to_b(str->data, SIZE(q) - skip, q + skip); +} + +/* Append a (char *, null terminated) string to a str. */ extern void str_append_string(struct str * str, const char * s) { str->data = add_s_to_b(str->data, s); @@ -209,14 +226,14 @@ extern void str_clear(struct str * str) { } /* Set a string */ -extern void str_assign(struct str * str, char * s) { +extern void str_assign(struct str * str, const char * s) { str_clear(str); str_append_string(str, s); } /* Copy a string. */ -extern struct str * str_copy(struct str * old) { +extern struct str * str_copy(const struct str * old) { struct str * newstr = str_new(); str_append(newstr, old); @@ -224,17 +241,25 @@ extern struct str * str_copy(struct str * old) { } /* Get the data stored in this str. */ -extern symbol * str_data(struct str * str) { +extern symbol * str_data(const struct str * str) { return str->data; } /* Get the length of the str. */ -extern int str_len(struct str * str) { +extern int str_len(const struct str * str) { return SIZE(str->data); } +/* Get the last character of the str. + * + * Or -1 if the string is empty. + */ +extern int str_back(const struct str *str) { + return SIZE(str->data) ? str->data[SIZE(str->data) - 1] : -1; +} + extern int get_utf8(const symbol * p, int * slot) { int b0, b1; b0 = *p++; @@ -260,4 +285,3 @@ extern int put_utf8(int ch, symbol * p) { p[1] = ((ch >> 6) & 0x3F) | 0x80; p[2] = (ch & 0x3F) | 0x80; return 3; } - diff --git a/contrib/snowball/compiler/syswords.h b/contrib/snowball/compiler/syswords.h index 917dd5751..c401d3e4f 100644 --- a/contrib/snowball/compiler/syswords.h +++ b/contrib/snowball/compiler/syswords.h @@ -1,5 +1,5 @@ -static const struct system_word vocab[80+1] = { - { 0, (const byte *)"", 80+1}, +static const struct system_word vocab[82+1] = { + { 0, (const byte *)"", 82+1}, { 1, (const byte *)"$", c_dollar }, { 1, (const byte *)"(", c_bra }, @@ -36,6 +36,7 @@ static const struct system_word vocab[80+1] = { { 3, (const byte *)"get", c_get }, { 3, (const byte *)"hex", c_hex }, { 3, (const byte *)"hop", c_hop }, + { 3, (const byte *)"len", c_len }, { 3, (const byte *)"non", c_non }, { 3, (const byte *)"not", c_not }, { 3, (const byte *)"set", c_set }, @@ -49,6 +50,7 @@ static const struct system_word vocab[80+1] = { { 4, (const byte *)"true", c_true }, { 5, (const byte *)"among", c_among }, { 5, (const byte *)"false", c_false }, + { 5, (const byte *)"lenof", c_lenof }, { 5, (const byte *)"limit", c_limit }, { 5, (const byte *)"unset", c_unset }, { 6, (const byte *)"atmark", c_atmark }, diff --git a/contrib/snowball/compiler/syswords2.h b/contrib/snowball/compiler/syswords2.h index eb8a91241..e853e32dc 100644 --- a/contrib/snowball/compiler/syswords2.h +++ b/contrib/snowball/compiler/syswords2.h @@ -4,8 +4,8 @@ c_decimal, c_define, c_delete, c_divide, c_divideassign, c_do, c_dollar, c_eq, c_externals, c_fail, c_false, c_for, c_ge, c_get, c_gopast, c_goto, c_gr, c_groupings, c_hex, c_hop, c_insert, - c_integers, c_ket, c_le, c_leftslice, c_limit, c_loop, c_ls, - c_maxint, c_minint, c_minus, c_minusassign, c_multiply, + c_integers, c_ket, c_le, c_leftslice, c_len, c_lenof, c_limit, c_loop, + c_ls, c_maxint, c_minint, c_minus, c_minusassign, c_multiply, c_multiplyassign, c_ne, c_next, c_non, c_not, c_or, c_plus, c_plusassign, c_repeat, c_reverse, c_rightslice, c_routines, c_set, c_setlimit, c_setmark, c_size, c_sizeof, c_slicefrom, diff --git a/contrib/snowball/compiler/tokeniser.c b/contrib/snowball/compiler/tokeniser.c index 3dae5f744..e6c63861a 100644 --- a/contrib/snowball/compiler/tokeniser.c +++ b/contrib/snowball/compiler/tokeniser.c @@ -16,57 +16,57 @@ struct system_word { #include "syswords.h" -static int smaller(int a, int b) { return a < b ? a : b; } +#define INITIAL_INPUT_BUFFER_SIZE 8192 + +static int hex_to_num(int ch); -extern symbol * get_input(symbol * p, char ** p_file) { +static int smaller(int a, int b) { return a < b ? a : b; } - char * s = b_to_s(p); +extern symbol * get_input(const char * filename) { + FILE * input = fopen(filename, "r"); + if (input == 0) { return 0; } { - FILE * input = fopen(s, "r"); - if (input == 0) { free(s); return 0; } - *p_file = s; - { - symbol * u = create_b(STARTSIZE); - int size = 0; - repeat - { int ch = getc(input); - if (ch == EOF) break; - if (size >= CAPACITY(u)) u = increase_capacity(u, size/2); - u[size++] = ch; - } - fclose(input); - SIZE(u) = size; return u; + symbol * u = create_b(INITIAL_INPUT_BUFFER_SIZE); + int size = 0; + while (true) { + int ch = getc(input); + if (ch == EOF) break; + if (size >= CAPACITY(u)) u = increase_capacity(u, size); + u[size++] = ch; } + fclose(input); + SIZE(u) = size; + return u; } } -static void error(struct tokeniser * t, char * s1, int n, symbol * p, char * s2) { +static void error(struct tokeniser * t, const char * s1, int n, symbol * p, const char * s2) { if (t->error_count == 20) { fprintf(stderr, "... etc\n"); exit(1); } fprintf(stderr, "%s:%d: ", t->file, t->line_number); - unless (s1 == 0) fprintf(stderr, "%s", s1); - unless (p == 0) { + if (s1) fprintf(stderr, "%s", s1); + if (p) { int i; for (i = 0; i < n; i++) fprintf(stderr, "%c", p[i]); } - unless (s2 == 0) fprintf(stderr, "%s", s2); + if (s2) fprintf(stderr, "%s", s2); fprintf(stderr, "\n"); t->error_count++; } -static void error1(struct tokeniser * t, char * s) { +static void error1(struct tokeniser * t, const char * s) { error(t, s, 0,0, 0); } -static void error2(struct tokeniser * t, char * s) { +static void error2(struct tokeniser * t, const char * s) { error(t, "unexpected end of text after ", 0,0, s); } static int compare_words(int m, symbol * p, int n, const byte * q) { - unless (m == n) return m - n; + if (m != n) return m - n; { int i; for (i = 0; i < n; i++) { int diff = p[i] - q[i]; - unless (diff == 0) return diff; + if (diff) return diff; } } return 0; @@ -74,14 +74,13 @@ static int compare_words(int m, symbol * p, int n, const byte * q) { static int find_word(int n, symbol * p) { int i = 0; int j = vocab->code; - repeat { + do { int k = i + (j - i)/2; const struct system_word * w = vocab + k; int diff = compare_words(n, p, w->s_size, w->s); if (diff == 0) return w->code; if (diff < 0) j = k; else i = k; - if (j - i == 1) break; - } + } while (j - i != 1); return -1; } @@ -91,7 +90,7 @@ static int get_number(int n, symbol * p) { return x; } -static int eq_s(struct tokeniser * t, char * s) { +static int eq_s(struct tokeniser * t, const char * s) { int l = strlen(s); if (SIZE(t->p) - t->c < l) return false; { @@ -103,64 +102,141 @@ static int eq_s(struct tokeniser * t, char * s) { static int white_space(struct tokeniser * t, int ch) { switch (ch) { - case '\n': t->line_number++; + case '\n': + t->line_number++; + /* fall through */ case '\r': case '\t': - case ' ': return true; + case ' ': + return true; } return false; } static symbol * find_in_m(struct tokeniser * t, int n, symbol * p) { - struct m_pair * q = t->m_pairs; - repeat { - if (q == 0) return 0; - { - symbol * name = q->name; - if (n == SIZE(name) && memcmp(name, p, n * sizeof(symbol)) == 0) return q->value; - } - q = q->next; + struct m_pair * q; + for (q = t->m_pairs; q; q = q->next) { + symbol * name = q->name; + if (n == SIZE(name) && memcmp(name, p, n * sizeof(symbol)) == 0) return q->value; } + return 0; } static int read_literal_string(struct tokeniser * t, int c) { symbol * p = t->p; int ch; SIZE(t->b) = 0; - repeat { + while (true) { if (c >= SIZE(p)) { error2(t, "'"); return c; } ch = p[c]; if (ch == '\n') { error1(t, "string not terminated"); return c; } c++; if (ch == t->m_start) { + /* Inside insert characters. */ int c0 = c; int newlines = false; /* no newlines as yet */ int black_found = false; /* no printing chars as yet */ - repeat { + while (true) { if (c >= SIZE(p)) { error2(t, "'"); return c; } ch = p[c]; c++; if (ch == t->m_end) break; - unless (white_space(t, ch)) black_found = true; + if (!white_space(t, ch)) black_found = true; if (ch == '\n') newlines = true; if (newlines && black_found) { error1(t, "string not terminated"); return c; } } - unless (newlines) { + if (!newlines) { int n = c - c0 - 1; /* macro size */ int firstch = p[c0]; symbol * q = find_in_m(t, n, p + c0); if (q == 0) { if (n == 1 && (firstch == '\'' || firstch == t->m_start)) t->b = add_to_b(t->b, 1, p + c0); - else + else if (n >= 3 && firstch == 'U' && p[c0 + 1] == '+') { + int codepoint = 0; + int x; + if (t->uplusmode == UPLUS_DEFINED) { + /* See if found with xxxx upper-cased. */ + symbol * uc = create_b(n); + int i; + for (i = 0; i != n; ++i) { + uc[i] = toupper(p[c0 + i]); + } + q = find_in_m(t, n, uc); + lose_b(uc); + if (q != 0) { + t->b = add_to_b(t->b, SIZE(q), q); + continue; + } + error1(t, "Some U+xxxx stringdefs seen but not this one"); + } else { + t->uplusmode = UPLUS_UNICODE; + } + for (x = c0 + 2; x != c - 1; ++x) { + int hex = hex_to_num(p[x]); + if (hex < 0) { + error1(t, "Bad hex digit following U+"); + break; + } + codepoint = (codepoint << 4) | hex; + } + if (t->encoding == ENC_UTF8) { + if (codepoint < 0 || codepoint > 0x01ffff) { + error1(t, "character values exceed 0x01ffff"); + } + /* Ensure there's enough space for a max length + * UTF-8 sequence. */ + if (CAPACITY(t->b) < SIZE(t->b) + 3) { + t->b = increase_capacity(t->b, 3); + } + SIZE(t->b) += put_utf8(codepoint, t->b + SIZE(t->b)); + } else { + symbol sym; + if (t->encoding == ENC_SINGLEBYTE) { + /* Only ISO-8859-1 is handled this way - for + * other single-byte character sets you need + * stringdef all the U+xxxx codes you use + * like - e.g.: + * + * stringdef U+0171 hex 'FB' + */ + if (codepoint < 0 || codepoint > 0xff) { + error1(t, "character values exceed 256"); + } + } else { + if (codepoint < 0 || codepoint > 0xffff) { + error1(t, "character values exceed 64K"); + } + } + sym = codepoint; + t->b = add_to_b(t->b, 1, &sym); + } + } else error(t, "string macro '", n, p + c0, "' undeclared"); } else t->b = add_to_b(t->b, SIZE(q), q); } } else { if (ch == '\'') return c; + if (ch < 0 || ch >= 0x80) { + if (t->encoding != ENC_WIDECHARS) { + /* We don't really want people using non-ASCII literal + * strings, but historically it's worked for single-byte + * and UTF-8 if the source encoding matches what the + * generated stemmer works in and it seems unfair to just + * suddenly make this a hard error.` + */ + fprintf(stderr, + "%s:%d: warning: Non-ASCII literal strings aren't " + "portable - use stringdef instead\n", + t->file, t->line_number); + } else { + error1(t, "Non-ASCII literal strings aren't " + "portable - use stringdef instead"); + } + } t->b = add_to_b(t->b, 1, p + c - 1); } } @@ -171,7 +247,7 @@ static int next_token(struct tokeniser * t) { int c = t->c; int ch; int code = -1; - repeat { + while (true) { if (c >= SIZE(p)) { t->c = c; return -1; } ch = p[c]; if (white_space(t, ch)) { c++; continue; } @@ -179,7 +255,7 @@ static int next_token(struct tokeniser * t) { int c0 = c; while (c < SIZE(p) && (isalnum(p[c]) || p[c] == '_')) c++; code = find_word(c - c0, p + c0); - if (code < 0) { + if (code < 0 || t->token_disabled[code]) { t->b = move_to_b(t->b, c - c0, p + c0); code = c_name; } @@ -218,10 +294,9 @@ static int next_char(struct tokeniser * t) { } static int next_real_char(struct tokeniser * t) { - repeat { + while (true) { int ch = next_char(t); - if (white_space(t, ch)) continue; - return ch; + if (!white_space(t, ch)) return ch; } } @@ -230,7 +305,7 @@ static void read_chars(struct tokeniser * t) { if (ch < 0) { error2(t, "stringdef"); return; } { int c0 = t->c-1; - repeat { + while (true) { ch = next_char(t); if (white_space(t, ch) || ch < 0) break; } @@ -246,19 +321,20 @@ static int decimal_to_num(int ch) { static int hex_to_num(int ch) { if ('0' <= ch && ch <= '9') return ch - '0'; if ('a' <= ch && ch <= 'f') return ch - 'a' + 10; + if ('A' <= ch && ch <= 'F') return ch - 'A' + 10; return -1; } static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) { int c = 0; int d = 0; - repeat { + while (true) { while (c < SIZE(p) && p[c] == ' ') c++; if (c == SIZE(p)) break; { int number = 0; - repeat { + while (c != SIZE(p)) { int ch = p[c]; - if (c == SIZE(p) || ch == ' ') break; + if (ch == ' ') break; if (base == 10) { ch = decimal_to_num(ch); if (ch < 0) { @@ -266,7 +342,7 @@ static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) { return; } } else { - ch = hex_to_num(tolower(ch)); + ch = hex_to_num(ch); if (ch < 0) { error1(t, "hex string contains non-hex characters"); return; @@ -275,18 +351,18 @@ static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) { number = base * number + ch; c++; } - if (t->widechars || t->utf8) { - unless (0 <= number && number <= 0xffff) { - error1(t, "character values exceed 64K"); + if (t->encoding == ENC_SINGLEBYTE) { + if (number < 0 || number > 0xff) { + error1(t, "character values exceed 256"); return; } } else { - unless (0 <= number && number <= 0xff) { - error1(t, "character values exceed 256"); + if (number < 0 || number > 0xffff) { + error1(t, "character values exceed 64K"); return; } } - if (t->utf8) + if (t->encoding == ENC_UTF8) d += put_utf8(number, p + d); else p[d++] = number; @@ -300,104 +376,118 @@ extern int read_token(struct tokeniser * t) { int held = t->token_held; t->token_held = false; if (held) return t->token; - repeat { + while (true) { int code = next_token(t); switch (code) { case c_comment1: /* slash-slash comment */ - while (t->c < SIZE(p) && p[t->c] != '\n') t->c++; - continue; + while (t->c < SIZE(p) && p[t->c] != '\n') t->c++; + continue; case c_comment2: /* slash-star comment */ - repeat { - if (t->c >= SIZE(p)) { - error1(t, "/* comment not terminated"); - t->token = -1; - return -1; - } - if (p[t->c] == '\n') t->line_number++; - if (eq_s(t, "*/")) break; - t->c++; - } - continue; - case c_stringescapes: - { - int ch1 = next_real_char(t); - int ch2 = next_real_char(t); - if (ch2 < 0) - { error2(t, "stringescapes"); continue; } - if (ch1 == '\'') - { error1(t, "first stringescape cannot be '"); continue; } - t->m_start = ch1; - t->m_end = ch2; - } - continue; - case c_stringdef: - { - int base = 0; - read_chars(t); - code = read_token(t); - if (code == c_hex) { base = 16; code = read_token(t); } else - if (code == c_decimal) { base = 10; code = read_token(t); } - unless (code == c_literalstring) - { error1(t, "string omitted after stringdef"); continue; } - if (base > 0) convert_numeric_string(t, t->b, base); - { NEW(m_pair, q); - q->next = t->m_pairs; - q->name = copy_b(t->b2); - q->value = copy_b(t->b); - t->m_pairs = q; - } - } - continue; + while (true) { + if (t->c >= SIZE(p)) { + error1(t, "/* comment not terminated"); + t->token = -1; + return -1; + } + if (p[t->c] == '\n') t->line_number++; + if (eq_s(t, "*/")) break; + t->c++; + } + continue; + case c_stringescapes: { + int ch1 = next_real_char(t); + int ch2 = next_real_char(t); + if (ch2 < 0) { + error2(t, "stringescapes"); + continue; + } + if (ch1 == '\'') { + error1(t, "first stringescape cannot be '"); + continue; + } + t->m_start = ch1; + t->m_end = ch2; + continue; + } + case c_stringdef: { + int base = 0; + read_chars(t); + code = read_token(t); + if (code == c_hex) { base = 16; code = read_token(t); } else + if (code == c_decimal) { base = 10; code = read_token(t); } + if (code != c_literalstring) { + error1(t, "string omitted after stringdef"); + continue; + } + if (base > 0) convert_numeric_string(t, t->b, base); + { NEW(m_pair, q); + q->next = t->m_pairs; + q->name = copy_b(t->b2); + q->value = copy_b(t->b); + t->m_pairs = q; + if (t->uplusmode != UPLUS_DEFINED && + (SIZE(t->b2) >= 3 && t->b2[0] == 'U' && t->b2[1] == '+')) { + if (t->uplusmode == UPLUS_UNICODE) { + error1(t, "U+xxxx already used with implicit meaning"); + } else { + t->uplusmode = UPLUS_DEFINED; + } + } + } + continue; + } case c_get: - code = read_token(t); - unless (code == c_literalstring) { - error1(t, "string omitted after get"); continue; - } - t->get_depth++; - if (t->get_depth > 10) { - fprintf(stderr, "get directives go 10 deep. Looping?\n"); - exit(1); - } - { - char * file; - NEW(input, q); - symbol * u = get_input(t->b, &file); - if (u == 0) { - struct include * r = t->includes; - until (r == 0) { - symbol * b = copy_b(r->b); - b = add_to_b(b, SIZE(t->b), t->b); - u = get_input(b, &file); - lose_b(b); - unless (u == 0) break; - r = r->next; - } - } - if (u == 0) { - error(t, "Can't get '", SIZE(t->b), t->b, "'"); - exit(1); - } - memmove(q, t, sizeof(struct input)); - t->next = q; - t->p = u; - t->c = 0; - t->file = file; - t->line_number = 1; - } - p = t->p; - continue; + code = read_token(t); + if (code != c_literalstring) { + error1(t, "string omitted after get"); continue; + } + t->get_depth++; + if (t->get_depth > 10) { + fprintf(stderr, "get directives go 10 deep. Looping?\n"); + exit(1); + } + { + NEW(input, q); + char * file = b_to_s(t->b); + symbol * u = get_input(file); + if (u == 0) { + struct include * r; + for (r = t->includes; r; r = r->next) { + symbol * b = copy_b(r->b); + b = add_to_b(b, SIZE(t->b), t->b); + free(file); + file = b_to_s(b); + u = get_input(file); + lose_b(b); + if (u != 0) break; + } + } + if (u == 0) { + error(t, "Can't get '", SIZE(t->b), t->b, "'"); + exit(1); + } + memmove(q, t, sizeof(struct input)); + t->next = q; + t->p = u; + t->c = 0; + t->file = file; + t->file_needs_freeing = true; + t->line_number = 1; + } + p = t->p; + continue; case -1: - unless (t->next == 0) { - lose_b(p); - { - struct input * q = t->next; - memmove(t, q, sizeof(struct input)); p = t->p; - FREE(q); - } - t->get_depth--; - continue; - } - /* drop through */ + if (t->next) { + lose_b(p); + { + struct input * q = t->next; + memmove(t, q, sizeof(struct input)); p = t->p; + FREE(q); + } + t->get_depth--; + continue; + } + /* fall through */ default: t->previous_token = t->token; t->token = code; @@ -425,12 +515,17 @@ extern const char * name_of_token(int code) { } } +extern void disable_token(struct tokeniser * t, int code) { + t->token_disabled[code] = 1; +} + extern struct tokeniser * create_tokeniser(symbol * p, char * file) { NEW(tokeniser, t); t->next = 0; t->p = p; t->c = 0; t->file = file; + t->file_needs_freeing = false; t->line_number = 1; t->b = create_b(0); t->b2 = create_b(0); @@ -441,6 +536,8 @@ extern struct tokeniser * create_tokeniser(symbol * p, char * file) { t->token_held = false; t->token = -2; t->previous_token = -2; + t->uplusmode = UPLUS_NONE; + memset(t->token_disabled, 0, sizeof(t->token_disabled)); return t; } @@ -449,7 +546,7 @@ extern void close_tokeniser(struct tokeniser * t) { lose_b(t->b2); { struct m_pair * q = t->m_pairs; - until (q == 0) { + while (q) { struct m_pair * q_next = q->next; lose_b(q->name); lose_b(q->value); @@ -459,12 +556,12 @@ extern void close_tokeniser(struct tokeniser * t) { } { struct input * q = t->next; - until (q == 0) { + while (q) { struct input * q_next = q->next; FREE(q); q = q_next; } } - free(t->file); + if (t->file_needs_freeing) free(t->file); FREE(t); } |