diff options
| author | Vsevolod Stakhov <vsevolod@rambler-co.ru> | 2011-05-06 19:18:40 +0400 |
|---|---|---|
| committer | Vsevolod Stakhov <vsevolod@rambler-co.ru> | 2011-05-06 19:18:40 +0400 |
| commit | 683b90f4c6c744557f7429ce6ff77c0f7d2175e1 (patch) | |
| tree | 6e5f5cfdb0070cc7387d4045e955c6226d9f225d /src/tokenizers/osb.c | |
| parent | 56586078f92c4cf71fad46e1f4888a49749a6313 (diff) | |
| download | rspamd-683b90f4c6c744557f7429ce6ff77c0f7d2175e1.tar.gz rspamd-683b90f4c6c744557f7429ce6ff77c0f7d2175e1.zip | |
* Major cleanup of cmake build system
* Add initial version of statshow utility for statfiles debugging
* Add debugging for statistics
* Remove unused utilities
Diffstat (limited to 'src/tokenizers/osb.c')
| -rw-r--r-- | src/tokenizers/osb.c | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/src/tokenizers/osb.c b/src/tokenizers/osb.c index ae59cf8ea..41bcce737 100644 --- a/src/tokenizers/osb.c +++ b/src/tokenizers/osb.c @@ -35,7 +35,7 @@ extern const int primes[]; int -osb_tokenize_text (struct tokenizer *tokenizer, memory_pool_t * pool, f_str_t * input, GTree ** tree) +osb_tokenize_text (struct tokenizer *tokenizer, memory_pool_t * pool, f_str_t * input, GTree ** tree, gboolean save_token) { token_node_t *new = NULL; f_str_t token = { NULL, 0, 0 }, *res; @@ -69,6 +69,9 @@ osb_tokenize_text (struct tokenizer *tokenizer, memory_pool_t * pool, f_str_t * new = memory_pool_alloc0 (pool, sizeof (token_node_t)); new->h1 = h1; new->h2 = h2; + if (save_token) { + new->extra = (uintptr_t)memory_pool_fstrdup (pool, &token); + } if (g_tree_lookup (*tree, new) == NULL) { g_tree_insert (*tree, new, new); |