Fix statshow utility.

diff --git a/src/classifiers/bayes.c b/src/classifiers/bayes.c
index dadd33e5ee08e424bc988dcb1653628ccd266f04..af79e0eaa11c2dc94f858596b5dead3d87ec4818 100644 (file)
--- a/src/classifiers/bayes.c
+++ b/src/classifiers/bayes.c
@@ -47,7 +47,7 @@ struct bayes_statfile_data {
        guint64                         total_hits;
        double                          local_probability;
        double                          post_probability;
-       guint                           value;
+       guint64                         value;
        struct statfile                *st;
        stat_file_t                    *file;
 };
@@ -145,7 +145,7 @@ bayes_classify_callback (gpointer key, gpointer value, gpointer data)
                        cur->post_probability = G_MINDOUBLE * 100;
                }
                if (cd->ctx->debug) {
-                       msg_info ("token: %s, statfile: %s, probability: %.4f, post_probability: %.4f",
+                       msg_info ("token: %s, statfile: %s, probability: %uL, post_probability: %.4f",
                                        node->extra, cur->st->symbol, cur->value, cur->post_probability);
                }
        }

diff --git a/utils/statshow/statshow.c b/utils/statshow/statshow.c
index a2e8dce6874d84cd880e975b1ee01f0d67a10298..9a574b80a298104e7e09003b349031031eba11ef 100644 (file)
--- a/utils/statshow/statshow.c
+++ b/utils/statshow/statshow.c
@@ -94,7 +94,7 @@ classifiers_callback (gpointer value, void *arg)
                                c.len = strlen (cur->data);
                                if (c.len > 0) {
                                        c.begin = cur->data;
-                                       if (!cl->tokenizer->tokenize_func (cl->tokenizer, task->task_pool, &c, &tokens, TRUE)) {
+                                       if (!cl->tokenizer->tokenize_func (cl->tokenizer, task->task_pool, &c, &tokens, TRUE, FALSE, NULL)) {
                                                msg_info ("cannot tokenize input");
                                                return;
                                        }
@@ -109,7 +109,8 @@ classifiers_callback (gpointer value, void *arg)
                                c.begin = text_part->content->data;
                                c.len = text_part->content->len;
                                /* Tree would be freed at task pool freeing */
-                               if (!cl->tokenizer->tokenize_func (cl->tokenizer, task->task_pool, &c, &tokens, TRUE)) {
+                               if (!cl->tokenizer->tokenize_func (cl->tokenizer, task->task_pool, &c, &tokens, TRUE,
+                                               text_part->is_utf, text_part->urls_offset)) {
                                        msg_info ("cannot tokenize input");
                                        return;
                                }