* Fix error in expression parser that causes bad errors with expressions that have regexp at the end

* Improve test for fuzzy hashes
* Add new object - TextPart to perl XS library that allows access to stripped parts and fuzzy hashes
* Add documentation for expressions parser and fot Mail::Rspamd::TextPart
* Allways calculate fuzzy hash for text parts
* Store text parts separately from other parts
* Add compare_parts_distance for expressions that calculates difference in 2 parts messages
* Do not try to substitute variables in empty strings

1 files changed, 7 insertions, 5 deletions

diff --git a/src/filter.c b/src/filter.c
index 766cd16e4..1b6cdc1b0 100644
--- a/src/filter.c
+++ b/src/filter.c
@@ -437,10 +437,10 @@ statfiles_callback (gpointer key, gpointer value, void *arg)
 	struct classifier *classifier;
 	struct statfile_result_data *res_data;
 	struct metric *metric;
+	struct mime_text_part *text_part;
 
 	GTree *tokens = NULL;
-	GList *cur = NULL;
-	GByteArray *content;
+	GList *cur;
 
 	char *filename;
 	f_str_t c;
@@ -457,10 +457,12 @@ statfiles_callback (gpointer key, gpointer value, void *arg)
 		return;
 	}
 	
+	cur = g_list_first (task->text_parts);
 	if ((tokens = g_hash_table_lookup (data->tokens, st->tokenizer)) == NULL) {
-		while ((content = get_next_text_part (task->task_pool, task->parts, &cur)) != NULL) {
-			c.begin = content->data;
-			c.len = content->len;
+		while (cur != NULL) {
+			text_part = (struct mime_text_part *)cur->data;
+			c.begin = text_part->content->data;
+			c.len = text_part->content->len;
 			/* Tree would be freed at task pool freeing */
 			if (!st->tokenizer->tokenize_func (st->tokenizer, task->task_pool, &c, &tokens)) {
 				msg_info ("statfiles_callback: cannot tokenize input");