* Fix error in expression parser that causes bad errors with expressions that have regexp at the end

* Improve test for fuzzy hashes
* Add new object - TextPart to perl XS library that allows access to stripped parts and fuzzy hashes
* Add documentation for expressions parser and fot Mail::Rspamd::TextPart
* Allways calculate fuzzy hash for text parts
* Store text parts separately from other parts
* Add compare_parts_distance for expressions that calculates difference in 2 parts messages
* Do not try to substitute variables in empty strings

6 files changed, 106 insertions, 2 deletions

diff --git a/perl/Makefile.PL.in b/perl/Makefile.PL.in
index 0d70d4270..10ca63e16 100644
--- a/perl/Makefile.PL.in
+++ b/perl/Makefile.PL.in
@@ -1,7 +1,7 @@
 use ExtUtils::MakeMaker;
 WriteMakefile(
     NAME         => 'Mail::Rspamd',
-    AUTHOR       => 'Vsevolod Stakhov <vsevolod@rambler-co.ru>',
+    AUTHOR       => 'Vsevolod Stakhov <vsevolod@highsecure.ru>',
     XS           => { 'Rspamd.xs'  => 'Rspamd.c' },
     VERSION_FROM => 'Rspamd.pm', # finds $VERSION
     LIBS         => ['${GLIB_LDFLAGS} ${GMIME_LDFLAGS} -levent'],   # e.g., '-lm'
@@ -15,6 +15,7 @@ WriteMakefile(
 	    'Rspamd.c'   => qw{
 Rspamd/ContentType.xs            Rspamd/Part.xs                   Rspamd/Hash.xs
 Rspamd/InternetAddress.xs        Rspamd/Message.xs                Rspamd/Object.xs
+Rspamd/TextPart.xs
 },
 	},
 );
diff --git a/perl/Rspamd.pod b/perl/Rspamd.pod
index d40574820..2af9b4965 100644
--- a/perl/Rspamd.pod
+++ b/perl/Rspamd.pod
@@ -197,7 +197,7 @@ E.g.:
 
 Mail::Rspamd::Header is a private structure. This structure contains
 all the headers except special ones (Content-* MIME-Version).
-Look for L<Header tied hash> for easy maintaining for header.
+Look for Header tied hash for easy maintaining for header.
 Use also the Mail::Rspamd::Message::get_header() and set_header() methods.
 
 =back
@@ -476,6 +476,10 @@ Return Mail::Rspamd::Config object.
 
 Return message's urls as array of strings.
 
+=item I<get_text_parts> ()
+
+Return message's text parts as array of Mail::Rspamd::TextPart objects.
+
 =back
 
 =head2 Mail::Rspamd::Config
@@ -493,15 +497,23 @@ Gets and sets specified parameter in config.
 =item I<get_metric> (metric)
 
 Returns hash of parameters of specified metric:
+
+=begin text
+
 {
 'name'             => name of metric
 'func_name'        => consolidation function
 'required_score'   => score for metric
 }
 
+=end text
+
 =item I<get_statfile> (statfile)
 
 Returns parameters of specified statfile:
+
+=begin text
+
 {
 'alias'         => alias of statfile
 'pattern'       => fs pattern
@@ -510,12 +522,38 @@ Returns parameters of specified statfile:
 'size'          => size of statfile
 }
 
+=end text
+
 =item I<get_module_param> (modulename, paramname)
 
 Return parameter's value for specified module.
 
 =back
 
+=head2 Mail::Rspamd::TextPart
+
+Object that represent a single text part of message.
+
+=over 4
+
+=item I<get_content> ()
+
+Returns content of part.
+
+=item I<get_fuzzy> ()
+
+Returns fuzzy hash of part as string.
+
+=item I<compare_distance> (other)
+
+Calculate distance between two parts using their fuzzy hashes. Return value from 0 (identical) to 100 (totally different).
+
+=item I<is_html> ()
+
+Return 0 if part is plain text and not 0 otherwise.
+
+=back
+
 =head1 CONSTANT VARIABLES
 
     GMIME_LENGTH_ENCODED
diff --git a/perl/Rspamd.xs b/perl/Rspamd.xs
index 9231d3e62..892e4b006 100644
--- a/perl/Rspamd.xs
+++ b/perl/Rspamd.xs
@@ -14,6 +14,7 @@
 #include "../src/cfg_file.h"
 #include "../src/perl.h"
 #include "../src/mem_pool.h"
+#include "../src/fuzzy.h"
 
 #define XSINTERFACE_FUNC_RSPAMD_MESSAGE_SET(cv,f)      \
 	CvXSUBANY(cv).any_dptr = (void (*) (pTHX_ void*))(CAT2( g_mime_message_,f ))
@@ -47,6 +48,7 @@ typedef GMimePartEncodingType	Mail__Rspamd__PartEncodingType;
 typedef GMimeObject *		Mail__Rspamd__Object;
 typedef GMimeParam *		Mail__Rspamd__Param;
 typedef GMimePart *		Mail__Rspamd__Part;
+typedef struct mime_text_part * Mail__Rspamd__TextPart;
 typedef GMimeParser *		Mail__Rspamd__Parser;
 typedef GMimeMultipart *	Mail__Rspamd__MultiPart;
 typedef GMimeMessage *		Mail__Rspamd__Message;
@@ -401,5 +403,6 @@ INCLUDE: Rspamd/Message.xs
 
 INCLUDE: Rspamd/InternetAddress.xs
 INCLUDE: Rspamd/Hash.xs
+INCLUDE: Rspamd/TextPart.xs
 
 
diff --git a/perl/Rspamd/Task.xs b/perl/Rspamd/Task.xs
index 16719ef7b..31928bf7b 100644
--- a/perl/Rspamd/Task.xs
+++ b/perl/Rspamd/Task.xs
@@ -81,3 +81,24 @@ rspamd_task_get_urls (task)
 	OUTPUT:
 		RETVAL
 
+AV*
+rspamd_task_get_text_parts (task)
+		Mail::Rspamd::Task task
+	PREINIT:
+		AV* retav;
+		GList *cur;
+		SV* ps;
+	CODE:
+		retav = newAV ();
+		cur = g_list_first (task->text_parts);
+		while (cur) {
+			ps = newSViv (0);
+			sv_setref_pv (ps, "Mail::Rspamd::TextPart", (Mail__Rspamd__TextPart)(cur->data));
+			av_push(retav, ps);
+			cur = g_list_next (task->text_parts);
+		}
+
+		RETVAL = retav;
+	OUTPUT:
+		RETVAL
+
diff --git a/perl/Rspamd/TextPart.xs b/perl/Rspamd/TextPart.xs
new file mode 100644
index 000000000..485ee7bcc
--- /dev/null
+++ b/perl/Rspamd/TextPart.xs
@@ -0,0 +1,40 @@
+MODULE = Mail::Rspamd PACKAGE = Mail::Rspamd::TextPart PREFIX = rspamd_text_part_
+
+SV *
+rspamd_text_part_get_content (mime_part)
+		Mail::Rspamd::TextPart	mime_part
+	PREINIT:
+		SV* content;
+	CODE:
+		ST(0) = &PL_sv_undef;
+		content = sv_newmortal ();
+		SvUPGRADE (content, SVt_PV);
+		SvREADONLY_on (content);
+		SvPVX(content) = (char *) (mime_part->content->data);
+		SvCUR_set (content, mime_part->content->len);
+		SvLEN_set (content, 0);
+		SvPOK_only (content);
+		ST(0) = content;
+
+char *
+rspamd_text_part_get_fuzzy (mime_part)
+		Mail::Rspamd::TextPart	mime_part
+	CODE:
+		RETVAL = mime_part->fuzzy->hash_pipe;
+
+int
+rspamd_text_part_compare_distance (mime_part, other)
+		Mail::Rspamd::TextPart	mime_part
+		Mail::Rspamd::TextPart	other
+	CODE:
+		RETVAL = fuzzy_compare_hashes (mime_part->fuzzy, other->fuzzy);
+	OUTPUT:
+		RETVAL
+
+int
+rspamd_text_part_is_html (mime_part)
+		Mail::Rspamd::TextPart	mime_part
+	CODE:
+		RETVAL = mime_part->is_html;
+	OUTPUT:
+		RETVAL
diff --git a/perl/typemap b/perl/typemap
index fa2dabb43..0ae0ef7de 100644
--- a/perl/typemap
+++ b/perl/typemap
@@ -20,6 +20,7 @@ Mail::Rspamd::Object			T_PTROBJ
 Mail::Rspamd::Param			T_PTROBJ
 Mail::Rspamd::Message			T_PTROBJ
 Mail::Rspamd::Part			T_PTROBJ
+Mail::Rspamd::TextPart		T_PTROBJ
 Mail::Rspamd::ContentType			T_PTROBJ
 Mail::Rspamd::InternetAddress		T_PTROBJ
 Mail::Rspamd::Hash::Header		T_PTROBJ