From: Vsevolod Stakhov <vsevolod@rambler-co.ru>
Date: Fri, 18 Feb 2011 14:33:21 +0000 (+0300)
Subject: * Add descriptions for all symbols in default configuration, adjust symbols' weights [1]
X-Git-Tag: 0.3.7~25
X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=f0eafb45ef94088c91ec552d8dbd7c30e37c1bb1;p=rspamd.git

* Add descriptions for all symbols in default configuration, adjust symbols' weights [1]
* Improve rspamc output by adding '--verbose' flag

Suggested by: Victor Ustugov [1]
---

diff --git a/rspamd.xml.sample b/rspamd.xml.sample
index 875cb6b93..44d6a6566 100644
--- a/rspamd.xml.sample
+++ b/rspamd.xml.sample
@@ -26,7 +26,7 @@
 <!-- End of logging section -->
 
 
-<!-- Factors section -->
+<!-- Metrics section -->
 <metric>
  <name>default</name>
  <required_score>10.0</required_score>
@@ -38,153 +38,158 @@
  <!-- Weights for symbols -->
 
  <!-- Subject is missing inside message -->
- <symbol weight="2.00">MISSING_SUBJECT</symbol>
+ <symbol weight="2.00" description="Subject is missing inside message">MISSING_SUBJECT</symbol>
  <!-- Message pretends to be send from Outlook but has 'strange' tags -->
- <symbol weight="2.10">FORGED_OUTLOOK_TAGS</symbol>
+ <symbol weight="2.10" description="Message pretends to be send from Outlook but has 'strange' tags ">FORGED_OUTLOOK_TAGS</symbol>
  <!-- Sender is forged (different From: header and smtp MAIL FROM: addresses) -->
- <symbol weight="5.00">FORGED_SENDER</symbol>
+ <symbol weight="5.00" description="Sender is forged (different From: header and smtp MAIL FROM: addresses)">FORGED_SENDER</symbol>
  <!-- Recipients seems to be autogenerated (works if recipients count is more than 5) -->
- <symbol weight="3.50">SUSPICIOUS_RECIPS</symbol>
- <!--  Fake reply (has RE in subject, but has not References header) --> 
- <symbol weight="6.00">FAKE_REPLY_C</symbol>
+ <symbol weight="3.50" description="Recipients seems to be autogenerated (works if recipients count is more than 5)">SUSPICIOUS_RECIPS</symbol>
+ <!-- Fake reply (has RE in subject, but has not References header) --> 
+ <symbol weight="6.00" description="Fake reply (has RE in subject, but has not References header)">FAKE_REPLY_C</symbol>
  <!-- Messages that have only HTML part -->
- <symbol weight="1.00">MIME_HTML_ONLY</symbol>
+ <symbol weight="1.00" description="Messages that have only HTML part">MIME_HTML_ONLY</symbol>
  <!-- Forged yahoo msgid -->
- <symbol weight="2.00">FORGED_MSGID_YAHOO</symbol>
+ <symbol weight="2.00" description="Forged yahoo msgid">FORGED_MSGID_YAHOO</symbol>
  <!-- Forged The Bat! MUA headers -->
- <symbol weight="2.00">FORGED_MUA_THEBAT_BOUN</symbol>
+ <symbol weight="2.00" description="Forged The Bat! MUA headers">FORGED_MUA_THEBAT_BOUN</symbol>
  <!-- Charset is missing in a message -->
- <symbol weight="5.00">R_MISSING_CHARSET</symbol>
+ <symbol weight="5.00" description="Charset is missing in a message">R_MISSING_CHARSET</symbol>
  <!-- Two received headers with ip addresses -->
- <symbol weight="2.00">RCVD_DOUBLE_IP_SPAM</symbol>
+ <symbol weight="2.00" description="Two received headers with ip addresses">RCVD_DOUBLE_IP_SPAM</symbol>
  <!-- Forged outlook HTML signature -->
- <symbol weight="5.00">FORGED_OUTLOOK_HTML</symbol>
+ <symbol weight="5.00" description="Forged outlook HTML signature">FORGED_OUTLOOK_HTML</symbol>
  <!-- Recipients are absent or undisclosed -->
- <symbol weight="5.00">R_UNDISC_RCPT</symbol>
+ <symbol weight="5.00" description="Recipients are absent or undisclosed">R_UNDISC_RCPT</symbol>
  <!-- White color on white background in HTML messages -->
- <symbol weight="9.00">R_WHITE_ON_WHITE</symbol>
+ <symbol weight="9.00" description="White color on white background in HTML messages">R_WHITE_ON_WHITE</symbol>
  <!-- Short html part with a link to an image -->
- <symbol weight="3.00">HTML_SHORT_LINK_IMG_2</symbol>
+ <symbol weight="3.00" description="Short html part with a link to an image">HTML_SHORT_LINK_IMG_2</symbol>
  <!-- Forged outlook MUA -->
- <symbol weight="3.00">FORGED_MUA_OUTLOOK</symbol>
+ <symbol weight="3.00" description="Forged outlook MUA ">FORGED_MUA_OUTLOOK</symbol>
  <!-- Fake helo for verizon provider -->
- <symbol weight="2.00">FM_FAKE_HELO_VERIZON</symbol>
+ <symbol weight="2.00" description="Fake helo for verizon provider">FM_FAKE_HELO_VERIZON</symbol>
  <!--Quoted reply-to from yahoo (seems to be forged) --> 
- <symbol weight="2.00">REPTO_QUOTE_YAHOO</symbol>
+ <symbol weight="2.00" description="Quoted reply-to from yahoo (seems to be forged)">REPTO_QUOTE_YAHOO</symbol>
  <!-- Mime-OLE is needed but absent (e.g. fake Outlook or fake Exchange) -->
- <symbol weight="5.00">MISSING_MIMEOLE</symbol>
+ <symbol weight="5.00" description="Mime-OLE is needed but absent (e.g. fake Outlook or fake Exchange)">MISSING_MIMEOLE</symbol>
  <!-- To header is missing -->
- <symbol weight="2.00">MISSING_TO</symbol>
+ <symbol weight="2.00" description="To header is missing">MISSING_TO</symbol>
  <!-- From that contains encoded characters while base 64 is not needed as all symbols are 7bit -->
- <symbol weight="0.33">FROM_EXCESS_BASE64</symbol>
+ <symbol weight="2.0" description="From that contains encoded characters while base 64 is not needed as all symbols are 7bit">FROM_EXCESS_BASE64</symbol>
  <!-- Mixed characters in a message -->
- <symbol weight="5.00">R_MIXED_CHARSET</symbol>
+ <symbol weight="5.00" description="Mixed characters in a message">R_MIXED_CHARSET</symbol>
  <!-- Recipients list seems to be sorted -->
- <symbol weight="3.50">SORTED_RECIPS</symbol>
+ <symbol weight="3.50" description="Recipients list seems to be sorted">SORTED_RECIPS</symbol>
  <!-- Spambots signatures in received headers -->
- <symbol weight="3.00">R_RCVD_SPAMBOTS</symbol>
+ <symbol weight="3.00" description="Spambots signatures in received headers">R_RCVD_SPAMBOTS</symbol>
  <!-- To header seems to be autogenerated -->
- <symbol weight="3.00">R_TO_SEEMS_AUTO</symbol>
+ <symbol weight="2.00" description="To header seems to be autogenerated">R_TO_SEEMS_AUTO</symbol>
  <!-- Subject needs encoding -->
- <symbol weight="1.00">SUBJECT_NEEDS_ENCODING</symbol>
+ <symbol weight="1.00" description="Subject needs encoding">SUBJECT_NEEDS_ENCODING</symbol>
  <!-- Spam string at the end of message to make statistics faults 0-->
- <symbol weight="3.84">TRACKER_ID</symbol>
+ <symbol weight="3.84" description="Spam string at the end of message to make statistics faults 0">TRACKER_ID</symbol>
  <!-- No space in from header -->
- <symbol weight="3.00">R_NO_SPACE_IN_FROM</symbol>
+ <symbol weight="3.00" description="No space in from header">R_NO_SPACE_IN_FROM</symbol>
  <!-- Subject seems to be spam --> 
- <symbol weight="8.00">R_SAJDING</symbol>
+ <symbol weight="8.00" description="Subject seems to be spam">R_SAJDING</symbol>
  <!-- Detects bad content-transfer-encoding for text parts -->
- <symbol weight="3.00">R_BAD_CTE_7BIT</symbol>
+ <symbol weight="3.00" description="Detects bad content-transfer-encoding for text parts">R_BAD_CTE_7BIT</symbol>
  <!-- Flash redirect on imageshack.us -->
- <symbol weight="10.00">R_FLASH_REDIR_IMGSHACK</symbol>
+ <symbol weight="10.00" description="Flash redirect on imageshack.us">R_FLASH_REDIR_IMGSHACK</symbol>
  <!-- Message id is incorrect -->
- <symbol weight="5.00">INVALID_MSGID</symbol>
+ <symbol weight="5.00" description="Message id is incorrect">INVALID_MSGID</symbol>
  <!-- Message id is missing -->
- <symbol weight="3.00">MISSING_MID</symbol>
+ <symbol weight="3.00" description="Message id is missing ">MISSING_MID</symbol>
  <!-- Recipients are not the same as RCPT TO: mail command -->
- <symbol weight="3.00">FORGED_RECIPIENTS</symbol>
+ <symbol weight="3.00" description="Recipients are not the same as RCPT TO: mail command">FORGED_RECIPIENTS</symbol>
  <!-- Forged Exchange messages -->
- <symbol weight="2.00">RATWARE_MS_HASH</symbol>
+ <symbol weight="2.00" description="Forged Exchange messages ">RATWARE_MS_HASH</symbol>
  <!-- Reply-type in content-type -->
- <symbol weight="1.00">STOX_REPLY_TYPE</symbol>
+ <symbol weight="1.00" description="Reply-type in content-type">STOX_REPLY_TYPE</symbol>
  <!-- IP in received headers is in PBL -->
- <symbol weight="3.00">R_IP_PBL</symbol>
+ <symbol weight="3.00" description="IP in received headers is in PBL">R_IP_PBL</symbol>
  <!-- One received header in a message -->
- <symbol weight="1.00">ONCE_RECEIVED</symbol>
+ <symbol weight="1.00" description="One received header in a message ">ONCE_RECEIVED</symbol>
  <!-- One received header with 'bad' patterns inside -->
- <symbol weight="4.00">ONCE_RECEIVED_STRICT</symbol>
+ <symbol weight="4.00" description="One received header with 'bad' patterns inside">ONCE_RECEIVED_STRICT</symbol>
  <!-- Received headers contains addresses from RBL -->
- <symbol weight="1.00">RECEIVED_RBL</symbol>
+ <symbol weight="1.00" description="Received headers contains addresses from RBL">RECEIVED_RBL</symbol>
  <!-- Text and HTML parts differ -->
- <symbol weight="3.00">R_PARTS_DIFFER</symbol>
+ <symbol weight="3.00" description="Text and HTML parts differ">R_PARTS_DIFFER</symbol>
  <!-- Only Content-Type header without other MIME headers -->
- <symbol weight="2.00">MIME_HEADER_CTYPE_ONLY</symbol>
+ <symbol weight="2.00" description="Only Content-Type header without other MIME headers">MIME_HEADER_CTYPE_ONLY</symbol>
  <!-- Message contains empty parts and image -->
- <symbol weight="2.00">R_EMPTY_IMAGE</symbol>
+ <symbol weight="2.00" description="Message contains empty parts and image ">R_EMPTY_IMAGE</symbol>
 
  <!-- Drugs patterns inside message -->
- <symbol weight="2.00">DRUGS_MANYKINDS</symbol>
+ <symbol weight="2.00" description="Drugs patterns inside message">DRUGS_MANYKINDS</symbol>
  <!-- Specific drugs signatures -->
- <symbol weight="2.00">DRUGS_ANXIETY</symbol>
- <symbol weight="2.00">DRUGS_MUSCLE</symbol>
- <symbol weight="2.00">DRUGS_ANXIETY_EREC</symbol>
- <symbol weight="2.00">DRUGS_DIET</symbol>
- <symbol weight="2.00">DRUGS_ERECTILE</symbol>
+ <symbol weight="2.00" description="">DRUGS_ANXIETY</symbol>
+ <symbol weight="2.00" description="">DRUGS_MUSCLE</symbol>
+ <symbol weight="2.00" description="">DRUGS_ANXIETY_EREC</symbol>
+ <symbol weight="2.00" description="">DRUGS_DIET</symbol>
+ <symbol weight="2.00" description="">DRUGS_ERECTILE</symbol>
 
  <!-- 2 or 3 'advance fee' patterns in a message -->
- <symbol weight="3.30">ADVANCE_FEE_2</symbol>
- <symbol weight="2.12">ADVANCE_FEE_3</symbol>
+ <symbol weight="3.30" description="2 'advance fee' patterns in a message">ADVANCE_FEE_2</symbol>
+ <symbol weight="2.12" description="3 'advance fee' patterns in a message">ADVANCE_FEE_3</symbol>
 
  <!-- Lotto signatures -->
- <symbol weight="8.00">R_LOTTO</symbol>
+ <symbol weight="8.00" description="Lotto signatures">R_LOTTO</symbol>
 
  <!-- Statistics -->
- <symbol weight="3.00">BAYES_SPAM</symbol>
- <symbol weight="-3.00">BAYES_HAM</symbol>
+ <symbol weight="3.00" description="Message probably spam, probability: ">BAYES_SPAM</symbol>
+ <symbol weight="-3.00" description="Message probably ham, probability: ">BAYES_HAM</symbol>
 
  <!-- Fuzzy lists example -->
- <symbol weight="1.00">R_FUZZY</symbol>
- <symbol weight="1.00">R_FUZZY1</symbol>
- <symbol weight="1.00">R_FUZZY2</symbol>
- <symbol weight="1.00">R_FUZZY3</symbol>
+ <symbol weight="1.00" description="">R_FUZZY</symbol>
+ <symbol weight="1.00" description="">R_FUZZY1</symbol>
+ <symbol weight="1.00" description="">R_FUZZY2</symbol>
+ <symbol weight="1.00" description="">R_FUZZY3</symbol>
 
  <!-- SPF rules -->
- <symbol weight="3.00">R_SPF_FAIL</symbol>
- <symbol weight="1.00">R_SPF_SOFTFAIL</symbol>
- <symbol weight="-3.00">R_SPF_ALLOW</symbol>
+ <symbol weight="3.00" description="SPF verification failed">R_SPF_FAIL</symbol>
+ <symbol weight="1.00" description="SPF verification soft-failed">R_SPF_SOFTFAIL</symbol>
+ <symbol weight="-3.00" description="SPF verification alowed">R_SPF_ALLOW</symbol>
 
  <!-- Whitelisted client's IP --> 
- <symbol weight="-2.00">WHITELIST_IP</symbol>
+ <symbol weight="-2.00" description="Whitelisted client's IP">WHITELIST_IP</symbol>
  <!-- Message seems to be from maillist -->
- <symbol weight="-2.00">MAILLIST</symbol>
+ <symbol weight="-2.00" description="Message seems to be from maillist">MAILLIST</symbol>
 
  <!-- multi.surbl.org lists (more details at http://www.surbl.org) -->
  <!-- Phishing and malware sites -->
- <symbol weight="5.50">PH_SURBL_MULTI</symbol>
+ <symbol weight="5.50" description="Phishing and malware sites">PH_SURBL_MULTI</symbol>
  <!-- Outblaze URI Blacklist -->
- <symbol weight="5.50">OB_SURBL_MULTI</symbol>
+ <symbol weight="5.50" description="Outblaze URI Blacklist">OB_SURBL_MULTI</symbol>
  <!-- AbuseButler web sites -->
- <symbol weight="5.50">AB_SURBL_MULTI</symbol>
+ <symbol weight="5.50" description="AbuseButler web sites">AB_SURBL_MULTI</symbol>
  <!-- SpamCop web sites -->
- <symbol weight="5.50">SC_SURBL_MULTI</symbol>
+ <symbol weight="5.50" description="SpamCop web sites">SC_SURBL_MULTI</symbol>
  <!-- jwSpamSpy + Prolocation sites -->
- <symbol weight="5.50">JP_SURBL_MULTI</symbol>
+ <symbol weight="5.50" description="jwSpamSpy + Prolocation sites">JP_SURBL_MULTI</symbol>
  <!-- sa-blacklist web sites -->
- <symbol weight="5.50">WS_SURBL_MULTI</symbol>
+ <symbol weight="5.50" description="sa-blacklist web sites ">WS_SURBL_MULTI</symbol>
 
  <!-- rambler.ru uribl -->
- <symbol weight="9.50">RAMBLER_URIBL</symbol>
+ <symbol weight="9.50" description="rambler.ru uribl">RAMBLER_URIBL</symbol>
 
  <!-- rambler.ru emailbl -->
- <symbol weight="9.50">RAMBLER_EMAILBL</symbol>
+ <symbol weight="9.50" description="rambler.ru emailbl">RAMBLER_EMAILBL</symbol>
 
  <!-- Phished mail -->
- <symbol weight="5.0">PHISHING</symbol>
+ <symbol weight="5.0" description="Phished mail">PHISHING</symbol>
+
+ <!-- Recipients are not the same as RCPT TO: mail command, but from maillist -->
+ <symbol weight="-0.1" description="Recipients are not the same as RCPT TO: mail command, but from maillist">FORGED_RECIPIENTS_MAILLIST</symbol>
+
 
 </metric>
-<!-- End of factors section -->
+<!-- End of metrics section -->
 
 <!-- Composites section -->
+<composite name="FORGED_RECIPIENTS_MAILLIST">FORGED_RECIPIENTS &amp; MAILLIST</composite>
 <!-- End of composites section -->
 
 <!-- Workers section -->
@@ -310,7 +315,7 @@
 
 <!-- Emails blacklist -->
 <module name="emails">
-  <option name="rule">symbol = RAMBLER_EMAILBL, dnsbl = emailbl.rambler.ru, domain_only = false</option>
+  <option name="rule">symbol = RAMBLER_EMAILBL, dnsbl = email-bl.rambler.ru, domain_only = false</option>
   <!--
   <option name="rule">symbol = R_BAD_EMAIL1, map = file:///tmp/emails.list, domain_only = true</option>
   -->
diff --git a/src/client/rspamc.c b/src/client/rspamc.c
index 52c29a395..ab6ba5ad7 100644
--- a/src/client/rspamc.c
+++ b/src/client/rspamc.c
@@ -38,6 +38,7 @@ static gint                     weight = 1;
 static gint                     flag;
 static gboolean                 pass_all;
 static gboolean                 tty = FALSE;
+static gboolean                 verbose = FALSE;
 
 static GOptionEntry entries[] =
 {
@@ -47,6 +48,7 @@ static GOptionEntry entries[] =
 		{ "weight", 'w', 0, G_OPTION_ARG_INT, &weight, "Weight for fuzzy operations", NULL },
 		{ "flag", 'f', 0, G_OPTION_ARG_INT, &flag, "Flag for fuzzy operations", NULL },
 		{ "pass", 'p', 0, G_OPTION_ARG_NONE, &pass_all, "Pass all filters", NULL },
+		{ "verbose", 'v', 0, G_OPTION_ARG_NONE, &verbose, "More verbose output", NULL },
 		{ "ip", 'i', 0, G_OPTION_ARG_STRING, &ip, "Emulate that message was received from specified ip address", NULL },
 		{ NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL }
 };
@@ -159,16 +161,16 @@ show_metric_result (gpointer key, gpointer value, gpointer ud)
 	gboolean                         first;
 
 	if (metric->is_skipped) {
-		PRINT_FUNC ("%s: Skipped\n", key);
+		PRINT_FUNC ("\n%s: Skipped\n", key);
 	}
 	else {
 		if (tty) {
-			PRINT_FUNC ("\033[1m%s:\033[0m %s [ %.2f / %.2f ]\n", key,
+			PRINT_FUNC ("\n\033[1m%s:\033[0m %s [ %.2f / %.2f ]\n", key,
 						metric->score > metric->required_score ? "True" : "False",
 						metric->score, metric->required_score);
 		}
 		else {
-			PRINT_FUNC ("%s: %s [ %.2f / %.2f ]\n", key,
+			PRINT_FUNC ("\n%s: %s [ %.2f / %.2f ]\n", key,
 						metric->score > metric->required_score ? "True" : "False",
 						metric->score, metric->required_score);
 		}
@@ -182,36 +184,62 @@ show_metric_result (gpointer key, gpointer value, gpointer ud)
 			if (metric->action) {
 				PRINT_FUNC ("Action: %s\n", metric->action);
 			}
-			PRINT_FUNC ("Symbols: ");
+			else {
+				PRINT_FUNC ("Symbols: ");
+			}
 		}
 		if (metric->symbols) {
 			first = TRUE;
 			g_hash_table_iter_init (&it, metric->symbols);
 			while (g_hash_table_iter_next (&it, &k, &v)) {
 				s = v;
-				if (! first) {
-					PRINT_FUNC (", ");
+				if (verbose) {
+					if (tty) {
+						PRINT_FUNC ("\n\033[1mSymbol\033[0m - %s(%.2f)", s->name, s->weight);
+					}
+					else {
+						PRINT_FUNC ("\nSymbol - %s(%.2f)", s->name, s->weight);
+					}
+					if (s->options) {
+						PRINT_FUNC (": ");
+						cur = g_list_first (s->options);
+						while (cur) {
+							if (cur->next) {
+								PRINT_FUNC ("%s,", cur->data);
+							}
+							else {
+								PRINT_FUNC ("%s", cur->data);
+							}
+							cur = g_list_next (cur);
+						}
+					}
+					if (s->description) {
+						PRINT_FUNC (" - \"%s\"", s->description);
+					}
 				}
 				else {
-					first = FALSE;
-				}
-				PRINT_FUNC ("%s(%.2f)", s->name, s->weight);
-
-				if (s->options) {
-					PRINT_FUNC ("(");
-					cur = g_list_first (s->options);
-					while (cur) {
-						if (cur->next) {
-							PRINT_FUNC ("%s,", cur->data);
-						}
-						else {
-							PRINT_FUNC ("%s)", cur->data);
+					if (! first) {
+						PRINT_FUNC (", ");
+					}
+					else {
+						first = FALSE;
+					}
+					PRINT_FUNC ("%s(%.2f)", s->name, s->weight);
+
+					if (s->options) {
+						PRINT_FUNC ("(");
+						cur = g_list_first (s->options);
+						while (cur) {
+							if (cur->next) {
+								PRINT_FUNC ("%s,", cur->data);
+							}
+							else {
+								PRINT_FUNC ("%s)", cur->data);
+							}
+							cur = g_list_next (cur);
 						}
-						cur = g_list_next (cur);
 					}
-				}
-				if (s->description) {
-					PRINT_FUNC (" - \"%s\"", s->description);
+
 				}
 			}
 		}
@@ -230,7 +258,7 @@ print_rspamd_result (struct rspamd_result *res)
 	if (tty) {
 		printf ("\033[1m");
 	}
-	PRINT_FUNC ("Results for host: %s\n\n", connect_str);
+	PRINT_FUNC ("Results for host: %s\n", connect_str);
 	if (tty) {
 		printf ("\033[0m");
 	}