reconf['HTML_SHORT_LINK_IMG_2'] = string.format('(%s) & (%s)', html_length_1024_1536, html_link_image)
-- Local rules
-local r_bgcolor = '/BGCOLOR=/iM'
-local r_font_color = '/font color=[\\"\']?\\#FFFFFF[\\"\']?/iM'
+local r_bgcolor = '/BGCOLOR=/iP'
+local r_font_color = '/font color=[\\"\']?\\#FFFFFF[\\"\']?/iP'
reconf['R_WHITE_ON_WHITE'] = string.format('(!(%s) & (%s))', r_bgcolor, r_font_color)
reconf['R_FLASH_REDIR_IMGSHACK'] = '/^(?:http:\\/\\/)?img\\d{1,5}\\.imageshack\\.us\\/\\S+\\.swf/U'
local r_rcvd_from_valuehost = 'Received=/\\sb0\\.valuehost\\.ru/H'
local r_cyr_phone = '/8 \\(\\xD799\\)/P'
reconf['R_SPAM_FROM_VALUEHOST'] = string.format('(%s) & (%s)', r_rcvd_from_valuehost, r_cyr_phone)
+-- Different text parts
+reconf['R_PARTS_DIFFER'] = 'compare_parts_distance(70)';
uint32_t log_buf_size; /**< length of log buffer */
gchar *debug_ip_map; /**< turn on debugging for specified ip addresses */
gboolean log_urls; /**< whether we should log URLs */
+ GList *debug_symbols; /**< symbols to debug */
gsize max_statfile_size; /**< maximum size for statfile */
gboolean delivery_enable; /**< is delivery agent is enabled */
gchar *deliver_host; /**< host for mail deliviring */
struct in_addr deliver_addr; /**< its address */
- uint16_t deliver_port; /**< port for deliviring */
- uint16_t deliver_family; /**< socket family for delivirnig */
+ guint16 deliver_port; /**< port for deliviring */
+ guint16 deliver_family; /**< socket family for delivirnig */
gchar *deliver_agent_path; /**< deliver to pipe instead of socket */
gboolean deliver_lmtp; /**< use LMTP instead of SMTP */
gchar* dump_checksum; /**< dump checksum of config file */
gpointer lua_state; /**< pointer to lua state */
- guint dns_timeout; /**< timeout in milliseconds for waiting for dns reply */
- guint dns_retransmits; /**< maximum retransmits count */
+ guint32 dns_timeout; /**< timeout in milliseconds for waiting for dns reply */
+ guint32 dns_retransmits; /**< maximum retransmits count */
GList *nameservers; /**< list of nameservers or NULL to parse resolv.conf */
};
G_STRUCT_OFFSET (struct config_file, cache_filename),
NULL
},
+ {
+ "dns_timeout",
+ xml_handle_seconds,
+ G_STRUCT_OFFSET (struct config_file, dns_timeout),
+ NULL
+ },
+ {
+ "dns_retransmits",
+ xml_handle_uint32,
+ G_STRUCT_OFFSET (struct config_file, dns_retransmits),
+ NULL
+ },
NULL_ATTR
},
NULL_ATTR
G_STRUCT_OFFSET (struct config_file, debug_ip_map),
NULL
},
+ {
+ "debug_symbols",
+ xml_handle_string_list,
+ G_STRUCT_OFFSET (struct config_file, debug_symbols),
+ NULL
+ },
NULL_ATTR
},
NULL_ATTR
return TRUE;
}
+gboolean
+xml_handle_string_list (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset)
+{
+ GList **dest;
+ gchar **tokens, **cur;
+
+ dest = (GList **)G_STRUCT_MEMBER_P (dest_struct, offset);
+ *dest = NULL;
+
+ tokens = g_strsplit (data, ";,", 0);
+ if (!tokens || !tokens[0]) {
+ return FALSE;
+ }
+ memory_pool_add_destructor (cfg->cfg_pool, (pool_destruct_func)g_strfreev, tokens);
+ cur = tokens;
+ while (*cur) {
+ *dest = g_list_prepend (*dest, *cur);
+ cur ++;
+ }
+
+ return TRUE;
+}
+
gboolean
xml_handle_size (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset)
gboolean
xml_handle_seconds (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset)
{
- time_t *dest;
+ guint32 *dest;
- dest = (time_t *)G_STRUCT_MEMBER_P (dest_struct, offset);
+ dest = (guint32 *)G_STRUCT_MEMBER_P (dest_struct, offset);
*dest = parse_seconds (data);
return TRUE;
/* Handlers */
/* Basic xml parsing functions */
gboolean xml_handle_string (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset);
+gboolean xml_handle_string_list (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset);
/* Numeric params */
gboolean xml_handle_size (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHashTable *attrs, gchar *data, gpointer user_data, gpointer dest_struct, int offset);
struct mime_text_part *p1, *p2;
GList *cur;
struct expression_argument *arg;
+ GMimeObject *parent;
+ const GMimeContentType *ct;
+
if (args == NULL) {
debug_task ("no threshold is specified, assume it 100");
return FALSE;
}
p2 = cur->data;
+ /* First of all check parent object */
+ if (p1->parent && p1->parent == p2->parent) {
+ parent = p1->parent;
+ ct = g_mime_object_get_content_type (parent);
+ if (ct == NULL || ! g_mime_content_type_is_type (ct, "multipart", "alternative")) {
+ debug_task ("two parts are not belong to multipart/alternative container, skip check");
+ return FALSE;
+ }
+ }
+ else {
+ debug_task ("message contains two parts but they are in different multi-parts");
+ return FALSE;
+ }
if (!p1->is_empty && !p2->is_empty) {
diff = fuzzy_compare_hashes (p1->fuzzy, p2->fuzzy);
debug_task ("got likeliness between parts of %d%%, threshold is %d%%", diff, threshold);
g_hash_table_insert (metric_res->symbols, (gpointer) symbol, s);
}
- debug_task ("got %.2f score for metric %s, factor: %f", s->score, metric->name, w);
+ debug_task ("symbol %s, score %.2f, metric %s, factor: %f", symbol, s->score, metric->name, w);
}
int fd;
gboolean is_buffered;
gboolean enabled;
+ gboolean is_debug;
+ gboolean throttling;
+ time_t throttling_time;
enum rspamd_log_type type;
pid_t pid;
enum process_type process_type;
}
else if (errno == EFAULT || errno == EINVAL || errno == EFBIG || errno == ENOSPC) {
/* Rare case */
- if (write (rspamd_log->fd, errmsg, r) == -1) {
- /* Don't know what to do */
- exit (EXIT_FAILURE);
- }
+ rspamd_log->throttling = TRUE;
+ rspamd_log->throttling_time = time (NULL);
}
else if (errno == EPIPE) {
/* We write to some pipe and it disappears, disable logging */
rspamd_log->enabled = FALSE;
}
}
+ else if (rspamd_log->throttling) {
+ rspamd_log->throttling = FALSE;
+ }
}
}
uint32_t cksum;
size_t mlen;
const char *cptype = NULL;
+ gboolean got_time = FALSE;
if (! rspamd_log->enabled) {
return;
if (forced || log_level <= rspamd_log->cfg->log_level) {
+ /* Check throttling due to write errors */
+ if (rspamd_log->throttling) {
+ now = time (NULL);
+ if (rspamd_log->throttling_time != now) {
+ rspamd_log->throttling_time = now;
+ got_time = TRUE;
+ }
+ else {
+ /* Do not try to write to file too often while throtling */
+ return;
+ }
+ }
/* Check repeats */
mlen = strlen (message);
cksum = rspamd_log_calculate_cksum (message, mlen);
}
}
- now = time (NULL);
+ if (! got_time) {
+ now = time (NULL);
+ }
+
tms = localtime (&now);
strftime (timebuf, sizeof (timebuf), "%F %H:%M:%S", tms);
va_list vp;
u_char *end;
- if (rspamd_log->cfg->log_level >= G_LOG_LEVEL_DEBUG ||
+ if (rspamd_log->cfg->log_level >= G_LOG_LEVEL_DEBUG || rspamd_log->is_debug ||
(rspamd_log->debug_ip != NULL && radix32tree_find (rspamd_log->debug_ip, ntohl (addr)) != RADIX_NO_VALUE)) {
va_start (vp, fmt);
rspamd_log->log_func (log_domain, NULL, log_level, message, FALSE, rspamd_log->cfg);
}
}
+
+void
+rspamd_log_debug ()
+{
+ rspamd_log->is_debug = TRUE;
+}
+
+void
+rspamd_log_nodebug ()
+{
+ rspamd_log->is_debug = FALSE;
+}
*/
void rspamd_conditional_debug (uint32_t addr, const char *function, const char *fmt, ...) ;
+/**
+ * Temporary turn on debug
+ */
+void rspamd_log_debug ();
+
+/**
+ * Turn off debug
+ */
+void rspamd_log_nodebug ();
/* Typical functions */
init_lua_filters (struct config_file *cfg)
{
struct config_file **pcfg;
+ gpointer *plogger;
GList *cur, *tmp;
struct script_module *module;
struct statfile *st;
continue;
}
- /* Call module init function */
+ /* Initialize config structure */
pcfg = lua_newuserdata (L, sizeof (struct config_file *));
lua_setclass (L, "rspamd{config}", -1);
*pcfg = cfg;
lua_setglobal (L, "rspamd_config");
+ /* Initialize logger */
+ plogger = lua_newuserdata (L, sizeof (gpointer));
+ lua_setclass (L, "rspamd{logger}", -1);
+ *plogger = NULL;
+ lua_setglobal (L, "rspamd_log");
+
/* do the call (1 arguments, 1 result) */
if (lua_pcall (L, 0, LUA_MULTRET, 0) != 0) {
msg_info ("init of %s failed: %s", module->path, lua_tostring (L, -1));
struct rspamd_async_session* s; /**< async session object */
int parts_count; /**< mime parts count */
GMimeMessage *message; /**< message, parsed with GMime */
+ GMimeObject *parser_parent_part; /**< current parent part */
InternetAddressList *rcpts; /**< list of all recipients */
GList *parts; /**< list of parsed parts */
GList *text_parts; /**< list of text parts */
}
static void
-process_text_part (struct worker_task *task, GByteArray * part_content, GMimeContentType * type, gboolean is_empty)
+process_text_part (struct worker_task *task, GByteArray *part_content, GMimeContentType *type,
+ GMimeObject *part, GMimeObject *parent, gboolean is_empty)
{
struct mime_text_part *text_part;
+ const char *cd;
+
+ /* Skip attachements */
+#ifndef GMIME24
+ cd = g_mime_part_get_content_disposition (GMIME_PART (part));
+ if (cd && g_ascii_strcasecmp (cd, "attachment") == 0) {
+ debug_task ("skip attachments for checking as text parts");
+ return;
+ }
+#else
+ cd = g_mime_object_get_disposition (GMIME_OBJECT (part));
+ if (cd && g_ascii_strcasecmp (cd, GMIME_DISPOSITION_ATTACHMENT) == 0) {
+ debug_task ("skip attachments for checking as text parts");
+ return;
+ }
+#endif
if (g_mime_content_type_is_type (type, "text", "html") || g_mime_content_type_is_type (type, "text", "xhtml")) {
debug_task ("got urls from text/html part");
text_part->orig = convert_text_to_utf (task, part_content, type, text_part);
text_part->is_balanced = TRUE;
text_part->html_nodes = NULL;
+ text_part->parent = parent;
text_part->html_urls = g_tree_new ((GCompareFunc) g_ascii_strcasecmp);
text_part->urls = g_tree_new ((GCompareFunc) g_ascii_strcasecmp);
text_part = memory_pool_alloc0 (task->task_pool, sizeof (struct mime_text_part));
text_part->is_html = FALSE;
+ text_part->parent = parent;
if (is_empty) {
text_part->is_empty = TRUE;
text_part->orig = NULL;
}
else if (GMIME_IS_MULTIPART (part)) {
/* multipart/mixed, multipart/alternative, multipart/related, multipart/signed, multipart/encrypted, etc... */
+ task->parser_parent_part = part;
#ifndef GMIME24
debug_task ("detected multipart part");
/* we'll get to finding out if this is a signed/encrypted multipart later... */
return;
}
#endif
- /* XXX: do nothing with multiparts in gmime 2.4 */
}
else if (GMIME_IS_PART (part)) {
/* a normal leaf part, could be text/plain or image/jpeg etc */
mime_part = memory_pool_alloc (task->task_pool, sizeof (struct mime_part));
mime_part->type = type;
mime_part->content = part_content;
+ mime_part->parent = task->parser_parent_part;
debug_task ("found part with content-type: %s/%s", type->type, type->subtype);
task->parts = g_list_prepend (task->parts, mime_part);
/* Skip empty parts */
- process_text_part (task, part_content, type, (part_content->len <= 0));
+ process_text_part (task, part_content, type, part, task->parser_parent_part, (part_content->len <= 0));
}
else {
msg_warn ("write to stream failed: %d, %s", errno, strerror (errno));
#ifdef GMIME24
g_mime_message_foreach (message, mime_foreach_callback, task);
#else
+ /*
+ * This is rather strange, but gmime 2.2 do NOT pass top-level part to foreach callback
+ * so we need to set up parent part by hands
+ */
+ task->parser_parent_part = g_mime_message_get_mime_part (message);
+ g_object_unref (task->parser_parent_part);
g_mime_message_foreach_part (message, mime_foreach_callback, task);
#endif
struct mime_part {
GMimeContentType *type;
GByteArray *content;
+ GMimeObject *parent;
};
struct mime_text_part {
GTree *urls;
GTree *html_urls;
fuzzy_hash_t *fuzzy;
+ GMimeObject *parent;
};
struct received_header {
return FALSE;
}
+static gboolean
+check_debug_symbol (struct config_file *cfg, const char *symbol)
+{
+ GList *cur;
+
+ cur = cfg->debug_symbols;
+ while (cur) {
+ if (strcmp (symbol, (const char *)cur->data) == 0) {
+ return TRUE;
+ }
+ cur = g_list_next (cur);
+ }
+
+ return FALSE;
+}
+
struct symbol_callback_data {
enum {
CACHE_STATE_NEGATIVE,
msg_warn ("gettimeofday failed: %s", strerror (errno));
}
#endif
- item->func (task, item->user_data);
+ if (G_UNLIKELY (check_debug_symbol (task->cfg, item->s->symbol))) {
+ rspamd_log_debug ();
+ item->func (task, item->user_data);
+ rspamd_log_nodebug ();
+ }
+ else {
+ item->func (task, item->user_data);
+ }
+
#ifdef HAVE_CLOCK_GETTIME
# ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID
projects / rspamd.git / commitdiff