From 388c08c712c8279a52ad82caa213191352639ca9 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Wed, 28 Sep 2016 14:30:13 +0100 Subject: [PATCH] [Feature] Stop using of GLists for headers, improve performance --- src/libmime/message.c | 303 +++++++-------------------------- src/libmime/message.h | 35 +--- src/libmime/mime_expressions.c | 6 +- src/libserver/dkim.c | 47 ++--- src/libserver/re_cache.c | 7 +- src/libserver/task.c | 4 +- src/libstat/stat_process.c | 10 +- src/lua/lua_common.h | 2 +- src/lua/lua_mimepart.c | 13 +- src/lua/lua_task.c | 44 ++--- src/plugins/dkim_check.c | 21 +-- src/plugins/fuzzy_check.c | 87 ---------- 12 files changed, 125 insertions(+), 454 deletions(-) diff --git a/src/libmime/message.c b/src/libmime/message.c index 346105438..e4ad8b954 100644 --- a/src/libmime/message.c +++ b/src/libmime/message.c @@ -56,17 +56,17 @@ static void append_raw_header (struct rspamd_task *task, GHashTable *target, struct raw_header *rh) { - struct raw_header *lp; + GPtrArray *ar; - rh->next = NULL; - rh->prev = rh; - if ((lp = - g_hash_table_lookup (target, rh->name)) != NULL) { - DL_APPEND (lp, rh); + if ((ar = g_hash_table_lookup (target, rh->name)) != NULL) { + g_ptr_array_add (ar, rh); } else { - g_hash_table_insert (target, rh->name, rh); + ar = g_ptr_array_sized_new (2); + g_ptr_array_add (ar, rh); + g_hash_table_insert (target, rh->name, ar); } + msg_debug_task ("add raw header %s: %s", rh->name, rh->value); } @@ -108,7 +108,6 @@ process_raw_headers (struct rspamd_task *task, GHashTable *target, new = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct raw_header)); - new->prev = new; l = p - c; tmp = rspamd_mempool_alloc (task->task_pool, l + 1); rspamd_strlcpy (tmp, c, l + 1); @@ -1173,8 +1172,8 @@ mime_foreach_callback (GMimeObject * part, gpointer user_data) sizeof (struct rspamd_mime_part)); hdrs = g_mime_object_get_headers (GMIME_OBJECT (part)); - mime_part->raw_headers = g_hash_table_new (rspamd_strcase_hash, - rspamd_strcase_equal); + mime_part->raw_headers = g_hash_table_new_full (rspamd_strcase_hash, + rspamd_strcase_equal, NULL, rspamd_ptr_array_free_hard); if (hdrs != NULL) { process_raw_headers (task, mime_part->raw_headers, @@ -1243,8 +1242,8 @@ mime_foreach_callback (GMimeObject * part, gpointer user_data) sizeof (struct rspamd_mime_part)); hdrs = g_mime_object_get_headers (GMIME_OBJECT (part)); - mime_part->raw_headers = g_hash_table_new (rspamd_strcase_hash, - rspamd_strcase_equal); + mime_part->raw_headers = g_hash_table_new_full (rspamd_strcase_hash, + rspamd_strcase_equal, NULL, rspamd_ptr_array_free_hard); if (hdrs != NULL) { process_raw_headers (task, mime_part->raw_headers, @@ -1402,7 +1401,7 @@ rspamd_message_parse (struct rspamd_task *task) GMimeParser *parser; GMimeStream *stream; GByteArray *tmp; - GList *first, *cur; + GPtrArray *hdrs; GMimeObject *parent; const GMimeContentType *ct; struct raw_header *rh; @@ -1493,7 +1492,6 @@ rspamd_message_parse (struct rspamd_task *task) RSPAMD_FILTER_ERROR, \ "cannot parse MIME in the message"); - /* TODO: backport to 0.9 */ g_object_unref (parser); return FALSE; } @@ -1566,12 +1564,11 @@ rspamd_message_parse (struct rspamd_task *task) rspamd_archives_process (task); /* Parse received headers */ - first = rspamd_message_get_header (task, "Received", FALSE); + hdrs = rspamd_message_get_header_array (task, "Received", FALSE); - for (cur = first, i = 0; cur != NULL; cur = g_list_next (cur), i ++) { + PTR_ARRAY_FOREACH (hdrs, i, rh) { recv = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct received_header)); - rh = cur->data; rspamd_smtp_recieved_parse (task, rh->decoded, strlen (rh->decoded), recv); /* * For the first header we must ensure that @@ -1640,20 +1637,20 @@ rspamd_message_parse (struct rspamd_task *task) } if (task->from_envelope == NULL) { - first = rspamd_message_get_header (task, "Return-Path", FALSE); + hdrs = rspamd_message_get_header_array (task, "Return-Path", FALSE); - if (first) { - rh = first->data; + if (hdrs && hdrs->len > 0) { + rh = g_ptr_array_index (hdrs, 0); task->from_envelope = rspamd_email_address_from_smtp (rh->decoded, strlen (rh->decoded)); } } if (task->deliver_to == NULL) { - first = rspamd_message_get_header (task, "Delivered-To", FALSE); + hdrs = rspamd_message_get_header_array (task, "Delivered-To", FALSE); - if (first) { - rh = first->data; + if (hdrs && hdrs->len > 0) { + rh = g_ptr_array_index (hdrs, 0); task->deliver_to = rspamd_mempool_strdup (task->task_pool, rh->decoded); } } @@ -1671,10 +1668,12 @@ rspamd_message_parse (struct rspamd_task *task) task->rcpt_mime); #endif } - first = rspamd_message_get_header (task, "From", FALSE); - if (first) { - rh = first->data; + + hdrs = rspamd_message_get_header_array (task, "From", FALSE); + + if (hdrs && hdrs->len > 0) { + rh = g_ptr_array_index (hdrs, 0); task->from_mime = internet_address_list_parse_string (rh->value); if (task->from_mime) { #ifdef GMIME24 @@ -1690,10 +1689,9 @@ rspamd_message_parse (struct rspamd_task *task) } /* Parse urls inside Subject header */ - cur = rspamd_message_get_header (task, "Subject", FALSE); + hdrs = rspamd_message_get_header_array (task, "Subject", FALSE); - for (; cur != NULL; cur = g_list_next (cur)) { - rh = cur->data; + PTR_ARRAY_FOREACH (hdrs, i, rh) { p = rh->decoded; len = strlen (p); rspamd_url_find_multiple (task->task_pool, p, len, FALSE, NULL, @@ -1783,99 +1781,73 @@ rspamd_message_parse (struct rspamd_task *task) return TRUE; } -GList * -rspamd_message_get_header (struct rspamd_task *task, - const gchar *field, - gboolean strong) -{ - GList *gret = NULL; - struct raw_header *rh; - - rh = g_hash_table_lookup (task->raw_headers, field); - - if (rh == NULL) { - return NULL; - } - - while (rh) { - if (strong) { - if (strcmp (rh->name, field) == 0) { - gret = g_list_prepend (gret, rh); - } - } - else { - gret = g_list_prepend (gret, rh); - } - rh = rh->next; - } - - if (gret != NULL) { - gret = g_list_reverse (gret); - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)g_list_free, gret); - } - - return gret; -} GPtrArray * -rspamd_message_get_header_array (struct rspamd_task *task, +rspamd_message_get_header_from_hash (GHashTable *htb, + rspamd_mempool_t *pool, const gchar *field, gboolean strong) { - GPtrArray *ret; - struct raw_header *rh, *cur; - guint nelems = 0; + GPtrArray *ret, *ar; + struct raw_header *cur; + guint i; - rh = g_hash_table_lookup (task->raw_headers, field); + ar = g_hash_table_lookup (htb, field); - if (rh == NULL) { + if (ar == NULL) { return NULL; } - LL_FOREACH (rh, cur) { - nelems ++; - } - - ret = g_ptr_array_sized_new (nelems); + if (strong && pool != NULL) { + /* Need to filter what we have */ + ret = g_ptr_array_sized_new (ar->len); - LL_FOREACH (rh, cur) { - if (strong) { - if (strcmp (rh->name, field) != 0) { + PTR_ARRAY_FOREACH (ar, i, cur) { + if (strcmp (cur->name, field) != 0) { continue; } - } - g_ptr_array_add (ret, cur); - } + g_ptr_array_add (ret, cur); + } - rspamd_mempool_add_destructor (task->task_pool, + rspamd_mempool_add_destructor (pool, (rspamd_mempool_destruct_t)rspamd_ptr_array_free_hard, ret); + } + else { + ret = ar; + } return ret; } +GPtrArray * +rspamd_message_get_header_array (struct rspamd_task *task, + const gchar *field, + gboolean strong) +{ + return rspamd_message_get_header_from_hash (task->raw_headers, + task->task_pool, field, strong); +} + GPtrArray * rspamd_message_get_mime_header_array (struct rspamd_task *task, const gchar *field, gboolean strong) { - GPtrArray *ret; - struct raw_header *rh, *cur; + GPtrArray *ret, *ar; + struct raw_header *cur; guint nelems = 0, i; struct rspamd_mime_part *mp; for (i = 0; i < task->parts->len; i ++) { mp = g_ptr_array_index (task->parts, i); - rh = g_hash_table_lookup (mp->raw_headers, field); + ar = g_hash_table_lookup (mp->raw_headers, field); - if (rh == NULL) { + if (ar == NULL) { continue; } - LL_FOREACH (rh, cur) { - nelems ++; - } + nelems += ar->len; } if (nelems == 0) { @@ -1886,11 +1858,11 @@ rspamd_message_get_mime_header_array (struct rspamd_task *task, for (i = 0; i < task->parts->len; i ++) { mp = g_ptr_array_index (task->parts, i); - rh = g_hash_table_lookup (mp->raw_headers, field); + ar = g_hash_table_lookup (mp->raw_headers, field); - LL_FOREACH (rh, cur) { + PTR_ARRAY_FOREACH (ar, i, cur) { if (strong) { - if (strcmp (rh->name, field) != 0) { + if (strcmp (cur->name, field) != 0) { continue; } } @@ -1904,152 +1876,3 @@ rspamd_message_get_mime_header_array (struct rspamd_task *task, return ret; } - -GPtrArray * -rspamd_message_get_headers_array (struct rspamd_task *task, ...) -{ - va_list ap; - GPtrArray *ret; - struct raw_header *rh, *cur; - guint nelems = 0; - const gchar *hname; - - va_start (ap, task); - - for (hname = va_arg (ap, const char *); hname != NULL; - hname = va_arg (ap, const char *)) { - rh = g_hash_table_lookup (task->raw_headers, hname); - - if (rh == NULL) { - continue; - } - LL_FOREACH (rh, cur) { - nelems ++; - } - } - - va_end (ap); - - if (nelems == 0) { - return NULL; - } - - ret = g_ptr_array_sized_new (nelems); - - /* Restart varargs processing */ - va_start (ap, task); - - for (hname = va_arg (ap, const char *); hname != NULL; - hname = va_arg (ap, const char *)) { - rh = g_hash_table_lookup (task->raw_headers, hname); - - if (rh == NULL) { - continue; - } - LL_FOREACH (rh, cur) { - g_ptr_array_add (ret, cur); - } - } - - va_end (ap); - - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)rspamd_ptr_array_free_hard, ret); - - return ret; -} - -GPtrArray * -rspamd_message_get_header_array_str (struct rspamd_task *task, - const gchar *field, - gboolean strong) -{ - GPtrArray *ret; - struct raw_header *rh, *cur; - guint nelems = 0; - - rh = g_hash_table_lookup (task->raw_headers, field); - - if (rh == NULL) { - return NULL; - } - - LL_FOREACH (rh, cur) { - nelems ++; - } - - ret = g_ptr_array_sized_new (nelems); - - LL_FOREACH (rh, cur) { - if (strong) { - if (strcmp (rh->name, field) != 0) { - continue; - } - } - - if (cur->decoded) { - g_ptr_array_add (ret, cur->decoded); - } - } - - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)rspamd_ptr_array_free_hard, ret); - - return ret; -} - -GPtrArray * -rspamd_message_get_headers_array_str (struct rspamd_task *task, ...) -{ - va_list ap; - GPtrArray *ret; - struct raw_header *rh, *cur; - guint nelems = 0; - const gchar *hname; - - va_start (ap, task); - - for (hname = va_arg (ap, const char *); hname != NULL; - hname = va_arg (ap, const char *)) { - rh = g_hash_table_lookup (task->raw_headers, hname); - - if (rh == NULL) { - continue; - } - LL_FOREACH (rh, cur) { - nelems ++; - } - } - - va_end (ap); - - if (nelems == 0) { - return NULL; - } - - ret = g_ptr_array_sized_new (nelems); - - /* Restart varargs processing */ - va_start (ap, task); - - for (hname = va_arg (ap, const char *); hname != NULL; - hname = va_arg (ap, const char *)) { - rh = g_hash_table_lookup (task->raw_headers, hname); - - if (rh == NULL) { - continue; - } - LL_FOREACH (rh, cur) { - if (cur->decoded) { - g_ptr_array_add (ret, cur->decoded); - } - } - } - - va_end (ap); - - rspamd_mempool_add_destructor (task->task_pool, - (rspamd_mempool_destruct_t)rspamd_ptr_array_free_hard, ret); - - return ret; -} diff --git a/src/libmime/message.h b/src/libmime/message.h index b6f7062dc..2a9201be8 100644 --- a/src/libmime/message.h +++ b/src/libmime/message.h @@ -98,7 +98,6 @@ struct raw_header { gboolean empty_separator; gchar *separator; gchar *decoded; - struct raw_header *prev, *next; }; /** @@ -108,17 +107,6 @@ struct raw_header { */ gboolean rspamd_message_parse (struct rspamd_task *task); -/** - * Get a list of header's values with specified header's name using raw headers - * @param task worker task structure - * @param field header's name - * @param strong if this flag is TRUE header's name is case sensitive, otherwise it is not - * @return A list of header's values or NULL. Unlike previous function it is NOT required to free list or values. I should rework one of these functions some time. - */ -GList * rspamd_message_get_header (struct rspamd_task *task, - const gchar *field, - gboolean strong); - /** * Get an array of header's values with specified header's name using raw headers * @param task worker task structure @@ -141,30 +129,15 @@ GPtrArray *rspamd_message_get_mime_header_array (struct rspamd_task *task, gboolean strong); /** - * Get array of all headers from the list specified - * @param task - * @param h1 - * @return An array of headers (should not be freed as well) - */ -GPtrArray *rspamd_message_get_headers_array (struct rspamd_task *task, ...); - -/** - * Get an array of header's values with specified header's name returning decoded strings as values - * @param task worker task structure + * Get an array of header's values with specified header's name using raw headers + * @param htb hash table indexed by header name (caseless) with ptr arrays as elements * @param field header's name * @param strong if this flag is TRUE header's name is case sensitive, otherwise it is not * @return An array of header's values or NULL. It is NOT permitted to free array or values. */ -GPtrArray *rspamd_message_get_header_array_str (struct rspamd_task *task, +GPtrArray *rspamd_message_get_header_from_hash (GHashTable *htb, + rspamd_mempool_t *pool, const gchar *field, gboolean strong); -/** - * Get array of all headers from the list specified returning decoded strings as values - * @param task - * @param h1 - * @return An array of headers (should not be freed as well) - */ -GPtrArray *rspamd_message_get_headers_array_str (struct rspamd_task *task, ...); - #endif diff --git a/src/libmime/mime_expressions.c b/src/libmime/mime_expressions.c index bdf5644fd..4fd1a893c 100644 --- a/src/libmime/mime_expressions.c +++ b/src/libmime/mime_expressions.c @@ -1012,7 +1012,7 @@ gboolean rspamd_header_exists (struct rspamd_task * task, GArray * args, void *unused) { struct expression_argument *arg; - GList *headerlist; + GPtrArray *headerlist; if (args == NULL || task == NULL) { return FALSE; @@ -1025,12 +1025,14 @@ rspamd_header_exists (struct rspamd_task * task, GArray * args, void *unused) } debug_task ("try to get header %s", (gchar *)arg->data); - headerlist = rspamd_message_get_header (task, + headerlist = rspamd_message_get_header_array (task, (gchar *)arg->data, FALSE); + if (headerlist) { return TRUE; } + return FALSE; } diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c index 2d968bd0f..aedcf99c9 100644 --- a/src/libserver/dkim.c +++ b/src/libserver/dkim.c @@ -1709,43 +1709,25 @@ rspamd_dkim_canonize_header (struct rspamd_dkim_common_ctx *ctx, const gchar *dkim_header, const gchar *dkim_domain) { - struct raw_header *rh, *rh_iter; - guint rh_num = 0; - guint i; - GPtrArray *sign_headers; + struct raw_header *rh; + guint rh_num = 0, i; + GPtrArray *ar; if (dkim_header == NULL) { - rh = g_hash_table_lookup (task->raw_headers, header_name); - - if (rh) { - LL_FOREACH (rh, rh_iter) { - rh_num++; - } + ar = g_hash_table_lookup (task->raw_headers, header_name); - if (rh_num > count) { + if (ar) { + if (ar->len > count) { /* Set skip count */ - rh_num -= count; + rh_num = ar->len - count; } else { rh_num = 0; } - sign_headers = g_ptr_array_sized_new (rh_num); - /* Skip number of headers */ - rh_iter = rh; - while (rh_num) { - rh_iter = rh_iter->next; - rh_num--; - } - /* Now insert required headers */ - while (rh_iter) { - g_ptr_array_add (sign_headers, rh_iter); - rh_iter = rh_iter->next; - } - - for (i = 0; i < sign_headers->len; i ++) { - rh = g_ptr_array_index (sign_headers, i); + for (i = rh_num; i < ar->len; i ++) { + rh = g_ptr_array_index (ar, i); if (ctx->header_canon_type == DKIM_CANON_SIMPLE) { rspamd_dkim_hash_update (ctx->headers_hash, rh->raw_value, @@ -1756,30 +1738,25 @@ rspamd_dkim_canonize_header (struct rspamd_dkim_common_ctx *ctx, else { if (!rspamd_dkim_canonize_header_relaxed (ctx, rh->value, header_name, FALSE)) { - - g_ptr_array_free (sign_headers, TRUE); return FALSE; } } } - - g_ptr_array_free (sign_headers, TRUE); } } else { /* For signature check just use the saved dkim header */ if (ctx->header_canon_type == DKIM_CANON_SIMPLE) { /* We need to find our own signature and use it */ - rh = g_hash_table_lookup (task->raw_headers, DKIM_SIGNHEADER); + ar = g_hash_table_lookup (task->raw_headers, DKIM_SIGNHEADER); - if (rh) { + if (ar) { /* We need to find our own signature */ if (!dkim_domain) { return FALSE; } - - LL_FOREACH (rh, rh_iter) { + PTR_ARRAY_FOREACH (ar, i, rh) { if (rspamd_substring_search_twoway (rh->raw_value, rh->raw_len, dkim_domain, strlen (dkim_domain)) != -1) { diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c index c2ade15c5..955aa91e0 100644 --- a/src/libserver/re_cache.c +++ b/src/libserver/re_cache.c @@ -742,7 +742,6 @@ rspamd_re_cache_exec_re (struct rspamd_task *task, { guint ret = 0, i, re_id; GPtrArray *headerlist; - GList *slist; GHashTableIter it; struct raw_header *rh; const gchar *in, *end; @@ -966,10 +965,10 @@ rspamd_re_cache_exec_re (struct rspamd_task *task, * of the body content. */ - slist = rspamd_message_get_header (task, "Subject", FALSE); + headerlist = rspamd_message_get_header_array (task, "Subject", FALSE); - if (slist) { - rh = slist->data; + if (headerlist && headerlist->len > 0) { + rh = g_ptr_array_index (headerlist, 0); scvec[0] = (guchar *)rh->decoded; lenvec[0] = strlen (rh->decoded); diff --git a/src/libserver/task.c b/src/libserver/task.c index 950564b2d..e6cede086 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -88,8 +88,8 @@ rspamd_task_new (struct rspamd_worker *worker, struct rspamd_config *cfg) (rspamd_mempool_destruct_t) g_hash_table_unref, new_task->results); - new_task->raw_headers = g_hash_table_new (rspamd_strcase_hash, - rspamd_strcase_equal); + new_task->raw_headers = g_hash_table_new_full (rspamd_strcase_hash, + rspamd_strcase_equal, NULL, rspamd_ptr_array_free_hard); new_task->request_headers = g_hash_table_new_full (rspamd_ftok_icase_hash, rspamd_ftok_icase_equal, rspamd_fstring_mapped_ftok_free, rspamd_request_header_dtor); diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c index c07268279..6a1480ec5 100644 --- a/src/libstat/stat_process.c +++ b/src/libstat/stat_process.c @@ -35,14 +35,16 @@ static void rspamd_stat_tokenize_header (struct rspamd_task *task, const gchar *name, const gchar *prefix, GArray *ar) { - struct raw_header *rh, *cur; + struct raw_header *cur; + GPtrArray *hdrs; + guint i; rspamd_ftok_t str; - rh = g_hash_table_lookup (task->raw_headers, name); + hdrs = g_hash_table_lookup (task->raw_headers, name); - if (rh != NULL) { + if (hdrs != NULL) { - LL_FOREACH (rh, cur) { + PTR_ARRAY_FOREACH (hdrs, i, cur) { if (cur->name != NULL) { str.begin = cur->name; str.len = strlen (cur->name); diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h index 1fefdc2cb..249ca9b90 100644 --- a/src/lua/lua_common.h +++ b/src/lua/lua_common.h @@ -192,7 +192,7 @@ struct rspamd_lua_text * lua_check_text (lua_State * L, gint pos); * Push specific header to lua */ gint rspamd_lua_push_header (lua_State * L, - GHashTable *hdrs, + GPtrArray *hdrs, const gchar *name, gboolean strong, gboolean full, diff --git a/src/lua/lua_mimepart.c b/src/lua/lua_mimepart.c index d519cccbe..a2b6cc49c 100644 --- a/src/lua/lua_mimepart.c +++ b/src/lua/lua_mimepart.c @@ -668,19 +668,22 @@ lua_mimepart_get_filename (lua_State * L) static gint lua_mimepart_get_header_common (lua_State *L, gboolean full, gboolean raw) { - gboolean strong = FALSE; struct rspamd_mime_part *part = lua_check_mimepart (L); const gchar *name; + GPtrArray *ar; name = luaL_checkstring (L, 2); if (name && part) { - if (lua_gettop (L) == 3) { - strong = lua_toboolean (L, 3); - } - return rspamd_lua_push_header (L, part->raw_headers, name, strong, full, raw); + + ar = rspamd_message_get_header_from_hash (part->raw_headers, NULL, + name, FALSE); + + return rspamd_lua_push_header (L, ar, name, FALSE, full, raw); } + lua_pushnil (L); + return 1; } diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index 8c6a3473c..db10abe01 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -1361,57 +1361,40 @@ lua_task_set_request_header (lua_State *L) gint rspamd_lua_push_header (lua_State * L, - GHashTable *hdrs, + GPtrArray *ar, const gchar *name, gboolean strong, gboolean full, gboolean raw) { - struct raw_header *rh, *cur; - gint i = 1; + struct raw_header *rh; + guint i; const gchar *val; - rh = g_hash_table_lookup (hdrs, name); - - if (rh == NULL) { + if (ar == NULL || ar->len == 0) { lua_pushnil (L); return 1; } if (full) { - i = 0; - LL_FOREACH (rh, cur) { - i ++; - } - - lua_createtable (L, i, 0); + lua_createtable (L, ar->len, 0); } - i = 1; - - while (rh) { - if (rh->name == NULL) { - rh = rh->next; - continue; - } - /* Check case sensivity */ - if (strong) { - if (strcmp (rh->name, name) != 0) { - rh = rh->next; - continue; - } - } + PTR_ARRAY_FOREACH (ar, i, rh) { if (full) { /* Create new associated table for a header */ lua_createtable (L, 0, 6); rspamd_lua_table_set (L, "name", rh->name); + if (rh->value) { rspamd_lua_table_set (L, "value", rh->value); } + if (rh->decoded) { rspamd_lua_table_set (L, "decoded", rh->value); } + lua_pushstring (L, "tab_separated"); lua_pushboolean (L, rh->tab_separated); lua_settable (L, -3); @@ -1419,9 +1402,7 @@ rspamd_lua_push_header (lua_State * L, lua_pushboolean (L, rh->empty_separator); lua_settable (L, -3); rspamd_lua_table_set (L, "separator", rh->separator); - lua_rawseti (L, -2, i++); - /* Process next element */ - rh = rh->next; + lua_rawseti (L, -2, i + 1); } else { if (!raw) { @@ -1451,6 +1432,7 @@ lua_task_get_header_common (lua_State *L, gboolean full, gboolean raw) gboolean strong = FALSE; struct rspamd_task *task = lua_check_task (L, 1); const gchar *name; + GPtrArray *ar; name = luaL_checkstring (L, 2); @@ -1459,7 +1441,9 @@ lua_task_get_header_common (lua_State *L, gboolean full, gboolean raw) strong = lua_toboolean (L, 3); } - return rspamd_lua_push_header (L, task->raw_headers, name, + ar = rspamd_message_get_header_array (task, name, strong); + + return rspamd_lua_push_header (L, ar, name, strong, full, raw); } else { diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index 6742fd0ef..7f9122212 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -684,13 +684,14 @@ dkim_module_key_handler (rspamd_dkim_key_t *key, static void dkim_symbol_callback (struct rspamd_task *task, void *unused) { - GList *hlist; + GPtrArray *hlist; rspamd_dkim_context_t *ctx; rspamd_dkim_key_t *key; GError *err = NULL; struct raw_header *rh; struct dkim_check_result *res = NULL, *cur; - guint checked = 0; + guint checked = 0, i; + /* First check if plugin should be enabled */ if (task->user != NULL || rspamd_inet_address_is_local (task->from_addr)) { msg_info_task ("skip DKIM checks for local networks and authorized users"); @@ -704,19 +705,16 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) } /* Now check if a message has its signature */ - hlist = rspamd_message_get_header (task, + hlist = rspamd_message_get_header_array (task, DKIM_SIGNHEADER, FALSE); - if (hlist != NULL) { + if (hlist != NULL && hlist->len > 0) { msg_debug_task ("dkim signature found"); - while (hlist != NULL) { - rh = (struct raw_header *)hlist->data; - + PTR_ARRAY_FOREACH (hlist, i, rh) { if (rh->decoded == NULL || rh->decoded[0] == '\0') { msg_info_task ("<%s> cannot load empty DKIM context", task->message_id); - hlist = g_list_next (hlist); continue; } @@ -740,6 +738,7 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) task->task_pool, dkim_module_ctx->time_jitter, &err); + if (ctx == NULL) { if (err != NULL) { msg_info_task ("<%s> cannot parse DKIM context: %e", @@ -753,7 +752,6 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) task->message_id); } - hlist = g_list_next (hlist); continue; } else { @@ -767,7 +765,6 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) rspamd_dkim_get_domain (ctx)) == NULL)) { msg_debug_task ("skip dkim check for %s domain", rspamd_dkim_get_domain (ctx)); - hlist = g_list_next (hlist); continue; } @@ -795,7 +792,7 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) } if (dkim_module_ctx->skip_multi) { - if (hlist->next) { + if (hlist->len > 1) { msg_info_task ("message has multiple signatures but we" " check only one as 'skip_multi' is set"); } @@ -811,8 +808,6 @@ dkim_symbol_callback (struct rspamd_task *task, void *unused) " is reached", checked); break; } - - hlist = g_list_next (hlist); } } else { diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index d2b9d6601..54b27e650 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -1064,93 +1064,6 @@ fuzzy_encrypt_cmd (struct fuzzy_rule *rule, rspamd_pubkey_alg (rule->peer_key)); } -static struct fuzzy_cmd_io * -fuzzy_cmd_from_task_meta (struct fuzzy_rule *rule, - int c, - gint flag, - guint32 weight, - rspamd_mempool_t *pool, - struct rspamd_task *task) -{ - struct rspamd_fuzzy_cmd *cmd; - struct rspamd_fuzzy_encrypted_cmd *enccmd; - struct fuzzy_cmd_io *io; - rspamd_cryptobox_hash_state_t st; - - GHashTableIter it; - gpointer k, v; - struct rspamd_url *u; - struct raw_header *rh; - GList *cur; - - if (rule->peer_key) { - enccmd = rspamd_mempool_alloc0 (pool, sizeof (*enccmd)); - cmd = &enccmd->cmd; - } - else { - cmd = rspamd_mempool_alloc0 (pool, sizeof (*cmd)); - } - - cmd->cmd = c; - cmd->version = RSPAMD_FUZZY_PLUGIN_VERSION; - if (c != FUZZY_CHECK) { - cmd->flag = flag; - cmd->value = weight; - } - cmd->shingles_count = 0; - cmd->tag = ottery_rand_uint32 (); - /* Use blake2b for digest */ - rspamd_cryptobox_hash_init (&st, rule->hash_key->str, rule->hash_key->len); - /* Hash URL's */ - g_hash_table_iter_init (&it, task->urls); - - while (g_hash_table_iter_next (&it, &k, &v)) { - u = v; - if (u->hostlen > 0) { - rspamd_cryptobox_hash_update (&st, u->host, u->hostlen); - } - if (u->datalen > 0) { - rspamd_cryptobox_hash_update (&st, u->data, u->datalen); - } - } - /* Now get some headers to iterate on */ - - cur = rule->fuzzy_headers; - - while (cur) { - rh = g_hash_table_lookup (task->raw_headers, cur->data); - - while (rh) { - if (rh->decoded) { - rspamd_cryptobox_hash_update (&st, rh->decoded, - strlen (rh->decoded)); - } - - rh = rh->next; - } - cur = g_list_next (cur); - } - - rspamd_cryptobox_hash_final (&st, cmd->digest); - - io = rspamd_mempool_alloc (pool, sizeof (*io)); - io->flags = 0; - io->tag = cmd->tag; - memcpy (&io->cmd, cmd, sizeof (io->cmd)); - - if (rule->peer_key) { - fuzzy_encrypt_cmd (rule, &enccmd->hdr, (guchar *)cmd, sizeof (*cmd)); - io->io.iov_base = enccmd; - io->io.iov_len = sizeof (*enccmd); - } - else { - io->io.iov_base = cmd; - io->io.iov_len = sizeof (*cmd); - } - - return io; -} - static struct fuzzy_cmd_io * fuzzy_cmd_stat (struct fuzzy_rule *rule, int c, -- 2.39.5