Rework LRU hash logic.

diff --git a/src/dkim.c b/src/dkim.c
index 72a14c1798467a07ba6eafb364858f9544d48343..b68eec879a917e3f923e6115052947ce1eb8b0ff 100644 (file)
--- a/src/dkim.c
+++ b/src/dkim.c
@@ -767,6 +767,7 @@ rspamd_dkim_dns_cb (struct rspamd_dns_reply *reply, gpointer arg)
                        if (elt->type == DNS_REQUEST_TXT) {
                                key = rspamd_dkim_parse_key (elt->content.txt.data, &keylen, &err);
                                if (key) {
+                                       key->ttl = elt->ttl;
                                        break;
                                }
                        }

diff --git a/src/dkim.h b/src/dkim.h
index f4fc21863e9c3dcee9fb84e753e08a2b33e847b5..e49b9c47953009a99e2a3e01c282b54de18d94a1 100644 (file)
--- a/src/dkim.h
+++ b/src/dkim.h
@@ -155,6 +155,7 @@ typedef struct rspamd_dkim_key_s {
        guint8 *keydata;
        guint keylen;
        gsize decoded_len;
+       guint ttl;
 #ifdef HAVE_OPENSSL
        RSA *key_rsa;
        BIO *key_bio;

diff --git a/src/hash.c b/src/hash.c
index e8cf2d773cc394515ab35f75a145708fdbaa5425..e5efc72a7a020407fe999da28d998e4eb394a7bc 100644 (file)
--- a/src/hash.c
+++ b/src/hash.c
@@ -313,20 +313,21 @@ rspamd_lru_hash_destroy_node (gpointer v)
        if (node->hash->value_destroy) {
                node->hash->value_destroy (node->data);
        }
-
+       g_queue_delete_link (node->hash->q, node->link);
        g_slice_free1 (sizeof (rspamd_lru_element_t), node);
 }
 
 static rspamd_lru_element_t*
-rspamd_lru_create_node (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now)
+rspamd_lru_create_node (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now, guint ttl)
 {
        rspamd_lru_element_t           *node;
 
        node = g_slice_alloc (sizeof (rspamd_lru_element_t));
-       node->hash = hash;
        node->data = value;
        node->key = key;
        node->store_time = now;
+       node->ttl = ttl;
+       node->hash = hash;
 
        return node;
 }
@@ -402,26 +403,24 @@ rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, gpointer key, time_t now)
        rspamd_lru_element_t           *res;
 
        if ((res = hash->lookup_func (hash->storage, key)) != NULL) {
+               if (res->ttl != 0) {
+                       if (now - res->store_time > res->ttl) {
+                               hash->delete_func (hash->storage, key);
+                               return NULL;
+                       }
+               }
                if (hash->maxage > 0) {
                        if (now - res->store_time > hash->maxage) {
+                               res = g_queue_peek_tail (hash->q);
                                /* Expire elements from queue tail */
-                               res = g_queue_pop_tail (hash->q);
-
                                while (res != NULL && now - res->store_time > hash->maxage) {
                                        hash->delete_func (hash->storage, res->key);
-                                       res = g_queue_pop_tail (hash->q);
-                               }
-                               /* Restore last element */
-                               if (res != NULL) {
-                                       g_queue_push_tail (hash->q, res);
+                                       res = g_queue_peek_tail (hash->q);
                                }
 
                                return NULL;
                        }
                }
-       }
-
-       if (res) {
                return res->data;
        }
 
@@ -434,32 +433,45 @@ rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, gpointer key, time_t now)
  * @param value value of key
  */
 void
-rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now)
+rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value,
+               time_t now, guint ttl)
 {
        rspamd_lru_element_t           *res;
        gint                            removed = 0;
 
-       if ((gint)g_queue_get_length (hash->q) >= hash->maxsize) {
-               /* Expire some elements */
-               res = g_queue_pop_tail (hash->q);
-               if (hash->maxage > 0) {
-                       while (res != NULL && now - res->store_time > hash->maxage) {
+       if ((res = hash->lookup_func (hash->storage, key)) != NULL) {
+               hash->delete_func (hash->storage, res->key);
+       }
+       else {
+               if (hash->maxsize > 0 &&
+                               (gint)g_queue_get_length (hash->q) >= hash->maxsize) {
+                       /* Expire some elements */
+                       res = g_queue_peek_tail (hash->q);
+                       if (hash->maxage > 0) {
+                               while (res != NULL && now - res->store_time > hash->maxage) {
+                                       if (res->key != NULL) {
+                                               hash->delete_func (hash->storage, res->key);
+                                       }
+                                       else {
+                                               break;
+                                       }
+                                       res = g_queue_peek_tail (hash->q);
+                                       removed ++;
+                               }
+                       }
+                       if (removed == 0) {
+                               /* Remove explicitly */
                                if (res->key != NULL) {
                                        hash->delete_func (hash->storage, res->key);
                                }
-                               res = g_queue_pop_tail (hash->q);
-                               removed ++;
                        }
                }
-               /* If elements are already removed do not expire extra elements */
-               if (removed != 0 && res != NULL) {
-                       g_queue_push_tail (hash->q, res);
-               }
        }
 
-       res = rspamd_lru_create_node (hash, key, value, now);
+       res = rspamd_lru_create_node (hash, key, value, now, ttl);
        hash->insert_func (hash->storage, key, res);
        g_queue_push_head (hash->q, res);
+       res->link = g_queue_peek_head_link (hash->q);
 }
 
 void

diff --git a/src/hash.h b/src/hash.h
index fc336b6bb7dffdddea46bd8cc7f2656e51be6686..adc9d2823bafffc5a5d6a9b52160c7ec48c9cad9 100644 (file)
--- a/src/hash.h
+++ b/src/hash.h
@@ -50,7 +50,9 @@ typedef struct rspamd_lru_element_s {
        gpointer                  data;
        gpointer                  key;
        time_t                    store_time;
+       guint                     ttl;
        rspamd_lru_hash_t        *hash;
+       GList                    *link;
 } rspamd_lru_element_t;
 
 
@@ -141,7 +143,8 @@ gpointer rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, gpointer key, time_t n
  * @param key key to insert
  * @param value value of key
  */
-void rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now);
+void rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value,
+               time_t now, guint ttl);
 
 /**
  * Remove lru hash

diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c
index 95e07370a220ee2630948646c824c6d65ff2e5e6..b7c5515e56a605678b8c104e0912a8eb335a2fa5 100644 (file)
--- a/src/plugins/dkim_check.c
+++ b/src/plugins/dkim_check.c
@@ -281,14 +281,16 @@ dkim_module_key_handler (rspamd_dkim_key_t *key, gsize keylen, rspamd_dkim_conte
 
        if (key != NULL) {
                /* Add new key to the lru cache */
-               rspamd_lru_hash_insert (dkim_module_ctx->dkim_hash, g_strdup (ctx->dns_key), key, task->tv.tv_sec);
+               rspamd_lru_hash_insert (dkim_module_ctx->dkim_hash, g_strdup (ctx->dns_key),
+                               key, task->tv.tv_sec, key->ttl);
                dkim_module_check (task, ctx, key);
        }
        else {
                /* Insert tempfail symbol */
                msg_info ("cannot get key for domain %s", ctx->dns_key);
                if (err != NULL) {
-                       insert_result (task, dkim_module_ctx->symbol_tempfail, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, err->message)));
+                       insert_result (task, dkim_module_ctx->symbol_tempfail, 1,
+                                       g_list_prepend (NULL, memory_pool_strdup (task->task_pool, err->message)));
 
                }
                else {
@@ -322,7 +324,10 @@ dkim_symbol_callback (struct worker_task *task, void *unused)
 #endif
                        /* Parse signature */
                        msg_debug ("create dkim signature");
-                       /* Check only last signature as there is no way to check embeded signatures after resend or something like this */
+                       /*
+                        * Check only last signature as there is no way to check embeded signatures after
+                        * resend or something like this
+                        */
                        if (dkim_module_ctx->skip_multi) {
                                if (hlist->next != NULL) {
                                        msg_info ("<%s> skip dkim check as it has several dkim signatures", task->message_id);
@@ -337,7 +342,8 @@ dkim_symbol_callback (struct worker_task *task, void *unused)
                        }
                        else {
                                /* Get key */
-                               if (dkim_module_ctx->trusted_only && (dkim_module_ctx->dkim_domains == NULL || g_hash_table_lookup (dkim_module_ctx->dkim_domains, ctx->domain) == NULL)) {
+                               if (dkim_module_ctx->trusted_only && (dkim_module_ctx->dkim_domains == NULL ||
+                                               g_hash_table_lookup (dkim_module_ctx->dkim_domains, ctx->domain) == NULL)) {
                                        msg_debug ("skip dkim check for %s domain", ctx->domain);
                                        return;
                                }

diff --git a/src/plugins/spf.c b/src/plugins/spf.c
index 8b27174d9c4e60455caa563e09964520a096e2fe..99e7f664e9c295c9f6939aa9e38d868f96a0cd10 100644 (file)
--- a/src/plugins/spf.c
+++ b/src/plugins/spf.c
@@ -289,7 +289,7 @@ spf_plugin_callback (struct spf_record *record, struct worker_task *task)
                if ((l = rspamd_lru_hash_lookup (spf_module_ctx->spf_hash, record->sender_domain, task->tv.tv_sec)) == NULL) {
                        l = spf_record_copy (record->addrs);
                        rspamd_lru_hash_insert (spf_module_ctx->spf_hash, g_strdup (record->sender_domain),
-                               l, task->tv.tv_sec);
+                               l, task->tv.tv_sec, record->ttl);
                }
                spf_check_list (l, task);
        }

diff --git a/src/spf.c b/src/spf.c
index f04395da5e9e5139026e468ee998d695da551f88..2d0b5b8eb5738d20bc9c7b33988f8a589b229ef2 100644 (file)
--- a/src/spf.c
+++ b/src/spf.c
@@ -90,7 +90,7 @@ do {                                                                                                          \
 } while (0)                                                                                                    \
 
 static gboolean parse_spf_record (struct worker_task *task, struct spf_record *rec);
-static void start_spf_parse (struct spf_record *rec, gchar *begin);
+static void start_spf_parse (struct spf_record *rec, gchar *begin, guint ttl);
 
 /* Determine spf mech */
 static spf_mech_t
@@ -506,7 +506,7 @@ spf_record_dns_callback (struct rspamd_dns_reply *reply, gpointer arg)
                                                g_list_free (cb->rec->addrs);
                                                cb->rec->addrs = NULL;
                                        }
-                                       start_spf_parse (cb->rec, begin);
+                                       start_spf_parse (cb->rec, begin, elt_data->ttl);
 
                                }
                                break;
@@ -520,7 +520,7 @@ spf_record_dns_callback (struct rspamd_dns_reply *reply, gpointer arg)
                                        tmp = cb->rec->addrs;
                                        cb->rec->addrs = NULL;
                                        cb->rec->in_include = TRUE;
-                                       start_spf_parse (cb->rec, begin);
+                                       start_spf_parse (cb->rec, begin, 0);
                                        cb->rec->in_include = FALSE;
 
 #ifdef SPF_DEBUG
@@ -1287,7 +1287,7 @@ parse_spf_scopes (struct spf_record *rec, gchar **begin)
 }
 
 static void
-start_spf_parse (struct spf_record *rec, gchar *begin)
+start_spf_parse (struct spf_record *rec, gchar *begin, guint ttl)
 {
        /* Skip spaces */
        while (g_ascii_isspace (*begin)) {
@@ -1305,6 +1305,9 @@ start_spf_parse (struct spf_record *rec, gchar *begin)
                        memory_pool_add_destructor (rec->task->task_pool, (pool_destruct_func)g_strfreev, rec->elts);
                        rec->cur_elt = rec->elts[0];
                        while (parse_spf_record (rec->task, rec));
+                       if (ttl != 0) {
+                               rec->ttl = ttl;
+                       }
                }
        }
        else if (g_ascii_strncasecmp (begin, SPF_VER2_STR, sizeof (SPF_VER2_STR) - 1) == 0) {
@@ -1328,6 +1331,9 @@ start_spf_parse (struct spf_record *rec, gchar *begin)
                        memory_pool_add_destructor (rec->task->task_pool, (pool_destruct_func)g_strfreev, rec->elts);
                        rec->cur_elt = rec->elts[0];
                        while (parse_spf_record (rec->task, rec));
+                       if (ttl != 0) {
+                               rec->ttl = ttl;
+                       }
                }
        }
        else {
@@ -1345,7 +1351,7 @@ spf_dns_callback (struct rspamd_dns_reply *reply, gpointer arg)
        rec->requests_inflight --;
        if (reply->code == DNS_RC_NOERROR) {
                LL_FOREACH (reply->entries, elt) {
-                       start_spf_parse (rec, elt->content.txt.data);
+                       start_spf_parse (rec, elt->content.txt.data, elt->ttl);
                }
        }
 

diff --git a/src/spf.h b/src/spf.h
index a9631e733b922dab7e810389a85a2c4602989734..2dd80cc0850ec5ac18a82bfea4896efa9a52cf21 100644 (file)
--- a/src/spf.h
+++ b/src/spf.h
@@ -56,6 +56,8 @@ struct spf_record {
        gint dns_requests;
        gint requests_inflight;
 
+       guint ttl;
+
        GList *addrs;
        gchar *cur_domain;
        gchar *sender;