From 6cbad30ffbc13f9ce49e8a560f5d0fb27f4557d1 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Tue, 18 Feb 2014 14:58:06 +0000 Subject: [PATCH] Rework LRU hash logic. --- src/dkim.c | 1 + src/dkim.h | 1 + src/hash.c | 64 ++++++++++++++++++++++++---------------- src/hash.h | 5 +++- src/plugins/dkim_check.c | 14 ++++++--- src/plugins/spf.c | 2 +- src/spf.c | 16 ++++++---- src/spf.h | 2 ++ 8 files changed, 68 insertions(+), 37 deletions(-) diff --git a/src/dkim.c b/src/dkim.c index d10132694..fe48668aa 100644 --- a/src/dkim.c +++ b/src/dkim.c @@ -767,6 +767,7 @@ rspamd_dkim_dns_cb (struct rspamd_dns_reply *reply, gpointer arg) if (elt->type == DNS_REQUEST_TXT) { key = rspamd_dkim_parse_key (elt->content.txt.data, &keylen, &err); if (key) { + key->ttl = elt->ttl; break; } } diff --git a/src/dkim.h b/src/dkim.h index f4fc21863..e49b9c479 100644 --- a/src/dkim.h +++ b/src/dkim.h @@ -155,6 +155,7 @@ typedef struct rspamd_dkim_key_s { guint8 *keydata; guint keylen; gsize decoded_len; + guint ttl; #ifdef HAVE_OPENSSL RSA *key_rsa; BIO *key_bio; diff --git a/src/hash.c b/src/hash.c index e8cf2d773..e5efc72a7 100644 --- a/src/hash.c +++ b/src/hash.c @@ -313,20 +313,21 @@ rspamd_lru_hash_destroy_node (gpointer v) if (node->hash->value_destroy) { node->hash->value_destroy (node->data); } - + g_queue_delete_link (node->hash->q, node->link); g_slice_free1 (sizeof (rspamd_lru_element_t), node); } static rspamd_lru_element_t* -rspamd_lru_create_node (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now) +rspamd_lru_create_node (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now, guint ttl) { rspamd_lru_element_t *node; node = g_slice_alloc (sizeof (rspamd_lru_element_t)); - node->hash = hash; node->data = value; node->key = key; node->store_time = now; + node->ttl = ttl; + node->hash = hash; return node; } @@ -402,26 +403,24 @@ rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, gpointer key, time_t now) rspamd_lru_element_t *res; if ((res = hash->lookup_func (hash->storage, key)) != NULL) { + if (res->ttl != 0) { + if (now - res->store_time > res->ttl) { + hash->delete_func (hash->storage, key); + return NULL; + } + } if (hash->maxage > 0) { if (now - res->store_time > hash->maxage) { + res = g_queue_peek_tail (hash->q); /* Expire elements from queue tail */ - res = g_queue_pop_tail (hash->q); - while (res != NULL && now - res->store_time > hash->maxage) { hash->delete_func (hash->storage, res->key); - res = g_queue_pop_tail (hash->q); - } - /* Restore last element */ - if (res != NULL) { - g_queue_push_tail (hash->q, res); + res = g_queue_peek_tail (hash->q); } return NULL; } } - } - - if (res) { return res->data; } @@ -434,32 +433,45 @@ rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, gpointer key, time_t now) * @param value value of key */ void -rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now) +rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value, + time_t now, guint ttl) { rspamd_lru_element_t *res; gint removed = 0; - if ((gint)g_queue_get_length (hash->q) >= hash->maxsize) { - /* Expire some elements */ - res = g_queue_pop_tail (hash->q); - if (hash->maxage > 0) { - while (res != NULL && now - res->store_time > hash->maxage) { + if ((res = hash->lookup_func (hash->storage, key)) != NULL) { + hash->delete_func (hash->storage, res->key); + } + else { + if (hash->maxsize > 0 && + (gint)g_queue_get_length (hash->q) >= hash->maxsize) { + /* Expire some elements */ + res = g_queue_peek_tail (hash->q); + if (hash->maxage > 0) { + while (res != NULL && now - res->store_time > hash->maxage) { + if (res->key != NULL) { + hash->delete_func (hash->storage, res->key); + } + else { + break; + } + res = g_queue_peek_tail (hash->q); + removed ++; + } + } + if (removed == 0) { + /* Remove explicitly */ if (res->key != NULL) { hash->delete_func (hash->storage, res->key); } - res = g_queue_pop_tail (hash->q); - removed ++; } } - /* If elements are already removed do not expire extra elements */ - if (removed != 0 && res != NULL) { - g_queue_push_tail (hash->q, res); - } } - res = rspamd_lru_create_node (hash, key, value, now); + res = rspamd_lru_create_node (hash, key, value, now, ttl); hash->insert_func (hash->storage, key, res); g_queue_push_head (hash->q, res); + res->link = g_queue_peek_head_link (hash->q); } void diff --git a/src/hash.h b/src/hash.h index fc336b6bb..adc9d2823 100644 --- a/src/hash.h +++ b/src/hash.h @@ -50,7 +50,9 @@ typedef struct rspamd_lru_element_s { gpointer data; gpointer key; time_t store_time; + guint ttl; rspamd_lru_hash_t *hash; + GList *link; } rspamd_lru_element_t; @@ -141,7 +143,8 @@ gpointer rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, gpointer key, time_t n * @param key key to insert * @param value value of key */ -void rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now); +void rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value, + time_t now, guint ttl); /** * Remove lru hash diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c index 95e07370a..b7c5515e5 100644 --- a/src/plugins/dkim_check.c +++ b/src/plugins/dkim_check.c @@ -281,14 +281,16 @@ dkim_module_key_handler (rspamd_dkim_key_t *key, gsize keylen, rspamd_dkim_conte if (key != NULL) { /* Add new key to the lru cache */ - rspamd_lru_hash_insert (dkim_module_ctx->dkim_hash, g_strdup (ctx->dns_key), key, task->tv.tv_sec); + rspamd_lru_hash_insert (dkim_module_ctx->dkim_hash, g_strdup (ctx->dns_key), + key, task->tv.tv_sec, key->ttl); dkim_module_check (task, ctx, key); } else { /* Insert tempfail symbol */ msg_info ("cannot get key for domain %s", ctx->dns_key); if (err != NULL) { - insert_result (task, dkim_module_ctx->symbol_tempfail, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, err->message))); + insert_result (task, dkim_module_ctx->symbol_tempfail, 1, + g_list_prepend (NULL, memory_pool_strdup (task->task_pool, err->message))); } else { @@ -322,7 +324,10 @@ dkim_symbol_callback (struct worker_task *task, void *unused) #endif /* Parse signature */ msg_debug ("create dkim signature"); - /* Check only last signature as there is no way to check embeded signatures after resend or something like this */ + /* + * Check only last signature as there is no way to check embeded signatures after + * resend or something like this + */ if (dkim_module_ctx->skip_multi) { if (hlist->next != NULL) { msg_info ("<%s> skip dkim check as it has several dkim signatures", task->message_id); @@ -337,7 +342,8 @@ dkim_symbol_callback (struct worker_task *task, void *unused) } else { /* Get key */ - if (dkim_module_ctx->trusted_only && (dkim_module_ctx->dkim_domains == NULL || g_hash_table_lookup (dkim_module_ctx->dkim_domains, ctx->domain) == NULL)) { + if (dkim_module_ctx->trusted_only && (dkim_module_ctx->dkim_domains == NULL || + g_hash_table_lookup (dkim_module_ctx->dkim_domains, ctx->domain) == NULL)) { msg_debug ("skip dkim check for %s domain", ctx->domain); return; } diff --git a/src/plugins/spf.c b/src/plugins/spf.c index 8b27174d9..99e7f664e 100644 --- a/src/plugins/spf.c +++ b/src/plugins/spf.c @@ -289,7 +289,7 @@ spf_plugin_callback (struct spf_record *record, struct worker_task *task) if ((l = rspamd_lru_hash_lookup (spf_module_ctx->spf_hash, record->sender_domain, task->tv.tv_sec)) == NULL) { l = spf_record_copy (record->addrs); rspamd_lru_hash_insert (spf_module_ctx->spf_hash, g_strdup (record->sender_domain), - l, task->tv.tv_sec); + l, task->tv.tv_sec, record->ttl); } spf_check_list (l, task); } diff --git a/src/spf.c b/src/spf.c index f04395da5..2d0b5b8eb 100644 --- a/src/spf.c +++ b/src/spf.c @@ -90,7 +90,7 @@ do { \ } while (0) \ static gboolean parse_spf_record (struct worker_task *task, struct spf_record *rec); -static void start_spf_parse (struct spf_record *rec, gchar *begin); +static void start_spf_parse (struct spf_record *rec, gchar *begin, guint ttl); /* Determine spf mech */ static spf_mech_t @@ -506,7 +506,7 @@ spf_record_dns_callback (struct rspamd_dns_reply *reply, gpointer arg) g_list_free (cb->rec->addrs); cb->rec->addrs = NULL; } - start_spf_parse (cb->rec, begin); + start_spf_parse (cb->rec, begin, elt_data->ttl); } break; @@ -520,7 +520,7 @@ spf_record_dns_callback (struct rspamd_dns_reply *reply, gpointer arg) tmp = cb->rec->addrs; cb->rec->addrs = NULL; cb->rec->in_include = TRUE; - start_spf_parse (cb->rec, begin); + start_spf_parse (cb->rec, begin, 0); cb->rec->in_include = FALSE; #ifdef SPF_DEBUG @@ -1287,7 +1287,7 @@ parse_spf_scopes (struct spf_record *rec, gchar **begin) } static void -start_spf_parse (struct spf_record *rec, gchar *begin) +start_spf_parse (struct spf_record *rec, gchar *begin, guint ttl) { /* Skip spaces */ while (g_ascii_isspace (*begin)) { @@ -1305,6 +1305,9 @@ start_spf_parse (struct spf_record *rec, gchar *begin) memory_pool_add_destructor (rec->task->task_pool, (pool_destruct_func)g_strfreev, rec->elts); rec->cur_elt = rec->elts[0]; while (parse_spf_record (rec->task, rec)); + if (ttl != 0) { + rec->ttl = ttl; + } } } else if (g_ascii_strncasecmp (begin, SPF_VER2_STR, sizeof (SPF_VER2_STR) - 1) == 0) { @@ -1328,6 +1331,9 @@ start_spf_parse (struct spf_record *rec, gchar *begin) memory_pool_add_destructor (rec->task->task_pool, (pool_destruct_func)g_strfreev, rec->elts); rec->cur_elt = rec->elts[0]; while (parse_spf_record (rec->task, rec)); + if (ttl != 0) { + rec->ttl = ttl; + } } } else { @@ -1345,7 +1351,7 @@ spf_dns_callback (struct rspamd_dns_reply *reply, gpointer arg) rec->requests_inflight --; if (reply->code == DNS_RC_NOERROR) { LL_FOREACH (reply->entries, elt) { - start_spf_parse (rec, elt->content.txt.data); + start_spf_parse (rec, elt->content.txt.data, elt->ttl); } } diff --git a/src/spf.h b/src/spf.h index a9631e733..2dd80cc08 100644 --- a/src/spf.h +++ b/src/spf.h @@ -56,6 +56,8 @@ struct spf_record { gint dns_requests; gint requests_inflight; + guint ttl; + GList *addrs; gchar *cur_domain; gchar *sender; -- 2.39.5