Rework LRU hash logic.

8 files changed, 68 insertions, 37 deletions

diff --git a/src/dkim.c b/src/dkim.c
index d10132694..fe48668aa 100644
--- a/src/dkim.c
+++ b/src/dkim.c
@@ -767,6 +767,7 @@ rspamd_dkim_dns_cb (struct rspamd_dns_reply *reply, gpointer arg)
 			if (elt->type == DNS_REQUEST_TXT) {
 				key = rspamd_dkim_parse_key (elt->content.txt.data, &keylen, &err);
 				if (key) {
+					key->ttl = elt->ttl;
 					break;
 				}
 			}
diff --git a/src/dkim.h b/src/dkim.h
index f4fc21863..e49b9c479 100644
--- a/src/dkim.h
+++ b/src/dkim.h
@@ -155,6 +155,7 @@ typedef struct rspamd_dkim_key_s {
 	guint8 *keydata;
 	guint keylen;
 	gsize decoded_len;
+	guint ttl;
 #ifdef HAVE_OPENSSL
 	RSA *key_rsa;
 	BIO *key_bio;
diff --git a/src/hash.c b/src/hash.c
index e8cf2d773..e5efc72a7 100644
--- a/src/hash.c
+++ b/src/hash.c
@@ -313,20 +313,21 @@ rspamd_lru_hash_destroy_node (gpointer v)
 	if (node->hash->value_destroy) {
 		node->hash->value_destroy (node->data);
 	}
-
+	g_queue_delete_link (node->hash->q, node->link);
 	g_slice_free1 (sizeof (rspamd_lru_element_t), node);
 }
 
 static rspamd_lru_element_t*
-rspamd_lru_create_node (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now)
+rspamd_lru_create_node (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now, guint ttl)
 {
 	rspamd_lru_element_t           *node;
 
 	node = g_slice_alloc (sizeof (rspamd_lru_element_t));
-	node->hash = hash;
 	node->data = value;
 	node->key = key;
 	node->store_time = now;
+	node->ttl = ttl;
+	node->hash = hash;
 
 	return node;
 }
@@ -402,26 +403,24 @@ rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, gpointer key, time_t now)
 	rspamd_lru_element_t           *res;
 
 	if ((res = hash->lookup_func (hash->storage, key)) != NULL) {
+		if (res->ttl != 0) {
+			if (now - res->store_time > res->ttl) {
+				hash->delete_func (hash->storage, key);
+				return NULL;
+			}
+		}
 		if (hash->maxage > 0) {
 			if (now - res->store_time > hash->maxage) {
+				res = g_queue_peek_tail (hash->q);
 				/* Expire elements from queue tail */
-				res = g_queue_pop_tail (hash->q);
-
 				while (res != NULL && now - res->store_time > hash->maxage) {
 					hash->delete_func (hash->storage, res->key);
-					res = g_queue_pop_tail (hash->q);
-				}
-				/* Restore last element */
-				if (res != NULL) {
-					g_queue_push_tail (hash->q, res);
+					res = g_queue_peek_tail (hash->q);
 				}
 
 				return NULL;
 			}
 		}
-	}
-
-	if (res) {
 		return res->data;
 	}
 
@@ -434,32 +433,45 @@ rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, gpointer key, time_t now)
  * @param value value of key
  */
 void
-rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now)
+rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value,
+		time_t now, guint ttl)
 {
 	rspamd_lru_element_t           *res;
 	gint                            removed = 0;
 
-	if ((gint)g_queue_get_length (hash->q) >= hash->maxsize) {
-		/* Expire some elements */
-		res = g_queue_pop_tail (hash->q);
-		if (hash->maxage > 0) {
-			while (res != NULL && now - res->store_time > hash->maxage) {
+	if ((res = hash->lookup_func (hash->storage, key)) != NULL) {
+		hash->delete_func (hash->storage, res->key);
+	}
+	else {
+		if (hash->maxsize > 0 &&
+				(gint)g_queue_get_length (hash->q) >= hash->maxsize) {
+			/* Expire some elements */
+			res = g_queue_peek_tail (hash->q);
+			if (hash->maxage > 0) {
+				while (res != NULL && now - res->store_time > hash->maxage) {
+					if (res->key != NULL) {
+						hash->delete_func (hash->storage, res->key);
+					}
+					else {
+						break;
+					}
+					res = g_queue_peek_tail (hash->q);
+					removed ++;
+				}
+			}
+			if (removed == 0) {
+				/* Remove explicitly */
 				if (res->key != NULL) {
 					hash->delete_func (hash->storage, res->key);
 				}
-				res = g_queue_pop_tail (hash->q);
-				removed ++;
 			}
 		}
-		/* If elements are already removed do not expire extra elements */
-		if (removed != 0 && res != NULL) {
-			g_queue_push_tail (hash->q, res);
-		}
 	}
 
-	res = rspamd_lru_create_node (hash, key, value, now);
+	res = rspamd_lru_create_node (hash, key, value, now, ttl);
 	hash->insert_func (hash->storage, key, res);
 	g_queue_push_head (hash->q, res);
+	res->link = g_queue_peek_head_link (hash->q);
 }
 
 void
diff --git a/src/hash.h b/src/hash.h
index fc336b6bb..adc9d2823 100644
--- a/src/hash.h
+++ b/src/hash.h
@@ -50,7 +50,9 @@ typedef struct rspamd_lru_element_s {
 	gpointer                  data;
 	gpointer                  key;
 	time_t                    store_time;
+	guint                     ttl;
 	rspamd_lru_hash_t        *hash;
+	GList                    *link;
 } rspamd_lru_element_t;
 
 
@@ -141,7 +143,8 @@ gpointer rspamd_lru_hash_lookup (rspamd_lru_hash_t *hash, gpointer key, time_t n
  * @param key key to insert
  * @param value value of key
  */
-void rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value, time_t now);
+void rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value,
+		time_t now, guint ttl);
 
 /**
  * Remove lru hash
diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c
index 95e07370a..b7c5515e5 100644
--- a/src/plugins/dkim_check.c
+++ b/src/plugins/dkim_check.c
@@ -281,14 +281,16 @@ dkim_module_key_handler (rspamd_dkim_key_t *key, gsize keylen, rspamd_dkim_conte
 
 	if (key != NULL) {
 		/* Add new key to the lru cache */
-		rspamd_lru_hash_insert (dkim_module_ctx->dkim_hash, g_strdup (ctx->dns_key), key, task->tv.tv_sec);
+		rspamd_lru_hash_insert (dkim_module_ctx->dkim_hash, g_strdup (ctx->dns_key),
+				key, task->tv.tv_sec, key->ttl);
 		dkim_module_check (task, ctx, key);
 	}
 	else {
 		/* Insert tempfail symbol */
 		msg_info ("cannot get key for domain %s", ctx->dns_key);
 		if (err != NULL) {
-			insert_result (task, dkim_module_ctx->symbol_tempfail, 1, g_list_prepend (NULL, memory_pool_strdup (task->task_pool, err->message)));
+			insert_result (task, dkim_module_ctx->symbol_tempfail, 1,
+					g_list_prepend (NULL, memory_pool_strdup (task->task_pool, err->message)));
 
 		}
 		else {
@@ -322,7 +324,10 @@ dkim_symbol_callback (struct worker_task *task, void *unused)
 #endif
 			/* Parse signature */
 			msg_debug ("create dkim signature");
-			/* Check only last signature as there is no way to check embeded signatures after resend or something like this */
+			/*
+			 * Check only last signature as there is no way to check embeded signatures after
+			 * resend or something like this
+			 */
 			if (dkim_module_ctx->skip_multi) {
 				if (hlist->next != NULL) {
 					msg_info ("<%s> skip dkim check as it has several dkim signatures", task->message_id);
@@ -337,7 +342,8 @@ dkim_symbol_callback (struct worker_task *task, void *unused)
 			}
 			else {
 				/* Get key */
-				if (dkim_module_ctx->trusted_only && (dkim_module_ctx->dkim_domains == NULL || g_hash_table_lookup (dkim_module_ctx->dkim_domains, ctx->domain) == NULL)) {
+				if (dkim_module_ctx->trusted_only && (dkim_module_ctx->dkim_domains == NULL ||
+						g_hash_table_lookup (dkim_module_ctx->dkim_domains, ctx->domain) == NULL)) {
 					msg_debug ("skip dkim check for %s domain", ctx->domain);
 					return;
 				}
diff --git a/src/plugins/spf.c b/src/plugins/spf.c
index 8b27174d9..99e7f664e 100644
--- a/src/plugins/spf.c
+++ b/src/plugins/spf.c
@@ -289,7 +289,7 @@ spf_plugin_callback (struct spf_record *record, struct worker_task *task)
 		if ((l = rspamd_lru_hash_lookup (spf_module_ctx->spf_hash, record->sender_domain, task->tv.tv_sec)) == NULL) {
 			l = spf_record_copy (record->addrs);
 			rspamd_lru_hash_insert (spf_module_ctx->spf_hash, g_strdup (record->sender_domain),
-				l, task->tv.tv_sec);
+				l, task->tv.tv_sec, record->ttl);
 		}
 		spf_check_list (l, task);
 	}
diff --git a/src/spf.c b/src/spf.c
index f04395da5..2d0b5b8eb 100644
--- a/src/spf.c
+++ b/src/spf.c
@@ -90,7 +90,7 @@ do {														\
 } while (0)													\
 
 static gboolean parse_spf_record (struct worker_task *task, struct spf_record *rec);
-static void start_spf_parse (struct spf_record *rec, gchar *begin);
+static void start_spf_parse (struct spf_record *rec, gchar *begin, guint ttl);
 
 /* Determine spf mech */
 static spf_mech_t
@@ -506,7 +506,7 @@ spf_record_dns_callback (struct rspamd_dns_reply *reply, gpointer arg)
 						g_list_free (cb->rec->addrs);
 						cb->rec->addrs = NULL;
 					}
-					start_spf_parse (cb->rec, begin);
+					start_spf_parse (cb->rec, begin, elt_data->ttl);
 
 				}
 				break;
@@ -520,7 +520,7 @@ spf_record_dns_callback (struct rspamd_dns_reply *reply, gpointer arg)
 					tmp = cb->rec->addrs;
 					cb->rec->addrs = NULL;
 					cb->rec->in_include = TRUE;
-					start_spf_parse (cb->rec, begin);
+					start_spf_parse (cb->rec, begin, 0);
 					cb->rec->in_include = FALSE;
 
 #ifdef SPF_DEBUG
@@ -1287,7 +1287,7 @@ parse_spf_scopes (struct spf_record *rec, gchar **begin)
 }
 
 static void
-start_spf_parse (struct spf_record *rec, gchar *begin)
+start_spf_parse (struct spf_record *rec, gchar *begin, guint ttl)
 {
 	/* Skip spaces */
 	while (g_ascii_isspace (*begin)) {
@@ -1305,6 +1305,9 @@ start_spf_parse (struct spf_record *rec, gchar *begin)
 			memory_pool_add_destructor (rec->task->task_pool, (pool_destruct_func)g_strfreev, rec->elts);
 			rec->cur_elt = rec->elts[0];
 			while (parse_spf_record (rec->task, rec));
+			if (ttl != 0) {
+				rec->ttl = ttl;
+			}
 		}
 	}
 	else if (g_ascii_strncasecmp (begin, SPF_VER2_STR, sizeof (SPF_VER2_STR) - 1) == 0) {
@@ -1328,6 +1331,9 @@ start_spf_parse (struct spf_record *rec, gchar *begin)
 			memory_pool_add_destructor (rec->task->task_pool, (pool_destruct_func)g_strfreev, rec->elts);
 			rec->cur_elt = rec->elts[0];
 			while (parse_spf_record (rec->task, rec));
+			if (ttl != 0) {
+				rec->ttl = ttl;
+			}
 		}
 	}
 	else {
@@ -1345,7 +1351,7 @@ spf_dns_callback (struct rspamd_dns_reply *reply, gpointer arg)
 	rec->requests_inflight --;
 	if (reply->code == DNS_RC_NOERROR) {
 		LL_FOREACH (reply->entries, elt) {
-			start_spf_parse (rec, elt->content.txt.data);
+			start_spf_parse (rec, elt->content.txt.data, elt->ttl);
 		}
 	}
 
diff --git a/src/spf.h b/src/spf.h
index a9631e733..2dd80cc08 100644
--- a/src/spf.h
+++ b/src/spf.h
@@ -56,6 +56,8 @@ struct spf_record {
 	gint dns_requests;
 	gint requests_inflight;
 
+	guint ttl;
+
 	GList *addrs;
 	gchar *cur_domain;
 	gchar *sender;