From c2a5be027167c3558e1f22955835eead50cecaba Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Thu, 17 Nov 2011 21:05:54 +0300 Subject: [PATCH] * Implement binary safe keys. * Use more fast hashing. --- src/kvstorage.c | 374 ++++++++++++++++++++++++++++++----------- src/kvstorage.h | 38 +++-- src/kvstorage_bdb.c | 32 ++-- src/kvstorage_file.c | 47 ++++-- src/kvstorage_server.c | 14 +- src/kvstorage_server.h | 1 + src/kvstorage_sqlite.c | 40 +++-- 7 files changed, 389 insertions(+), 157 deletions(-) diff --git a/src/kvstorage.c b/src/kvstorage.c index bc9e600bc..4afc826d8 100644 --- a/src/kvstorage.c +++ b/src/kvstorage.c @@ -78,7 +78,7 @@ rspamd_kv_storage_new (gint id, const gchar *name, struct rspamd_kv_cache *cache /** Internal insertion to the kv storage from backend */ gboolean -rspamd_kv_storage_insert_internal (struct rspamd_kv_storage *storage, gpointer key, +rspamd_kv_storage_insert_internal (struct rspamd_kv_storage *storage, gpointer key, guint keylen, gpointer data, gsize len, gint flags, guint expire, struct rspamd_kv_element **pelt) { gint steps = 0; @@ -108,7 +108,7 @@ rspamd_kv_storage_insert_internal (struct rspamd_kv_storage *storage, gpointer k } /* Insert elt to the cache */ - elt = storage->cache->insert_func (storage->cache, key, data, len); + elt = storage->cache->insert_func (storage->cache, key, keylen, data, len); if (elt == NULL) { return FALSE; } @@ -130,7 +130,7 @@ rspamd_kv_storage_insert_internal (struct rspamd_kv_storage *storage, gpointer k /** Insert new element to the kv storage */ gboolean -rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, +rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, guint keylen, gpointer data, gsize len, gint flags, guint expire) { gint steps = 0; @@ -140,14 +140,14 @@ rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, /* Hard limit */ if (storage->max_memory > 0) { - if (len + sizeof (struct rspamd_kv_element) >= storage->max_memory) { + if (len + sizeof (struct rspamd_kv_element) + keylen >= storage->max_memory) { msg_warn ("<%s>: trying to insert value of length %z while limit is %z", storage->name, len, storage->max_memory); return FALSE; } /* Now check limits */ - while (storage->memory + len > storage->max_memory) { + while (storage->memory + len + keylen > storage->max_memory) { if (storage->expire) { storage->expire->step_func (storage->expire, storage, time (NULL), steps); } @@ -178,7 +178,7 @@ rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, } /* First try to search it in cache */ - elt = storage->cache->lookup_func (storage->cache, key); + elt = storage->cache->lookup_func (storage->cache, key, keylen); if (elt) { if (storage->expire) { storage->expire->delete_func (storage->expire, elt); @@ -198,7 +198,7 @@ rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, /* First of all check element for integer */ if (rspamd_strtol (data, len, &longval)) { - elt = storage->cache->insert_func (storage->cache, key, &longval, sizeof (glong)); + elt = storage->cache->insert_func (storage->cache, key, keylen, &longval, sizeof (glong)); if (elt == NULL) { return FALSE; } @@ -207,7 +207,7 @@ rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, } } else { - elt = storage->cache->insert_func (storage->cache, key, data, len); + elt = storage->cache->insert_func (storage->cache, key, keylen, data, len); if (elt == NULL) { return FALSE; } @@ -221,7 +221,7 @@ rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, /* Place to the backend */ if (storage->backend) { - res = storage->backend->insert_func (storage->backend, key, elt); + res = storage->backend->insert_func (storage->backend, key, keylen, elt); } /* Insert to the expire */ @@ -237,7 +237,7 @@ rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, /** Replace an element in the kv storage */ gboolean -rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, struct rspamd_kv_element *elt) +rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, guint keylen, struct rspamd_kv_element *elt) { gboolean res = TRUE; gint steps = 0; @@ -251,7 +251,7 @@ rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, stru } /* Now check limits */ - while (storage->memory + elt->size > storage->max_memory) { + while (storage->memory + ELT_SIZE (elt) > storage->max_memory) { if (storage->expire) { storage->expire->step_func (storage->expire, storage, time (NULL), steps); } @@ -266,11 +266,11 @@ rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, stru } /* Insert elt to the cache */ - res = storage->cache->replace_func (storage->cache, key, elt); + res = storage->cache->replace_func (storage->cache, key, keylen, elt); /* Place to the backend */ if (res && storage->backend) { - res = storage->backend->replace_func (storage->backend, key, elt); + res = storage->backend->replace_func (storage->backend, key, keylen, elt); } return res; @@ -278,20 +278,20 @@ rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, stru /** Increment value in kvstorage */ gboolean -rspamd_kv_storage_increment (struct rspamd_kv_storage *storage, gpointer key, glong *value) +rspamd_kv_storage_increment (struct rspamd_kv_storage *storage, gpointer key, guint keylen, glong *value) { struct rspamd_kv_element *elt = NULL, *belt; glong *lp; /* First try to look at cache */ - elt = storage->cache->lookup_func (storage->cache, key); + elt = storage->cache->lookup_func (storage->cache, key, keylen); if (elt == NULL && storage->backend) { - belt = storage->backend->lookup_func (storage->backend, key); + belt = storage->backend->lookup_func (storage->backend, key, keylen); if (belt) { /* Put this element into cache */ if ((belt->flags & KV_ELT_INTEGER) != 0) { - rspamd_kv_storage_insert_internal (storage, ELT_KEY (belt), ELT_DATA (belt), + rspamd_kv_storage_insert_internal (storage, ELT_KEY (belt), keylen, ELT_DATA (belt), belt->size, belt->flags, belt->expire, &elt); } @@ -305,7 +305,7 @@ rspamd_kv_storage_increment (struct rspamd_kv_storage *storage, gpointer key, gl *lp += *value; *value = *lp; if (storage->backend) { - return storage->backend->replace_func (storage->backend, key, elt); + return storage->backend->replace_func (storage->backend, key, keylen, elt); } else { return TRUE; @@ -317,19 +317,19 @@ rspamd_kv_storage_increment (struct rspamd_kv_storage *storage, gpointer key, gl /** Lookup an element inside kv storage */ struct rspamd_kv_element* -rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage, gpointer key, time_t now) +rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage, gpointer key, guint keylen, time_t now) { struct rspamd_kv_element *elt = NULL, *belt; /* First try to look at cache */ - elt = storage->cache->lookup_func (storage->cache, key); + elt = storage->cache->lookup_func (storage->cache, key, keylen); /* Next look at the backend */ if (elt == NULL && storage->backend) { - belt = storage->backend->lookup_func (storage->backend, key); + belt = storage->backend->lookup_func (storage->backend, key, keylen); if (belt) { /* Put this element into cache */ - rspamd_kv_storage_insert_internal (storage, ELT_KEY (belt), ELT_DATA (belt), + rspamd_kv_storage_insert_internal (storage, ELT_KEY (belt), keylen, ELT_DATA (belt), belt->size, belt->flags, belt->expire, &elt); if ((belt->flags & KV_ELT_DIRTY) == 0) { @@ -350,16 +350,16 @@ rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage, gpointer key, time_ /** Expire an element from kv storage */ struct rspamd_kv_element * -rspamd_kv_storage_delete (struct rspamd_kv_storage *storage, gpointer key) +rspamd_kv_storage_delete (struct rspamd_kv_storage *storage, gpointer key, guint keylen) { struct rspamd_kv_element *elt; /* First delete key from cache */ - elt = storage->cache->delete_func (storage->cache, key); + elt = storage->cache->delete_func (storage->cache, key, keylen); /* Now delete from backend */ if (storage->backend) { - storage->backend->delete_func (storage->backend, key); + storage->backend->delete_func (storage->backend, key, keylen); } /* Notify expire */ if (elt) { @@ -393,7 +393,7 @@ rspamd_kv_storage_destroy (struct rspamd_kv_storage *storage) /** Insert array */ gboolean -rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage, gpointer key, +rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage, gpointer key, guint keylen, guint elt_size, gpointer data, gsize len, gint flags, guint expire) { struct rspamd_kv_element *elt; @@ -405,7 +405,7 @@ rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage, gpointer key, es = arr_data; *es = elt_size; memcpy (arr_data, (gchar *)data + sizeof (guint), len); - if (!rspamd_kv_storage_insert_internal (storage, key, arr_data, len + sizeof (guint), + if (!rspamd_kv_storage_insert_internal (storage, key, keylen, arr_data, len + sizeof (guint), flags, expire, &elt)) { g_slice_free1 (len + sizeof (guint), arr_data); return FALSE; @@ -415,7 +415,7 @@ rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage, gpointer key, g_slice_free1 (len + sizeof (guint), arr_data); /* Place to the backend */ if (storage->backend) { - return storage->backend->insert_func (storage->backend, key, elt); + return storage->backend->insert_func (storage->backend, key, keylen, elt); } return TRUE; @@ -423,14 +423,14 @@ rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage, gpointer key, /** Set element inside array */ gboolean -rspamd_kv_storage_set_array (struct rspamd_kv_storage *storage, gpointer key, +rspamd_kv_storage_set_array (struct rspamd_kv_storage *storage, gpointer key, guint keylen, guint elt_num, gpointer data, gsize len, time_t now) { struct rspamd_kv_element *elt; guint *es; gpointer target; - elt = rspamd_kv_storage_lookup (storage, key, now); + elt = rspamd_kv_storage_lookup (storage, key, keylen, now); if (elt == NULL) { return FALSE; } @@ -452,7 +452,7 @@ rspamd_kv_storage_set_array (struct rspamd_kv_storage *storage, gpointer key, memcpy (target, data, len); /* Place to the backend */ if (storage->backend) { - return storage->backend->replace_func (storage->backend, key, elt); + return storage->backend->replace_func (storage->backend, key, keylen, elt); } return TRUE; @@ -460,14 +460,14 @@ rspamd_kv_storage_set_array (struct rspamd_kv_storage *storage, gpointer key, /** Get element inside array */ gboolean -rspamd_kv_storage_get_array (struct rspamd_kv_storage *storage, gpointer key, +rspamd_kv_storage_get_array (struct rspamd_kv_storage *storage, gpointer key, guint keylen, guint elt_num, gpointer *data, gsize *len, time_t now) { struct rspamd_kv_element *elt; guint *es; gpointer target; - elt = rspamd_kv_storage_lookup (storage, key, now); + elt = rspamd_kv_storage_lookup (storage, key, keylen, now); if (elt == NULL) { return FALSE; } @@ -636,26 +636,28 @@ struct rspamd_kv_hash_cache { * Insert an element inside cache */ static struct rspamd_kv_element* -rspamd_kv_hash_insert (struct rspamd_kv_cache *c, gpointer key, gpointer value, gsize len) +rspamd_kv_hash_insert (struct rspamd_kv_cache *c, gpointer key, guint keylen, gpointer value, gsize len) { struct rspamd_kv_element *elt; struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; - guint keylen; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; - if ((elt = g_hash_table_lookup (cache->hash, key)) == NULL) { - keylen = strlen (key); + if ((elt = g_hash_table_lookup (cache->hash, &search_elt)) == NULL) { elt = g_slice_alloc (sizeof (struct rspamd_kv_element) + len + keylen + 1); elt->age = time (NULL); elt->keylen = keylen; elt->size = len; - elt->hash = rspamd_strcase_hash (key); elt->flags = 0; - memcpy (elt->data, key, keylen + 1); + memcpy (ELT_KEY (elt), key, keylen + 1); memcpy (ELT_DATA (elt), value, len); - g_hash_table_insert (cache->hash, ELT_KEY (elt), elt); + elt->p = &elt->data; + g_hash_table_insert (cache->hash, elt, elt); } else { - g_hash_table_steal (cache->hash, ELT_KEY (elt)); + g_hash_table_steal (cache->hash, elt); if ((elt->flags & KV_ELT_DIRTY) != 0) { elt->flags |= KV_ELT_NEED_FREE; } @@ -663,16 +665,15 @@ rspamd_kv_hash_insert (struct rspamd_kv_cache *c, gpointer key, gpointer value, /* Free it by self */ g_slice_free1 (ELT_SIZE (elt), elt); } - keylen = strlen (key); elt = g_slice_alloc (sizeof (struct rspamd_kv_element) + len + keylen + 1); elt->age = time (NULL); elt->keylen = keylen; elt->size = len; elt->flags = 0; - elt->hash = rspamd_strcase_hash (key); - memcpy (elt->data, key, keylen + 1); + memcpy (ELT_KEY (elt), key, keylen + 1); memcpy (ELT_DATA (elt), value, len); - g_hash_table_insert (cache->hash, ELT_KEY (elt), elt); + elt->p = &elt->data; + g_hash_table_insert (cache->hash, elt, elt); } return elt; @@ -682,24 +683,31 @@ rspamd_kv_hash_insert (struct rspamd_kv_cache *c, gpointer key, gpointer value, * Lookup an item inside hash */ static struct rspamd_kv_element* -rspamd_kv_hash_lookup (struct rspamd_kv_cache *c, gpointer key) +rspamd_kv_hash_lookup (struct rspamd_kv_cache *c, gpointer key, guint keylen) { struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; - return g_hash_table_lookup (cache->hash, key); + return g_hash_table_lookup (cache->hash, &search_elt); } /** * Replace an element inside cache */ static gboolean -rspamd_kv_hash_replace (struct rspamd_kv_cache *c, gpointer key, struct rspamd_kv_element *elt) +rspamd_kv_hash_replace (struct rspamd_kv_cache *c, gpointer key, guint keylen, struct rspamd_kv_element *elt) { struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; - struct rspamd_kv_element *oldelt; + struct rspamd_kv_element *oldelt, search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; - if ((oldelt = g_hash_table_lookup (cache->hash, key)) != NULL) { - g_hash_table_steal (cache->hash, key); + if ((oldelt = g_hash_table_lookup (cache->hash, &search_elt)) != NULL) { + g_hash_table_steal (cache->hash, oldelt); if ((oldelt->flags & KV_ELT_DIRTY) != 0) { oldelt->flags |= KV_ELT_NEED_FREE; @@ -708,7 +716,7 @@ rspamd_kv_hash_replace (struct rspamd_kv_cache *c, gpointer key, struct rspamd_k /* Free it by self */ g_slice_free1 (ELT_SIZE (oldelt), oldelt); } - g_hash_table_insert (cache->hash, ELT_KEY (elt), elt); + g_hash_table_insert (cache->hash, elt, elt); return TRUE; } @@ -719,14 +727,18 @@ rspamd_kv_hash_replace (struct rspamd_kv_cache *c, gpointer key, struct rspamd_k * Delete an element from cache */ static struct rspamd_kv_element * -rspamd_kv_hash_delete (struct rspamd_kv_cache *c, gpointer key) +rspamd_kv_hash_delete (struct rspamd_kv_cache *c, gpointer key, guint keylen) { struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; struct rspamd_kv_element *elt; + struct rspamd_kv_element search_elt; - elt = g_hash_table_lookup (cache->hash, key); + search_elt.keylen = keylen; + search_elt.p = key; + + elt = g_hash_table_lookup (cache->hash, &search_elt); if (elt) { - g_hash_table_steal (cache->hash, key); + g_hash_table_steal (cache->hash, &search_elt); } return elt; } @@ -739,7 +751,7 @@ rspamd_kv_hash_steal (struct rspamd_kv_cache *c, struct rspamd_kv_element *elt) { struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; - g_hash_table_steal (cache->hash, ELT_KEY (elt)); + g_hash_table_steal (cache->hash, elt); } /** @@ -755,13 +767,193 @@ rspamd_kv_hash_destroy (struct rspamd_kv_cache *c) } /** - * Destroy kv_element structure + * Make hash for element */ -static void -kv_elt_destroy_func (gpointer e) +#define rot(x,k) (((x)<<(k)) ^ ((x)>>(32-(k)))) +#define mix(a,b,c) \ +{ \ + a -= c; a ^= rot(c, 4); c += b; \ + b -= a; b ^= rot(a, 6); a += c; \ + c -= b; c ^= rot(b, 8); b += a; \ + a -= c; a ^= rot(c,16); c += b; \ + b -= a; b ^= rot(a,19); a += c; \ + c -= b; c ^= rot(b, 4); b += a; \ +} +#define final(a,b,c) \ +{ \ + c ^= b; c -= rot(b,14); \ + a ^= c; a -= rot(c,11); \ + b ^= a; b -= rot(a,25); \ + c ^= b; c -= rot(b,16); \ + a ^= c; a -= rot(c,4); \ + b ^= a; b -= rot(a,14); \ + c ^= b; c -= rot(b,24); \ +} +/* + * The hash function used here is by Bob Jenkins, 1996: + * + * "By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. + * You may use this code any way you wish, private, educational, + * or commercial. It's free." + * + */ +guint +kv_elt_hash_func (gconstpointer e) { - struct rspamd_kv_element *elt = e; - g_slice_free1 (sizeof (struct rspamd_kv_element) + elt->size, elt); + struct rspamd_kv_element *elt = (struct rspamd_kv_element *)e; + guint32 a, b, c; + union { const void *ptr; size_t i; } u; + guint length; + + /* Set up the internal state */ + length = elt->keylen; + a = b = c = 0xdeadbeef + length; + + u.ptr = elt->p; + if (((u.i & 0x3) == 0)) { + const guint32 *k = (const guint32 *)elt->p; /* read 32-bit chunks */ + + /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ + while (length > 12) { + a += k[0]; + b += k[1]; + c += k[2]; + mix (a,b,c); + length -= 12; + k += 3; + } + + /*----------------------------- handle the last (probably partial) block */ + /* + * "k[2]&0xffffff" actually reads beyond the end of the string, but + * then masks off the part it's not allowed to read. Because the + * string is aligned, the masked-off tail is in the same word as the + * rest of the string. Every machine with memory protection I've seen + * does it on word boundaries, so is OK with this. But VALGRIND will + * still catch it and complain. The masking trick does make the hash + * noticably faster for short strings (like English words). + */ + switch (length) + { + case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; + case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break; + case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break; + case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break; + case 8 : b+=k[1]; a+=k[0]; break; + case 7 : b+=k[1]&0xffffff; a+=k[0]; break; + case 6 : b+=k[1]&0xffff; a+=k[0]; break; + case 5 : b+=k[1]&0xff; a+=k[0]; break; + case 4 : a+=k[0]; break; + case 3 : a+=k[0]&0xffffff; break; + case 2 : a+=k[0]&0xffff; break; + case 1 : a+=k[0]&0xff; break; + case 0 : return c; /* zero length strings require no mixing */ + } + + } else if (((u.i & 0x1) == 0)) { + const guint16 *k = (const guint16 *)elt->p; /* read 16-bit chunks */ + const guint8 *k8; + + /*--------------- all but last block: aligned reads and different mixing */ + while (length > 12) { + a += k[0] + (((guint32)k[1])<<16); + b += k[2] + (((guint32)k[3])<<16); + c += k[4] + (((guint32)k[5])<<16); + mix (a,b,c); + length -= 12; + k += 6; + } + + /*----------------------------- handle the last (probably partial) block */ + k8 = (const guint8 *)k; + switch (length) + { + case 12: c+=k[4]+(((guint32)k[5])<<16); + b+=k[2]+(((guint32)k[3])<<16); + a+=k[0]+(((guint32)k[1])<<16); + break; + case 11: c+=((guint32)k8[10])<<16; /* @fallthrough */ + case 10: c+=k[4]; /* @fallthrough@ */ + b+=k[2]+(((guint32)k[3])<<16); + a+=k[0]+(((guint32)k[1])<<16); + break; + case 9 : c+=k8[8]; /* @fallthrough */ + case 8 : b+=k[2]+(((guint32)k[3])<<16); + a+=k[0]+(((guint32)k[1])<<16); + break; + case 7 : b+=((guint32)k8[6])<<16; /* @fallthrough */ + case 6 : b+=k[2]; + a+=k[0]+(((guint32)k[1])<<16); + break; + case 5 : b+=k8[4]; /* @fallthrough */ + case 4 : a+=k[0]+(((guint32)k[1])<<16); + break; + case 3 : a+=((guint32)k8[2])<<16; /* @fallthrough */ + case 2 : a+=k[0]; + break; + case 1 : a+=k8[0]; + break; + case 0 : return c; /* zero length strings require no mixing */ + } + + } else { /* need to read the key one byte at a time */ + const guint8 *k = elt->p; + + /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ + while (length > 12) + { + a += k[0]; + a += ((guint32)k[1])<<8; + a += ((guint32)k[2])<<16; + a += ((guint32)k[3])<<24; + b += k[4]; + b += ((guint32)k[5])<<8; + b += ((guint32)k[6])<<16; + b += ((guint32)k[7])<<24; + c += k[8]; + c += ((guint32)k[9])<<8; + c += ((guint32)k[10])<<16; + c += ((guint32)k[11])<<24; + mix(a,b,c); + length -= 12; + k += 12; + } + + /*-------------------------------- last block: affect all 32 bits of (c) */ + switch (length) /* all the case statements fall through */ + { + case 12: c+=((guint32)k[11])<<24; + case 11: c+=((guint32)k[10])<<16; + case 10: c+=((guint32)k[9])<<8; + case 9 : c+=k[8]; + case 8 : b+=((guint32)k[7])<<24; + case 7 : b+=((guint32)k[6])<<16; + case 6 : b+=((guint32)k[5])<<8; + case 5 : b+=k[4]; + case 4 : a+=((guint32)k[3])<<24; + case 3 : a+=((guint32)k[2])<<16; + case 2 : a+=((guint32)k[1])<<8; + case 1 : a+=k[0]; + break; + case 0 : return c; /* zero length strings require no mixing */ + } + } + + final (a,b,c); + return c; /* zero length strings require no mixing */ +} + +gboolean +kv_elt_compare_func (gconstpointer e1, gconstpointer e2) +{ + struct rspamd_kv_element *elt1 = (struct rspamd_kv_element *) e1, + *elt2 = (struct rspamd_kv_element *) e2; + + if (elt1->keylen == elt2->keylen) { + return memcmp (elt1->p, elt2->p, elt1->keylen) == 0; + } + + return FALSE; } /** @@ -773,7 +965,7 @@ rspamd_kv_hash_new (void) struct rspamd_kv_hash_cache *new; new = g_slice_alloc (sizeof (struct rspamd_kv_hash_cache)); - new->hash = g_hash_table_new_full (rspamd_strcase_hash, rspamd_strcase_equal, NULL, NULL); + new->hash = g_hash_table_new_full (kv_elt_hash_func, kv_elt_compare_func, NULL, NULL); new->init_func = NULL; new->insert_func = rspamd_kv_hash_insert; new->lookup_func = rspamd_kv_hash_lookup; @@ -803,7 +995,7 @@ struct rspamd_kv_radix_cache { * Validate a key for radix */ static guint32 -rspamd_kv_radix_validate (gpointer key) +rspamd_kv_radix_validate (gpointer key, guint keylen) { struct in_addr addr; @@ -818,12 +1010,11 @@ rspamd_kv_radix_validate (gpointer key) * Insert an element inside cache */ static struct rspamd_kv_element* -rspamd_kv_radix_insert (struct rspamd_kv_cache *c, gpointer key, gpointer value, gsize len) +rspamd_kv_radix_insert (struct rspamd_kv_cache *c, gpointer key, guint keylen, gpointer value, gsize len) { struct rspamd_kv_element *elt; struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; - guint32 rkey = rspamd_kv_radix_validate (key); - guint keylen; + guint32 rkey = rspamd_kv_radix_validate (key, keylen); if (rkey == 0) { return NULL; @@ -831,15 +1022,14 @@ rspamd_kv_radix_insert (struct rspamd_kv_cache *c, gpointer key, gpointer value, elt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey); if ((uintptr_t)elt == RADIX_NO_VALUE) { - keylen = strlen (key); elt = g_slice_alloc (sizeof (struct rspamd_kv_element) + len + keylen + 1); elt->age = time (NULL); elt->keylen = keylen; elt->size = len; - elt->hash = rkey; elt->flags = 0; - memcpy (elt->data, key, keylen + 1); + memcpy (ELT_KEY (elt), key, keylen + 1); memcpy (ELT_DATA (elt), value, len); + elt->p = &elt->data; radix32tree_insert (cache->tree, rkey, 0xffffffff, (uintptr_t)elt); } else { @@ -851,15 +1041,14 @@ rspamd_kv_radix_insert (struct rspamd_kv_cache *c, gpointer key, gpointer value, /* Free it by self */ g_slice_free1 (ELT_SIZE (elt), elt); } - keylen = strlen (key); elt = g_slice_alloc (sizeof (struct rspamd_kv_element) + len + keylen + 1); elt->age = time (NULL); elt->keylen = keylen; elt->size = len; - elt->hash = rkey; elt->flags = 0; - memcpy (elt->data, key, keylen + 1); + memcpy (ELT_KEY (elt), key, keylen + 1); memcpy (ELT_DATA (elt), value, len); + elt->p = &elt->data; radix32tree_insert (cache->tree, rkey, 0xffffffff, (uintptr_t)elt); } @@ -870,10 +1059,10 @@ rspamd_kv_radix_insert (struct rspamd_kv_cache *c, gpointer key, gpointer value, * Lookup an item inside radix */ static struct rspamd_kv_element* -rspamd_kv_radix_lookup (struct rspamd_kv_cache *c, gpointer key) +rspamd_kv_radix_lookup (struct rspamd_kv_cache *c, gpointer key, guint keylen) { struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; - guint32 rkey = rspamd_kv_radix_validate (key); + guint32 rkey = rspamd_kv_radix_validate (key, keylen); struct rspamd_kv_element *elt; elt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey); @@ -888,10 +1077,10 @@ rspamd_kv_radix_lookup (struct rspamd_kv_cache *c, gpointer key) * Replace an element inside cache */ static gboolean -rspamd_kv_radix_replace (struct rspamd_kv_cache *c, gpointer key, struct rspamd_kv_element *elt) +rspamd_kv_radix_replace (struct rspamd_kv_cache *c, gpointer key, guint keylen, struct rspamd_kv_element *elt) { struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; - guint32 rkey = rspamd_kv_radix_validate (key); + guint32 rkey = rspamd_kv_radix_validate (key, keylen); struct rspamd_kv_element *oldelt; oldelt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey); @@ -916,11 +1105,11 @@ rspamd_kv_radix_replace (struct rspamd_kv_cache *c, gpointer key, struct rspamd_ * Delete an element from cache */ static struct rspamd_kv_element * -rspamd_kv_radix_delete (struct rspamd_kv_cache *c, gpointer key) +rspamd_kv_radix_delete (struct rspamd_kv_cache *c, gpointer key, guint keylen) { struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; struct rspamd_kv_element *elt; - guint32 rkey = rspamd_kv_radix_validate (key); + guint32 rkey = rspamd_kv_radix_validate (key, keylen); elt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey); if ((uintptr_t)elt != RADIX_NO_VALUE) { @@ -939,7 +1128,7 @@ static void rspamd_kv_radix_steal (struct rspamd_kv_cache *c, struct rspamd_kv_element *elt) { struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; - guint32 rkey = rspamd_kv_radix_validate (ELT_KEY (elt)); + guint32 rkey = rspamd_kv_radix_validate (ELT_KEY (elt), elt->keylen); radix32tree_delete (cache->tree, rkey, 0xffffffff); @@ -999,12 +1188,12 @@ struct rspamd_kv_judy_cache { * Lookup an item inside judy */ static struct rspamd_kv_element* -rspamd_kv_judy_lookup (struct rspamd_kv_cache *c, gpointer key) +rspamd_kv_judy_lookup (struct rspamd_kv_cache *c, gpointer key, guint keylen) { struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c; struct rspamd_kv_element *elt = NULL, **pelt; - JHSG (pelt, cache->judy, key, strlen (key)); + JHSG (pelt, cache->judy, key, keylen); if (pelt != NULL) { elt = *pelt; } @@ -1015,13 +1204,13 @@ rspamd_kv_judy_lookup (struct rspamd_kv_cache *c, gpointer key) * Delete an element from cache */ static struct rspamd_kv_element * -rspamd_kv_judy_delete (struct rspamd_kv_cache *c, gpointer key) +rspamd_kv_judy_delete (struct rspamd_kv_cache *c, gpointer key, guint keylen) { struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c; struct rspamd_kv_element *elt; gint rc; - elt = rspamd_kv_judy_lookup (c, key); + elt = rspamd_kv_judy_lookup (c, key, keylen); if (elt) { JHSD (rc, cache->judy, ELT_KEY (elt), elt->keylen); } @@ -1044,23 +1233,21 @@ rspamd_kv_judy_steal (struct rspamd_kv_cache *c, struct rspamd_kv_element *elt) * Insert an element inside cache */ static struct rspamd_kv_element* -rspamd_kv_judy_insert (struct rspamd_kv_cache *c, gpointer key, gpointer value, gsize len) +rspamd_kv_judy_insert (struct rspamd_kv_cache *c, gpointer key, guint keylen, gpointer value, gsize len) { struct rspamd_kv_element *elt, **pelt; struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c; - guint keylen; - if ((elt = rspamd_kv_judy_lookup (c, key)) == NULL) { - keylen = strlen (key); + if ((elt = rspamd_kv_judy_lookup (c, key, keylen)) == NULL) { elt = g_slice_alloc (sizeof (struct rspamd_kv_element) + len + keylen + 1); elt->age = time (NULL); elt->keylen = keylen; elt->size = len; - elt->hash = rspamd_strcase_hash (key); elt->flags = 0; - memcpy (elt->data, key, keylen + 1); + memcpy (ELT_KEY (elt), key, keylen); memcpy (ELT_DATA (elt), value, len); JHSI (pelt, cache->judy, ELT_KEY (elt), elt->keylen); + elt->p = &elt->data; *pelt = elt; } else { @@ -1072,15 +1259,14 @@ rspamd_kv_judy_insert (struct rspamd_kv_cache *c, gpointer key, gpointer value, /* Free it by self */ g_slice_free1 (ELT_SIZE (elt), elt); } - keylen = strlen (key); elt = g_slice_alloc0 (sizeof (struct rspamd_kv_element) + len + keylen + 1); elt->age = time (NULL); elt->keylen = keylen; elt->size = len; - elt->hash = rspamd_strcase_hash (key); elt->flags = 0; - memcpy (elt->data, key, keylen + 1); + memcpy (ELT_KEY (elt), key, keylen); memcpy (ELT_DATA (elt), value, len); + elt->p = &elt->data; JHSI (pelt, cache->judy, ELT_KEY (elt), elt->keylen); *pelt = elt; } @@ -1092,12 +1278,12 @@ rspamd_kv_judy_insert (struct rspamd_kv_cache *c, gpointer key, gpointer value, * Replace an element inside cache */ static gboolean -rspamd_kv_judy_replace (struct rspamd_kv_cache *c, gpointer key, struct rspamd_kv_element *elt) +rspamd_kv_judy_replace (struct rspamd_kv_cache *c, gpointer key, guint keylen, struct rspamd_kv_element *elt) { struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c; struct rspamd_kv_element *oldelt, **pelt; - if ((oldelt = rspamd_kv_judy_lookup (c, key)) != NULL) { + if ((oldelt = rspamd_kv_judy_lookup (c, key, keylen)) != NULL) { rspamd_kv_judy_steal (c, elt); if ((oldelt->flags & KV_ELT_DIRTY) != 0) { diff --git a/src/kvstorage.h b/src/kvstorage.h index 586b93e6f..f970a7086 100644 --- a/src/kvstorage.h +++ b/src/kvstorage.h @@ -34,19 +34,19 @@ struct rspamd_kv_element; /* Callbacks for cache */ typedef void (*cache_init)(struct rspamd_kv_cache *cache); -typedef struct rspamd_kv_element* (*cache_insert)(struct rspamd_kv_cache *cache, gpointer key, gpointer value, gsize len); -typedef gboolean (*cache_replace)(struct rspamd_kv_cache *cache, gpointer key, struct rspamd_kv_element *elt); -typedef struct rspamd_kv_element* (*cache_lookup)(struct rspamd_kv_cache *cache, gpointer key); -typedef struct rspamd_kv_element* (*cache_delete)(struct rspamd_kv_cache *cache, gpointer key); +typedef struct rspamd_kv_element* (*cache_insert)(struct rspamd_kv_cache *cache, gpointer key, guint keylen, gpointer value, gsize len); +typedef gboolean (*cache_replace)(struct rspamd_kv_cache *cache, gpointer key, guint keylen, struct rspamd_kv_element *elt); +typedef struct rspamd_kv_element* (*cache_lookup)(struct rspamd_kv_cache *cache, gpointer key, guint keylen); +typedef struct rspamd_kv_element* (*cache_delete)(struct rspamd_kv_cache *cache, gpointer key, guint keylen); typedef void (*cache_steal)(struct rspamd_kv_cache *cache, struct rspamd_kv_element* elt); typedef void (*cache_destroy)(struct rspamd_kv_cache *cache); /* Callbacks for backend */ typedef void (*backend_init)(struct rspamd_kv_backend *backend); -typedef gboolean (*backend_insert)(struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt); -typedef gboolean (*backend_replace)(struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt); -typedef struct rspamd_kv_element* (*backend_lookup)(struct rspamd_kv_backend *backend, gpointer key); -typedef void (*backend_delete)(struct rspamd_kv_backend *backend, gpointer key); +typedef gboolean (*backend_insert)(struct rspamd_kv_backend *backend, gpointer key, guint keylen, struct rspamd_kv_element *elt); +typedef gboolean (*backend_replace)(struct rspamd_kv_backend *backend, gpointer key, guint keylen, struct rspamd_kv_element *elt); +typedef struct rspamd_kv_element* (*backend_lookup)(struct rspamd_kv_backend *backend, gpointer key, guint keylen); +typedef void (*backend_delete)(struct rspamd_kv_backend *backend, gpointer key, guint keylen); typedef gboolean (*backend_sync)(struct rspamd_kv_backend *backend); typedef void (*backend_destroy)(struct rspamd_kv_backend *backend); @@ -81,9 +81,9 @@ struct rspamd_kv_element { enum rspamd_kv_flags flags; /*< element flags */ gsize size; /*< size of element */ TAILQ_ENTRY (rspamd_kv_element) entry; /*< list entry */ - guint32 hash; /*< numeric hash */ guint keylen; /*< length of key */ + gpointer p; /*< pointer to data */ gchar data[1]; /*< expandable data */ }; @@ -138,34 +138,38 @@ struct rspamd_kv_storage *rspamd_kv_storage_new (gint id, const gchar *name, gsize max_elts, gsize max_memory); /** Insert new element to the kv storage */ -gboolean rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, gpointer data, gsize len, gint flags, guint expire); +gboolean rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, guint keylen, gpointer data, gsize len, gint flags, guint expire); /** Replace an element in the kv storage */ -gboolean rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, struct rspamd_kv_element *elt); +gboolean rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, guint keylen, struct rspamd_kv_element *elt); /** Increment value in kvstorage */ -gboolean rspamd_kv_storage_increment (struct rspamd_kv_storage *storage, gpointer key, glong *value); +gboolean rspamd_kv_storage_increment (struct rspamd_kv_storage *storage, gpointer key, guint keylen, glong *value); /** Lookup an element inside kv storage */ -struct rspamd_kv_element* rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage, gpointer key, time_t now); +struct rspamd_kv_element* rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage, gpointer key, guint keylen, time_t now); /** Expire an element from kv storage */ -struct rspamd_kv_element* rspamd_kv_storage_delete (struct rspamd_kv_storage *storage, gpointer key); +struct rspamd_kv_element* rspamd_kv_storage_delete (struct rspamd_kv_storage *storage, gpointer key, guint keylen); /** Destroy kv storage */ void rspamd_kv_storage_destroy (struct rspamd_kv_storage *storage); /** Insert array */ -gboolean rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage, gpointer key, guint elt_size, gpointer data, gsize len, gint flags, guint expire); +gboolean rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage, gpointer key, guint keylen, guint elt_size, gpointer data, gsize len, gint flags, guint expire); /** Set element inside array */ -gboolean rspamd_kv_storage_set_array (struct rspamd_kv_storage *storage, gpointer key, guint elt_num, +gboolean rspamd_kv_storage_set_array (struct rspamd_kv_storage *storage, gpointer key, guint keylen, guint elt_num, gpointer data, gsize len, time_t now); /** Get element inside array */ -gboolean rspamd_kv_storage_get_array (struct rspamd_kv_storage *storage, gpointer key, guint elt_num, +gboolean rspamd_kv_storage_get_array (struct rspamd_kv_storage *storage, gpointer key, guint keylen, guint elt_num, gpointer *data, gsize *len, time_t now); +/* Hash table functions */ +guint kv_elt_hash_func (gconstpointer e); +gboolean kv_elt_compare_func (gconstpointer e1, gconstpointer e2); + /** * LRU expire */ diff --git a/src/kvstorage_bdb.c b/src/kvstorage_bdb.c index 07fcfd8d8..99203285a 100644 --- a/src/kvstorage_bdb.c +++ b/src/kvstorage_bdb.c @@ -182,7 +182,7 @@ err: } static gboolean -rspamd_bdb_insert (struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt) +rspamd_bdb_insert (struct rspamd_kv_backend *backend, gpointer key, guint keylen, struct rspamd_kv_element *elt) { struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; struct bdb_op *op; @@ -197,7 +197,7 @@ rspamd_bdb_insert (struct rspamd_kv_backend *backend, gpointer key, struct rspam elt->flags |= KV_ELT_DIRTY; g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, ELT_KEY (elt), op); + g_hash_table_insert (db->ops_hash, elt, op); if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { return bdb_process_queue (backend); @@ -207,7 +207,7 @@ rspamd_bdb_insert (struct rspamd_kv_backend *backend, gpointer key, struct rspam } static gboolean -rspamd_bdb_replace (struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt) +rspamd_bdb_replace (struct rspamd_kv_backend *backend, gpointer key, guint keylen, struct rspamd_kv_element *elt) { struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; struct bdb_op *op; @@ -222,7 +222,7 @@ rspamd_bdb_replace (struct rspamd_kv_backend *backend, gpointer key, struct rspa elt->flags |= KV_ELT_DIRTY; g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, ELT_KEY (elt), op); + g_hash_table_insert (db->ops_hash, elt, op); if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { return bdb_process_queue (backend); @@ -232,18 +232,22 @@ rspamd_bdb_replace (struct rspamd_kv_backend *backend, gpointer key, struct rspa } static struct rspamd_kv_element* -rspamd_bdb_lookup (struct rspamd_kv_backend *backend, gpointer key) +rspamd_bdb_lookup (struct rspamd_kv_backend *backend, gpointer key, guint keylen) { struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; struct bdb_op *op; DBT db_key, db_data; struct rspamd_kv_element *elt = NULL; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; if (!db->initialized) { return NULL; } /* First search in ops queue */ - if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { if (op->op == BDB_OP_DELETE) { /* To delete, so assume it as not found */ return NULL; @@ -253,7 +257,7 @@ rspamd_bdb_lookup (struct rspamd_kv_backend *backend, gpointer key) memset (&db_key, 0, sizeof(DBT)); memset (&db_data, 0, sizeof(DBT)); - db_key.size = strlen (key); + db_key.size = keylen; db_key.data = key; db_data.flags = DB_DBT_MALLOC; @@ -266,22 +270,26 @@ rspamd_bdb_lookup (struct rspamd_kv_backend *backend, gpointer key) } static void -rspamd_bdb_delete (struct rspamd_kv_backend *backend, gpointer key) +rspamd_bdb_delete (struct rspamd_kv_backend *backend, gpointer key, guint keylen) { struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; struct bdb_op *op; struct rspamd_kv_element *elt; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; if (!db->initialized) { return; } - if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { op->op = BDB_OP_DELETE; return; } - elt = rspamd_bdb_lookup (backend, key); + elt = rspamd_bdb_lookup (backend, key, keylen); if (elt == NULL) { return; } @@ -291,7 +299,7 @@ rspamd_bdb_delete (struct rspamd_kv_backend *backend, gpointer key) elt->flags |= KV_ELT_DIRTY; g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, ELT_KEY(elt), op); + g_hash_table_insert (db->ops_hash, elt, op); if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { bdb_process_queue (backend); @@ -347,7 +355,7 @@ rspamd_kv_bdb_new (const gchar *filename, guint sync_ops) new->filename = g_strdup (filename); new->sync_ops = sync_ops; new->ops_queue = g_queue_new (); - new->ops_hash = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); + new->ops_hash = g_hash_table_new (kv_elt_hash_func, kv_elt_compare_func); /* Init callbacks */ new->init_func = rspamd_bdb_init; diff --git a/src/kvstorage_file.c b/src/kvstorage_file.c index 3460c213f..7a10de11f 100644 --- a/src/kvstorage_file.c +++ b/src/kvstorage_file.c @@ -78,7 +78,8 @@ get_file_name (struct rspamd_file_backend *db, gchar *key, guint keylen, gchar * /* Filebuf is not large enough */ return FALSE; } - *p++ = hexdigits[(*k) % 16]; + t = *k; + *p++ = hexdigits[(t & 0xf) ^ ((t & 0xf0) >> 4)]; *p++ = G_DIR_SEPARATOR; k ++; } @@ -249,16 +250,20 @@ err: } static gboolean -rspamd_file_insert (struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt) +rspamd_file_insert (struct rspamd_kv_backend *backend, gpointer key, guint keylen, struct rspamd_kv_element *elt) { struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; struct file_op *op; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; if (!db->initialized) { return FALSE; } - if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { /* We found another op with such key in this queue */ if (op->op == FILE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != 0) { /* Also clean memory */ @@ -274,7 +279,7 @@ rspamd_file_insert (struct rspamd_kv_backend *backend, gpointer key, struct rspa elt->flags |= KV_ELT_DIRTY; g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, ELT_KEY (elt), op); + g_hash_table_insert (db->ops_hash, elt, op); } if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { @@ -285,15 +290,19 @@ rspamd_file_insert (struct rspamd_kv_backend *backend, gpointer key, struct rspa } static gboolean -rspamd_file_replace (struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt) +rspamd_file_replace (struct rspamd_kv_backend *backend, gpointer key, guint keylen, struct rspamd_kv_element *elt) { struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; struct file_op *op; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; if (!db->initialized) { return FALSE; } - if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { /* We found another op with such key in this queue */ if (op->op == FILE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != 0) { /* Also clean memory */ @@ -309,7 +318,7 @@ rspamd_file_replace (struct rspamd_kv_backend *backend, gpointer key, struct rsp elt->flags |= KV_ELT_DIRTY; g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, ELT_KEY (elt), op); + g_hash_table_insert (db->ops_hash, elt, op); } if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { @@ -320,7 +329,7 @@ rspamd_file_replace (struct rspamd_kv_backend *backend, gpointer key, struct rsp } static struct rspamd_kv_element* -rspamd_file_lookup (struct rspamd_kv_backend *backend, gpointer key) +rspamd_file_lookup (struct rspamd_kv_backend *backend, gpointer key, guint keylen) { struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; struct file_op *op; @@ -328,12 +337,16 @@ rspamd_file_lookup (struct rspamd_kv_backend *backend, gpointer key) gchar filebuf[PATH_MAX]; gint fd, flags; struct stat st; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; if (!db->initialized) { return NULL; } /* First search in ops queue */ - if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { if (op->op == FILE_OP_DELETE) { /* To delete, so assume it as not found */ return NULL; @@ -342,7 +355,7 @@ rspamd_file_lookup (struct rspamd_kv_backend *backend, gpointer key) } /* Get filename */ - if (!get_file_name (db, key, strlen (key), filebuf, sizeof (filebuf))) { + if (!get_file_name (db, key, keylen, filebuf, sizeof (filebuf))) { return NULL; } @@ -378,22 +391,26 @@ rspamd_file_lookup (struct rspamd_kv_backend *backend, gpointer key) } static void -rspamd_file_delete (struct rspamd_kv_backend *backend, gpointer key) +rspamd_file_delete (struct rspamd_kv_backend *backend, gpointer key, guint keylen) { struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; struct file_op *op; struct rspamd_kv_element *elt; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; if (!db->initialized) { return; } - if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { op->op = FILE_OP_DELETE; return; } - elt = rspamd_file_lookup (backend, key); + elt = rspamd_file_lookup (backend, key, keylen); if (elt == NULL) { return; } @@ -403,7 +420,7 @@ rspamd_file_delete (struct rspamd_kv_backend *backend, gpointer key) elt->flags |= KV_ELT_DIRTY; g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, ELT_KEY(elt), op); + g_hash_table_insert (db->ops_hash, elt, op); if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { file_process_queue (backend); @@ -459,7 +476,7 @@ rspamd_kv_file_new (const gchar *filename, guint sync_ops, guint levels) new->sync_ops = sync_ops; new->levels = levels; new->ops_queue = g_queue_new (); - new->ops_hash = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); + new->ops_hash = g_hash_table_new (kv_elt_hash_func, kv_elt_compare_func); /* Init callbacks */ new->init_func = rspamd_file_init; diff --git a/src/kvstorage_server.c b/src/kvstorage_server.c index dd0c18b57..3f072654d 100644 --- a/src/kvstorage_server.c +++ b/src/kvstorage_server.c @@ -307,6 +307,7 @@ parse_kvstorage_line (struct kvstorage_session *session, f_str_t *in) else { session->key = memory_pool_alloc (session->pool, p - c + 1); rspamd_strlcpy (session->key, c, p - c + 1); + session->keylen = p - c; /* Now we must select next state based on command */ if (session->command == KVSTORAGE_CMD_SET || session->command == KVSTORAGE_CMD_INCR || @@ -473,7 +474,7 @@ kvstorage_process_command (struct kvstorage_session *session, gboolean is_redis) } else if (session->command == KVSTORAGE_CMD_GET) { g_static_rw_lock_reader_lock (&session->cf->storage->rwlock); - elt = rspamd_kv_storage_lookup (session->cf->storage, session->key, session->now); + elt = rspamd_kv_storage_lookup (session->cf->storage, session->key, session->keylen, session->now); if (elt == NULL) { g_static_rw_lock_reader_unlock (&session->cf->storage->rwlock); if (!is_redis) { @@ -537,7 +538,7 @@ kvstorage_process_command (struct kvstorage_session *session, gboolean is_redis) } else if (session->command == KVSTORAGE_CMD_DELETE) { g_static_rw_lock_writer_lock (&session->cf->storage->rwlock); - elt = rspamd_kv_storage_delete (session->cf->storage, session->key); + elt = rspamd_kv_storage_delete (session->cf->storage, session->key, session->keylen); if (elt != NULL) { if ((elt->flags & KV_ELT_DIRTY) == 0) { /* Free memory if backend has deleted this element */ @@ -568,7 +569,7 @@ kvstorage_process_command (struct kvstorage_session *session, gboolean is_redis) else if (session->command == KVSTORAGE_CMD_INCR || session->command == KVSTORAGE_CMD_DECR) { g_static_rw_lock_writer_lock (&session->cf->storage->rwlock); longval = session->arg_data.value; - if (!rspamd_kv_storage_increment (session->cf->storage, session->key, &longval)) { + if (!rspamd_kv_storage_increment (session->cf->storage, session->key, session->keylen, &longval)) { g_static_rw_lock_writer_unlock (&session->cf->storage->rwlock); if (!is_redis) { return rspamd_dispatcher_write (session->dispather, ERROR_NOT_FOUND, @@ -815,6 +816,7 @@ kvstorage_read_socket (f_str_t * in, void *arg) if (session->command != KVSTORAGE_CMD_SELECT) { /* This argument is a key for normal command */ session->key = memory_pool_fstrdup (session->pool, in); + session->keylen = in->len; if (session->argnum == session->argc - 1) { session->state = KVSTORAGE_STATE_READ_CMD; rspamd_set_dispatcher_policy (session->dispather, BUFFER_LINE, -1); @@ -841,7 +843,8 @@ kvstorage_read_socket (f_str_t * in, void *arg) session->state = KVSTORAGE_STATE_READ_CMD; rspamd_set_dispatcher_policy (session->dispather, BUFFER_LINE, -1); g_static_rw_lock_writer_lock (&session->cf->storage->rwlock); - if (rspamd_kv_storage_insert (session->cf->storage, session->key, in->begin, in->len, + if (rspamd_kv_storage_insert (session->cf->storage, session->key, session->keylen, + in->begin, in->len, session->flags, session->expire)) { g_static_rw_lock_writer_unlock (&session->cf->storage->rwlock); return rspamd_dispatcher_write (session->dispather, "+OK" CRLF, @@ -868,7 +871,8 @@ kvstorage_read_socket (f_str_t * in, void *arg) session->state = KVSTORAGE_STATE_READ_CMD; rspamd_set_dispatcher_policy (session->dispather, BUFFER_LINE, -1); g_static_rw_lock_writer_lock (&session->cf->storage->rwlock); - if (rspamd_kv_storage_insert (session->cf->storage, session->key, in->begin, in->len, + if (rspamd_kv_storage_insert (session->cf->storage, session->key, session->keylen, + in->begin, in->len, session->flags, session->expire)) { g_static_rw_lock_writer_unlock (&session->cf->storage->rwlock); if (!is_redis) { diff --git a/src/kvstorage_server.h b/src/kvstorage_server.h index ed4ce6eb0..e2d670bad 100644 --- a/src/kvstorage_server.h +++ b/src/kvstorage_server.h @@ -78,6 +78,7 @@ struct kvstorage_session { guint argnum; memory_pool_t *pool; gchar *key; + guint keylen; struct kvstorage_config *cf; struct kvstorage_worker_thread *thr; struct rspamd_kv_element *elt; diff --git a/src/kvstorage_sqlite.c b/src/kvstorage_sqlite.c index 52181e633..7a3a25fcf 100644 --- a/src/kvstorage_sqlite.c +++ b/src/kvstorage_sqlite.c @@ -245,16 +245,20 @@ err: } static gboolean -rspamd_sqlite_insert (struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt) +rspamd_sqlite_insert (struct rspamd_kv_backend *backend, gpointer key, guint keylen, struct rspamd_kv_element *elt) { struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; struct sqlite_op *op; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; if (!db->initialized) { return FALSE; } - if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { /* We found another op with such key in this queue */ if (op->op == SQLITE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != 0) { /* Also clean memory */ @@ -270,7 +274,7 @@ rspamd_sqlite_insert (struct rspamd_kv_backend *backend, gpointer key, struct rs elt->flags |= KV_ELT_DIRTY; g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, ELT_KEY (elt), op); + g_hash_table_insert (db->ops_hash, elt, op); } if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { @@ -281,7 +285,7 @@ rspamd_sqlite_insert (struct rspamd_kv_backend *backend, gpointer key, struct rs } static gboolean -rspamd_sqlite_replace (struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt) +rspamd_sqlite_replace (struct rspamd_kv_backend *backend, gpointer key, guint keylen, struct rspamd_kv_element *elt) { struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; struct sqlite_op *op; @@ -289,7 +293,7 @@ rspamd_sqlite_replace (struct rspamd_kv_backend *backend, gpointer key, struct r if (!db->initialized) { return FALSE; } - if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if ((op = g_hash_table_lookup (db->ops_hash, elt)) != NULL) { /* We found another op with such key in this queue */ if (op->op == SQLITE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != 0) { /* Also clean memory */ @@ -305,7 +309,7 @@ rspamd_sqlite_replace (struct rspamd_kv_backend *backend, gpointer key, struct r elt->flags |= KV_ELT_DIRTY; g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, ELT_KEY (elt), op); + g_hash_table_insert (db->ops_hash, elt, op); } if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { @@ -316,19 +320,23 @@ rspamd_sqlite_replace (struct rspamd_kv_backend *backend, gpointer key, struct r } static struct rspamd_kv_element* -rspamd_sqlite_lookup (struct rspamd_kv_backend *backend, gpointer key) +rspamd_sqlite_lookup (struct rspamd_kv_backend *backend, gpointer key, guint keylen) { struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; struct sqlite_op *op; struct rspamd_kv_element *elt = NULL; gint l; gconstpointer d; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; if (!db->initialized) { return NULL; } /* First search in ops queue */ - if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { if (op->op == SQLITE_OP_DELETE) { /* To delete, so assume it as not found */ return NULL; @@ -336,7 +344,7 @@ rspamd_sqlite_lookup (struct rspamd_kv_backend *backend, gpointer key) return op->elt; } - if (sqlite3_bind_text (db->get_stmt, 1, key, strlen (key), SQLITE_STATIC) == SQLITE_OK) { + if (sqlite3_bind_text (db->get_stmt, 1, key, keylen, SQLITE_STATIC) == SQLITE_OK) { if (sqlite3_step (db->get_stmt) == SQLITE_ROW) { l = sqlite3_column_bytes (db->get_stmt, 0); elt = g_malloc (l); @@ -351,22 +359,26 @@ rspamd_sqlite_lookup (struct rspamd_kv_backend *backend, gpointer key) } static void -rspamd_sqlite_delete (struct rspamd_kv_backend *backend, gpointer key) +rspamd_sqlite_delete (struct rspamd_kv_backend *backend, gpointer key, guint keylen) { struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; struct sqlite_op *op; struct rspamd_kv_element *elt; + struct rspamd_kv_element search_elt; + + search_elt.keylen = keylen; + search_elt.p = key; if (!db->initialized) { return; } - if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { op->op = SQLITE_OP_DELETE; return; } - elt = rspamd_sqlite_lookup (backend, key); + elt = rspamd_sqlite_lookup (backend, key, keylen); if (elt == NULL) { return; } @@ -376,7 +388,7 @@ rspamd_sqlite_delete (struct rspamd_kv_backend *backend, gpointer key) elt->flags |= KV_ELT_DIRTY; g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, ELT_KEY(elt), op); + g_hash_table_insert (db->ops_hash, elt, op); if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { sqlite_process_queue (backend); @@ -437,7 +449,7 @@ rspamd_kv_sqlite_new (const gchar *filename, guint sync_ops) new->filename = g_strdup (filename); new->sync_ops = sync_ops; new->ops_queue = g_queue_new (); - new->ops_hash = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); + new->ops_hash = g_hash_table_new (kv_elt_hash_func, kv_elt_compare_func); /* Init callbacks */ new->init_func = rspamd_sqlite_init; -- 2.39.5