From 62e32c7663cd766721577d9cdaa1e23e7bc4d19e Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Sun, 6 Sep 2015 21:50:37 +0100 Subject: [PATCH] Remove outdated and deprecated code. --- src/kvstorage.c | 1560 ------------------------------------- src/kvstorage.h | 283 ------- src/kvstorage_bdb.c | 392 ---------- src/kvstorage_bdb.h | 39 - src/kvstorage_config.c | 603 -------------- src/kvstorage_config.h | 100 --- src/kvstorage_file.c | 739 ------------------ src/kvstorage_file.h | 39 - src/kvstorage_server.c | 1413 --------------------------------- src/kvstorage_server.h | 98 --- src/kvstorage_sqlite.c | 507 ------------ src/kvstorage_sqlite.h | 39 - src/libserver/cfg_utils.c | 1 - src/main.c | 1 - 14 files changed, 5814 deletions(-) delete mode 100644 src/kvstorage.c delete mode 100644 src/kvstorage.h delete mode 100644 src/kvstorage_bdb.c delete mode 100644 src/kvstorage_bdb.h delete mode 100644 src/kvstorage_config.c delete mode 100644 src/kvstorage_config.h delete mode 100644 src/kvstorage_file.c delete mode 100644 src/kvstorage_file.h delete mode 100644 src/kvstorage_server.c delete mode 100644 src/kvstorage_server.h delete mode 100644 src/kvstorage_sqlite.c delete mode 100644 src/kvstorage_sqlite.h diff --git a/src/kvstorage.c b/src/kvstorage.c deleted file mode 100644 index caa131afe..000000000 --- a/src/kvstorage.c +++ /dev/null @@ -1,1560 +0,0 @@ -/* Copyright (c) 2011, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "kvstorage.h" -#include "main.h" -#include "radix.h" -#ifdef WITH_JUDY -#include -#endif - -#define MAX_EXPIRE_STEPS 10 - -/** Create new kv storage */ -struct rspamd_kv_storage * -rspamd_kv_storage_new (gint id, - const gchar *name, - struct rspamd_kv_cache *cache, - struct rspamd_kv_backend *backend, - struct rspamd_kv_expire *expire, - gsize max_elts, - gsize max_memory, - gboolean no_overwrite) -{ - struct rspamd_kv_storage *new; - - new = g_slice_alloc (sizeof (struct rspamd_kv_storage)); - new->elts = 0; - new->memory = 0; - - new->cache = cache; - new->backend = backend; - new->expire = expire; - - new->max_elts = max_elts; - new->max_memory = max_memory; - - new->id = id; - - new->no_overwrite = no_overwrite; - - if (name != NULL) { - new->name = g_strdup (name); - } - else { - /* Name is absent, use ID as name */ - new->name = g_malloc (sizeof ("18446744073709551616")); - rspamd_snprintf (new->name, sizeof ("18446744073709551616"), "%d", id); - } -#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30)) - g_rw_lock_init (&new->rwlock); -#else - g_static_rw_lock_init (&new->rwlock); -#endif - - - /* Init structures */ - if (new->cache->init_func) { - new->cache->init_func (new->cache); - } - if (new->backend && new->backend->init_func) { - new->backend->init_func (new->backend); - } - if (new->expire && new->expire->init_func) { - new->expire->init_func (new->expire); - } - - return new; -} - -/** Internal insertion to the kv storage from backend */ -gboolean -rspamd_kv_storage_insert_cache (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - gpointer data, - gsize len, - gint flags, - guint expire, - struct rspamd_kv_element **pelt) -{ - gint steps = 0; - struct rspamd_kv_element *elt; - - RW_W_LOCK (&storage->rwlock); - /* Hard limit */ - if (storage->max_memory > 0) { - if (len > storage->max_memory) { - msg_info ( - "<%s>: trying to insert value of length %z while limit is %z", - storage->name, - len, - storage->max_memory); - RW_W_UNLOCK (&storage->rwlock); - return FALSE; - } - - /* Now check limits */ - while (storage->memory + len > storage->max_memory) { - if (storage->expire) { - storage->expire->step_func (storage->expire, storage, time ( - NULL), steps); - } - else { - msg_warn ( - "<%s>: storage is full and no expire function is defined", - storage->name); - } - if (++steps > MAX_EXPIRE_STEPS) { - RW_W_UNLOCK (&storage->rwlock); - msg_warn ("<%s>: cannot expire enough keys in storage", - storage->name); - return FALSE; - } - } - } - - /* Insert elt to the cache */ - - elt = storage->cache->insert_func (storage->cache, key, keylen, data, len); - - - /* Copy data */ - elt->flags = flags; - elt->expire = expire; - - if (pelt != NULL) { - *pelt = elt; - } - - /* Insert to the expire */ - if (storage->expire) { - storage->expire->insert_func (storage->expire, elt); - } - - storage->elts++; - storage->memory += ELT_SIZE (elt); - RW_W_UNLOCK (&storage->rwlock); - - return TRUE; -} - -/** Insert new element to the kv storage */ -gboolean -rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - gpointer data, - gsize len, - gint flags, - guint expire) -{ - gint steps = 0; - struct rspamd_kv_element *elt; - gboolean res = TRUE; - glong longval; - - /* Hard limit */ - RW_W_LOCK (&storage->rwlock); - if (storage->max_memory > 0) { - if (len + sizeof (struct rspamd_kv_element) + keylen >= - storage->max_memory) { - msg_warn ( - "<%s>: trying to insert value of length %z while limit is %z", - storage->name, - len, - storage->max_memory); - RW_W_UNLOCK (&storage->rwlock); - return FALSE; - } - - /* Now check limits */ - while (storage->memory + len + keylen > storage->max_memory) { - if (storage->expire) { - storage->expire->step_func (storage->expire, storage, time ( - NULL), steps); - } - else { - msg_warn ( - "<%s>: storage is full and no expire function is defined", - storage->name); - } - if (++steps > MAX_EXPIRE_STEPS) { - RW_W_UNLOCK (&storage->rwlock); - msg_warn ("<%s>: cannot expire enough keys in storage", - storage->name); - return FALSE; - } - } - } - if (storage->max_elts > 0 && storage->elts > storage->max_elts) { - /* More expire */ - steps = 0; - while (storage->elts > storage->max_elts) { - if (storage->expire) { - storage->expire->step_func (storage->expire, storage, time ( - NULL), steps); - } - else { - msg_warn ( - "<%s>: storage is full and no expire function is defined", - storage->name); - } - if (++steps > MAX_EXPIRE_STEPS) { - RW_W_UNLOCK (&storage->rwlock); - msg_warn ("<%s>: cannot expire enough keys in storage", - storage->name); - return FALSE; - } - } - } - - /* First try to search it in cache */ - - elt = storage->cache->lookup_func (storage->cache, key, keylen); - if (elt) { - if (!storage->no_overwrite) { - /* Remove old elt */ - if (storage->expire) { - storage->expire->delete_func (storage->expire, elt); - } - storage->memory -= ELT_SIZE (elt); - storage->cache->steal_func (storage->cache, elt); - if (elt->flags & KV_ELT_DIRTY) { - /* Element is in backend storage queue */ - elt->flags |= KV_ELT_NEED_FREE; - } - else { - g_slice_free1 (ELT_SIZE (elt), elt); - } - } - else { - /* Just do incref and nothing more */ - if (storage->backend && storage->backend->incref_func) { - if (storage->backend->incref_func (storage->backend, key, - keylen)) { - RW_W_UNLOCK (&storage->rwlock); - return TRUE; - } - else { - RW_W_UNLOCK (&storage->rwlock); - return FALSE; - } - } - } - } - - /* Insert elt to the cache */ - - /* First of all check element for integer */ - if (rspamd_strtol (data, len, &longval)) { - elt = storage->cache->insert_func (storage->cache, - key, - keylen, - &longval, - sizeof (glong)); - if (elt == NULL) { - return FALSE; - } - else { - elt->flags |= KV_ELT_INTEGER; - } - } - else { - elt = storage->cache->insert_func (storage->cache, - key, - keylen, - data, - len); - if (elt == NULL) { - RW_W_UNLOCK (&storage->rwlock); - return FALSE; - } - } - - elt->flags |= flags; - elt->expire = expire; - if (expire == 0) { - elt->flags |= KV_ELT_PERSISTENT; - } - - /* Place to the backend */ - if (storage->backend) { - res = - storage->backend->insert_func (storage->backend, key, keylen, elt); - } - - /* Insert to the expire */ - if (storage->expire) { - storage->expire->insert_func (storage->expire, elt); - } - - storage->elts++; - storage->memory += ELT_SIZE (elt); - RW_W_UNLOCK (&storage->rwlock); - - return res; -} - -/** Replace an element in the kv storage */ -gboolean -rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt) -{ - gboolean res = TRUE; - gint steps = 0; - - /* Hard limit */ - if (storage->max_memory > 0) { - if (elt->size > storage->max_memory) { - msg_info ( - "<%s>: trying to replace value of length %z while limit is %z", - storage->name, - elt->size, - storage->max_memory); - return FALSE; - } - - /* Now check limits */ - while (storage->memory + ELT_SIZE (elt) > storage->max_memory) { - if (storage->expire) { - RW_W_LOCK (&storage->rwlock); - storage->expire->step_func (storage->expire, storage, time ( - NULL), steps); - RW_W_UNLOCK (&storage->rwlock); - } - else { - msg_warn ( - "<%s>: storage is full and no expire function is defined", - storage->name); - } - if (++steps > MAX_EXPIRE_STEPS) { - msg_warn ("<%s>: cannot expire enough keys in storage", - storage->name); - return FALSE; - } - } - } - - RW_W_LOCK (&storage->rwlock); - /* Insert elt to the cache */ - res = storage->cache->replace_func (storage->cache, key, keylen, elt); - - /* Place to the backend */ - if (res && storage->backend) { - res = - storage->backend->replace_func (storage->backend, key, keylen, elt); - } - RW_W_UNLOCK (&storage->rwlock); - - return res; -} - -/** Increment value in kvstorage */ -gboolean -rspamd_kv_storage_increment (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - glong *value) -{ - struct rspamd_kv_element *elt = NULL, *belt; - glong *lp; - - /* First try to look at cache */ - RW_W_LOCK (&storage->rwlock); - elt = storage->cache->lookup_func (storage->cache, key, keylen); - - if (elt == NULL && storage->backend) { - belt = storage->backend->lookup_func (storage->backend, key, keylen); - if (belt) { - /* Put this element into cache */ - if ((belt->flags & KV_ELT_INTEGER) != 0) { - RW_W_UNLOCK (&storage->rwlock); - rspamd_kv_storage_insert_cache (storage, ELT_KEY ( - belt), keylen, ELT_DATA (belt), - belt->size, belt->flags, - belt->expire, &elt); - RW_W_LOCK (&storage->rwlock); - } - if ((belt->flags & KV_ELT_DIRTY) == 0) { - g_free (belt); - } - } - } - if (elt && (elt->flags & KV_ELT_INTEGER) != 0) { - lp = &ELT_LONG (elt); - /* Handle need expire here */ - if (elt->flags & KV_ELT_NEED_EXPIRE) { - *lp = *value; - } - else { - *lp += *value; - *value = *lp; - } - elt->age = time (NULL); - if (storage->backend) { - if (storage->backend->replace_func (storage->backend, key, keylen, - elt)) { - RW_W_UNLOCK (&storage->rwlock); - return TRUE; - } - else { - RW_W_UNLOCK (&storage->rwlock); - return FALSE; - } - } - else { - RW_W_UNLOCK (&storage->rwlock); - return TRUE; - } - } - - RW_W_UNLOCK (&storage->rwlock); - - return FALSE; -} - -/** Lookup an element inside kv storage */ -struct rspamd_kv_element * -rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - time_t now) -{ - struct rspamd_kv_element *elt = NULL, *belt; - - /* First try to look at cache */ - RW_R_LOCK (&storage->rwlock); - elt = storage->cache->lookup_func (storage->cache, key, keylen); - - /* Next look at the backend */ - if (elt == NULL && storage->backend) { - belt = storage->backend->lookup_func (storage->backend, key, keylen); - - if (belt) { - /* Put this element into cache */ - if ((belt->flags & KV_ELT_DIRTY) == 0) { - belt->flags |= KV_ELT_NEED_INSERT; - return belt; - } - else { - elt = belt; - } - } - } - - if (elt && (elt->flags & KV_ELT_PERSISTENT) == 0 && elt->expire > 0) { - /* Check expiration */ - if (now - elt->age > (gint)elt->expire) { - /* Set need expire as we have no write lock here */ - elt->flags |= KV_ELT_NEED_EXPIRE; - elt = NULL; - } - } - - /* RWlock is still locked */ - return elt; -} - -/** Expire an element from kv storage */ -struct rspamd_kv_element * -rspamd_kv_storage_delete (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen) -{ - struct rspamd_kv_element *elt; - - /* First delete key from cache */ - RW_W_LOCK (&storage->rwlock); - elt = storage->cache->delete_func (storage->cache, key, keylen); - - /* Now delete from backend */ - if (storage->backend) { - storage->backend->delete_func (storage->backend, key, keylen); - } - /* Notify expire */ - if (elt) { - if (storage->expire) { - storage->expire->delete_func (storage->expire, elt); - } - storage->elts--; - storage->memory -= elt->size; - if ((elt->flags & KV_ELT_DIRTY) != 0) { - elt->flags |= KV_ELT_NEED_FREE; - } - else { - g_slice_free1 (ELT_SIZE (elt), elt); - } - } - - RW_W_UNLOCK (&storage->rwlock); - - return elt; -} - -/** Destroy kv storage */ -void -rspamd_kv_storage_destroy (struct rspamd_kv_storage *storage) -{ - RW_W_LOCK (&storage->rwlock); - if (storage->backend && storage->backend->destroy_func) { - storage->backend->destroy_func (storage->backend); - } - if (storage->expire && storage->expire->destroy_func) { - storage->expire->destroy_func (storage->expire); - } - if (storage->cache && storage->cache->destroy_func) { - storage->cache->destroy_func (storage->cache); - } - - g_free (storage->name); - - RW_W_UNLOCK (&storage->rwlock); - g_slice_free1 (sizeof (struct rspamd_kv_storage), storage); -} - -/** Insert array */ -gboolean -rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - guint elt_size, - gpointer data, - gsize len, - gint flags, - guint expire) -{ - struct rspamd_kv_element *elt; - guint *es; - gpointer arr_data; - - /* Make temporary copy */ - arr_data = g_slice_alloc (len + sizeof (guint)); - es = arr_data; - *es = elt_size; - memcpy (arr_data, (gchar *)data + sizeof (guint), len); - if (!rspamd_kv_storage_insert_cache (storage, key, keylen, arr_data, len + - sizeof (guint), - flags, expire, &elt)) { - g_slice_free1 (len + sizeof (guint), arr_data); - return FALSE; - } - /* Now set special data of element */ - elt->flags |= KV_ELT_ARRAY; - g_slice_free1 (len + sizeof (guint), arr_data); - /* Place to the backend */ - - if (storage->backend) { - return storage->backend->insert_func (storage->backend, key, keylen, - elt); - } - - return TRUE; -} - -/** Set element inside array */ -gboolean -rspamd_kv_storage_set_array (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - guint elt_num, - gpointer data, - gsize len, - time_t now) -{ - struct rspamd_kv_element *elt; - guint *es; - gpointer target; - - elt = rspamd_kv_storage_lookup (storage, key, keylen, now); - if (elt == NULL) { - return FALSE; - } - - if ((elt->flags & KV_ELT_ARRAY) == 0) { - return FALSE; - } - /* Get element size */ - es = (guint *)ELT_DATA (elt); - if (elt_num > (elt->size - sizeof (guint)) / (*es)) { - /* Invalid index */ - return FALSE; - } - target = (gchar *)ELT_DATA (elt) + sizeof (guint) + (*es) * elt_num; - if (len != *es) { - /* Invalid size */ - return FALSE; - } - memcpy (target, data, len); - /* Place to the backend */ - if (storage->backend) { - return storage->backend->replace_func (storage->backend, - key, - keylen, - elt); - } - - return TRUE; -} - -/** Get element inside array */ -gboolean -rspamd_kv_storage_get_array (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - guint elt_num, - gpointer *data, - gsize *len, - time_t now) -{ - struct rspamd_kv_element *elt; - guint *es; - gpointer target; - - elt = rspamd_kv_storage_lookup (storage, key, keylen, now); - if (elt == NULL) { - return FALSE; - } - - if ((elt->flags & KV_ELT_ARRAY) == 0) { - return FALSE; - } - /* Get element size */ - es = (guint *)ELT_DATA (elt); - if (elt_num > (elt->size - sizeof (guint)) / (*es)) { - /* Invalid index */ - return FALSE; - } - target = ELT_DATA (elt) + sizeof (guint) + (*es) * elt_num; - - *len = *es; - *data = target; - - return TRUE; -} - -/** - * LRU expire functions - */ - -struct rspamd_kv_lru_expire { - expire_init init_func; /*< this callback is called on kv storage initialization */ - expire_insert insert_func; /*< this callback is called when element is inserted */ - expire_step step_func; /*< this callback is used when cache is full */ - expire_delete delete_func; /*< this callback is called when an element is deleted */ - expire_destroy destroy_func; /*< this callback is used for destroying all elements inside expire */ - - TAILQ_HEAD (eltq, rspamd_kv_element) head; -}; - -/** - * Insert an element into expire queue - */ -static void -rspamd_lru_insert (struct rspamd_kv_expire *e, struct rspamd_kv_element *elt) -{ - struct rspamd_kv_lru_expire *expire = (struct rspamd_kv_lru_expire *)e; - - /* Get a proper queue */ - TAILQ_INSERT_TAIL (&expire->head, elt, entry); -} -/** - * Delete an element from expire queue - */ -static void -rspamd_lru_delete (struct rspamd_kv_expire *e, struct rspamd_kv_element *elt) -{ - struct rspamd_kv_lru_expire *expire = (struct rspamd_kv_lru_expire *)e; - - /* Unlink element */ - TAILQ_REMOVE (&expire->head, elt, entry); -} - -/** - * Expire elements - */ -static gboolean -rspamd_lru_expire_step (struct rspamd_kv_expire *e, - struct rspamd_kv_storage *storage, - time_t now, - gboolean forced) -{ - struct rspamd_kv_lru_expire *expire = (struct rspamd_kv_lru_expire *)e; - struct rspamd_kv_element *elt, *oldest_elt = NULL, *temp; - time_t diff; - gboolean res = FALSE; - - elt = TAILQ_FIRST (&expire->head); - if (elt && - (forced || (elt->flags & (KV_ELT_PERSISTENT | KV_ELT_DIRTY)) == 0)) { - diff = elt->expire - (now - elt->age); - if (diff > 0 || (forced && elt->expire == 0)) { - oldest_elt = elt; - } - else { - /* This element is already expired */ - storage->cache->steal_func (storage->cache, elt); - storage->memory -= ELT_SIZE (elt); - storage->elts--; - TAILQ_REMOVE (&expire->head, elt, entry); - /* Free memory */ - if ((elt->flags & (KV_ELT_DIRTY | KV_ELT_NEED_INSERT)) != 0) { - elt->flags |= KV_ELT_NEED_FREE; - } - else { - g_slice_free1 (ELT_SIZE (elt), elt); - } - res = TRUE; - /* Check other elements in this queue */ - TAILQ_FOREACH_SAFE (elt, &expire->head, entry, temp) - { - if ((!forced && - (elt->flags & (KV_ELT_PERSISTENT | KV_ELT_DIRTY)) != 0) || - (gint)elt->expire < (now - elt->age)) { - break; - } - storage->memory -= ELT_SIZE (elt); - storage->elts--; - storage->cache->steal_func (storage->cache, elt); - TAILQ_REMOVE (&expire->head, elt, entry); - /* Free memory */ - if ((elt->flags & (KV_ELT_DIRTY | KV_ELT_NEED_INSERT)) != 0) { - elt->flags |= KV_ELT_NEED_FREE; - } - else { - g_slice_free1 (ELT_SIZE (elt), elt); - } - - } - } - } - - if (!res && oldest_elt != NULL) { - storage->memory -= ELT_SIZE (oldest_elt); - storage->elts--; - storage->cache->steal_func (storage->cache, oldest_elt); - TAILQ_REMOVE (&expire->head, oldest_elt, entry); - /* Free memory */ - if ((oldest_elt->flags & (KV_ELT_DIRTY | KV_ELT_NEED_INSERT)) != 0) { - oldest_elt->flags |= KV_ELT_NEED_FREE; - } - else { - g_slice_free1 (ELT_SIZE (oldest_elt), oldest_elt); - } - } - - return TRUE; -} - -/** - * Destroy LRU expire memory - */ -static void -rspamd_lru_destroy (struct rspamd_kv_expire *e) -{ - struct rspamd_kv_lru_expire *expire = (struct rspamd_kv_lru_expire *)e; - - g_slice_free1 (sizeof (struct rspamd_kv_lru_expire), expire); -} - -/** - * Create new LRU cache - */ -struct rspamd_kv_expire * -rspamd_lru_expire_new (void) -{ - struct rspamd_kv_lru_expire *new; - - new = g_slice_alloc (sizeof (struct rspamd_kv_lru_expire)); - TAILQ_INIT (&new->head); - - /* Set callbacks */ - new->init_func = NULL; - new->insert_func = rspamd_lru_insert; - new->delete_func = rspamd_lru_delete; - new->step_func = rspamd_lru_expire_step; - new->destroy_func = rspamd_lru_destroy; - - return (struct rspamd_kv_expire *)new; -} - -/* - * KV cache hash table - */ -struct rspamd_kv_hash_cache { - cache_init init_func; /*< this callback is called on kv storage initialization */ - cache_insert insert_func; /*< this callback is called when element is inserted */ - cache_replace replace_func; /*< this callback is called when element is replace */ - cache_lookup lookup_func; /*< this callback is used for lookup of element */ - cache_delete delete_func; /*< this callback is called when an element is deleted */ - cache_steal steal_func; /*< this callback is used to replace duplicates in cache */ - cache_destroy destroy_func; /*< this callback is used for destroying all elements inside cache */ - GHashTable *hash; -}; - -/** - * Insert an element inside cache - */ -static struct rspamd_kv_element * -rspamd_kv_hash_insert (struct rspamd_kv_cache *c, - gpointer key, - guint keylen, - gpointer value, - gsize len) -{ - struct rspamd_kv_element *elt; - struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; - struct rspamd_kv_element search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - if ((elt = g_hash_table_lookup (cache->hash, &search_elt)) == NULL) { - elt = g_slice_alloc ( - sizeof (struct rspamd_kv_element) + len + keylen + 1); - elt->age = time (NULL); - elt->keylen = keylen; - elt->size = len; - elt->flags = 0; - memcpy (ELT_KEY (elt), key, keylen + 1); - memcpy (ELT_DATA (elt), value, len); - elt->p = &elt->data; - g_hash_table_insert (cache->hash, elt, elt); - } - else { - g_hash_table_steal (cache->hash, elt); - if ((elt->flags & KV_ELT_DIRTY) != 0) { - elt->flags |= KV_ELT_NEED_FREE; - } - else { - /* Free it by self */ - g_slice_free1 (ELT_SIZE (elt), elt); - } - elt = g_slice_alloc ( - sizeof (struct rspamd_kv_element) + len + keylen + 1); - elt->age = time (NULL); - elt->keylen = keylen; - elt->size = len; - elt->flags = 0; - memcpy (ELT_KEY (elt), key, keylen + 1); - memcpy (ELT_DATA (elt), value, len); - elt->p = &elt->data; - g_hash_table_insert (cache->hash, elt, elt); - } - - return elt; -} - -/** - * Lookup an item inside hash - */ -static struct rspamd_kv_element * -rspamd_kv_hash_lookup (struct rspamd_kv_cache *c, gpointer key, guint keylen) -{ - struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; - struct rspamd_kv_element search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - return g_hash_table_lookup (cache->hash, &search_elt); -} - -/** - * Replace an element inside cache - */ -static gboolean -rspamd_kv_hash_replace (struct rspamd_kv_cache *c, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt) -{ - struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; - struct rspamd_kv_element *oldelt, search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - if ((oldelt = g_hash_table_lookup (cache->hash, &search_elt)) != NULL) { - g_hash_table_steal (cache->hash, oldelt); - - if ((oldelt->flags & KV_ELT_DIRTY) != 0) { - oldelt->flags |= KV_ELT_NEED_FREE; - } - else { - /* Free it by self */ - g_slice_free1 (ELT_SIZE (oldelt), oldelt); - } - g_hash_table_insert (cache->hash, elt, elt); - return TRUE; - } - - return FALSE; -} - -/** - * Delete an element from cache - */ -static struct rspamd_kv_element * -rspamd_kv_hash_delete (struct rspamd_kv_cache *c, gpointer key, guint keylen) -{ - struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; - struct rspamd_kv_element *elt; - struct rspamd_kv_element search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - elt = g_hash_table_lookup (cache->hash, &search_elt); - if (elt) { - g_hash_table_steal (cache->hash, &search_elt); - } - return elt; -} - -/** - * Steal an element from cache - */ -static void -rspamd_kv_hash_steal (struct rspamd_kv_cache *c, struct rspamd_kv_element *elt) -{ - struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; - - g_hash_table_steal (cache->hash, elt); -} - -/** - * Destroy the whole cache - */ - -static void -rspamd_kv_hash_destroy_cb (gpointer key, gpointer value, gpointer unused) -{ - struct rspamd_kv_element *elt = value; - - g_slice_free1 (ELT_SIZE (elt), elt); -} - -static void -rspamd_kv_hash_destroy (struct rspamd_kv_cache *c) -{ - struct rspamd_kv_hash_cache *cache = (struct rspamd_kv_hash_cache *)c; - - g_hash_table_foreach (cache->hash, rspamd_kv_hash_destroy_cb, NULL); - g_hash_table_destroy (cache->hash); - g_slice_free1 (sizeof (struct rspamd_kv_hash_cache), cache); -} - -/** - * Make hash for element - */ -#define rot(x,k) (((x) << (k)) ^ ((x) >> (32 - (k)))) -#define mix(a,b,c) \ - { \ - a -= c; a ^= rot (c, 4); c += b; \ - b -= a; b ^= rot (a, 6); a += c; \ - c -= b; c ^= rot (b, 8); b += a; \ - a -= c; a ^= rot (c,16); c += b; \ - b -= a; b ^= rot (a,19); a += c; \ - c -= b; c ^= rot (b, 4); b += a; \ - } -#define final(a,b,c) \ - { \ - c ^= b; c -= rot (b,14); \ - a ^= c; a -= rot (c,11); \ - b ^= a; b -= rot (a,25); \ - c ^= b; c -= rot (b,16); \ - a ^= c; a -= rot (c,4); \ - b ^= a; b -= rot (a,14); \ - c ^= b; c -= rot (b,24); \ - } -/* - * The hash function used here is by Bob Jenkins, 1996: - * - * "By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. - * You may use this code any way you wish, private, educational, - * or commercial. It's free." - * - */ -guint -kv_elt_hash_func (gconstpointer e) -{ - struct rspamd_kv_element *elt = (struct rspamd_kv_element *)e; - guint32 a, b, c; - union { const void *ptr; size_t i; } u; - guint length; - - /* Set up the internal state */ - length = elt->keylen; - a = b = c = 0xdeadbeef + length; - - u.ptr = elt->p; - if (((u.i & 0x3) == 0)) { - const guint32 *k = (const guint32 *)elt->p; /* read 32-bit chunks */ - - /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ - while (length > 12) { - a += k[0]; - b += k[1]; - c += k[2]; - mix (a,b,c); - length -= 12; - k += 3; - } - - /*----------------------------- handle the last (probably partial) block */ - /* - * "k[2]&0xffffff" actually reads beyond the end of the string, but - * then masks off the part it's not allowed to read. Because the - * string is aligned, the masked-off tail is in the same word as the - * rest of the string. Every machine with memory protection I've seen - * does it on word boundaries, so is OK with this. But VALGRIND will - * still catch it and complain. The masking trick does make the hash - * noticably faster for short strings (like English words). - */ - switch (length) - { - case 12: c += k[2]; b += k[1]; a += k[0]; break; - case 11: c += k[2] & 0xffffff; b += k[1]; a += k[0]; break; - case 10: c += k[2] & 0xffff; b += k[1]; a += k[0]; break; - case 9: c += k[2] & 0xff; b += k[1]; a += k[0]; break; - case 8: b += k[1]; a += k[0]; break; - case 7: b += k[1] & 0xffffff; a += k[0]; break; - case 6: b += k[1] & 0xffff; a += k[0]; break; - case 5: b += k[1] & 0xff; a += k[0]; break; - case 4: a += k[0]; break; - case 3: a += k[0] & 0xffffff; break; - case 2: a += k[0] & 0xffff; break; - case 1: a += k[0] & 0xff; break; - case 0: return c; /* zero length strings require no mixing */ - } - - } else if (((u.i & 0x1) == 0)) { - const guint16 *k = (const guint16 *)elt->p; /* read 16-bit chunks */ - const guint8 *k8; - - /*--------------- all but last block: aligned reads and different mixing */ - while (length > 12) { - a += k[0] + (((guint32)k[1]) << 16); - b += k[2] + (((guint32)k[3]) << 16); - c += k[4] + (((guint32)k[5]) << 16); - mix (a,b,c); - length -= 12; - k += 6; - } - - /*----------------------------- handle the last (probably partial) block */ - k8 = (const guint8 *)k; - switch (length) - { - case 12: c += k[4] + (((guint32)k[5]) << 16); - b += k[2] + (((guint32)k[3]) << 16); - a += k[0] + (((guint32)k[1]) << 16); - break; - case 11: c += ((guint32)k8[10]) << 16; /* @fallthrough */ - case 10: c += k[4]; /* @fallthrough@ */ - b += k[2] + (((guint32)k[3]) << 16); - a += k[0] + (((guint32)k[1]) << 16); - break; - case 9: c += k8[8]; /* @fallthrough */ - case 8: b += k[2] + (((guint32)k[3]) << 16); - a += k[0] + (((guint32)k[1]) << 16); - break; - case 7: b += ((guint32)k8[6]) << 16; /* @fallthrough */ - case 6: b += k[2]; - a += k[0] + (((guint32)k[1]) << 16); - break; - case 5: b += k8[4]; /* @fallthrough */ - case 4: a += k[0] + (((guint32)k[1]) << 16); - break; - case 3: a += ((guint32)k8[2]) << 16; /* @fallthrough */ - case 2: a += k[0]; - break; - case 1: a += k8[0]; - break; - case 0: return c; /* zero length strings require no mixing */ - } - - } else { /* need to read the key one byte at a time */ - const guint8 *k = elt->p; - - /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ - while (length > 12) - { - a += k[0]; - a += ((guint32)k[1]) << 8; - a += ((guint32)k[2]) << 16; - a += ((guint32)k[3]) << 24; - b += k[4]; - b += ((guint32)k[5]) << 8; - b += ((guint32)k[6]) << 16; - b += ((guint32)k[7]) << 24; - c += k[8]; - c += ((guint32)k[9]) << 8; - c += ((guint32)k[10]) << 16; - c += ((guint32)k[11]) << 24; - mix (a,b,c); - length -= 12; - k += 12; - } - - /*-------------------------------- last block: affect all 32 bits of (c) */ - switch (length) /* all the case statements fall through */ - { - case 12: c += ((guint32)k[11]) << 24; - case 11: c += ((guint32)k[10]) << 16; - case 10: c += ((guint32)k[9]) << 8; - case 9: c += k[8]; - case 8: b += ((guint32)k[7]) << 24; - case 7: b += ((guint32)k[6]) << 16; - case 6: b += ((guint32)k[5]) << 8; - case 5: b += k[4]; - case 4: a += ((guint32)k[3]) << 24; - case 3: a += ((guint32)k[2]) << 16; - case 2: a += ((guint32)k[1]) << 8; - case 1: a += k[0]; - break; - case 0: return c; /* zero length strings require no mixing */ - } - } - - final (a,b,c); - return c; /* zero length strings require no mixing */ -} - -gboolean -kv_elt_compare_func (gconstpointer e1, gconstpointer e2) -{ - struct rspamd_kv_element *elt1 = (struct rspamd_kv_element *) e1, - *elt2 = (struct rspamd_kv_element *) e2; - - if (elt1->keylen == elt2->keylen) { - return memcmp (elt1->p, elt2->p, elt1->keylen) == 0; - } - - return FALSE; -} - -/** - * Create new hash kv cache - */ -struct rspamd_kv_cache * -rspamd_kv_hash_new (void) -{ - struct rspamd_kv_hash_cache *new; - - new = g_slice_alloc (sizeof (struct rspamd_kv_hash_cache)); - new->hash = g_hash_table_new_full (kv_elt_hash_func, - kv_elt_compare_func, - NULL, - NULL); - new->init_func = NULL; - new->insert_func = rspamd_kv_hash_insert; - new->lookup_func = rspamd_kv_hash_lookup; - new->replace_func = rspamd_kv_hash_replace; - new->delete_func = rspamd_kv_hash_delete; - new->steal_func = rspamd_kv_hash_steal; - new->destroy_func = rspamd_kv_hash_destroy; - - return (struct rspamd_kv_cache *)new; -} - -/* - * Radix cache hash table - */ -struct rspamd_kv_radix_cache { - cache_init init_func; /*< this callback is called on kv storage initialization */ - cache_insert insert_func; /*< this callback is called when element is inserted */ - cache_replace replace_func; /*< this callback is called when element is replace */ - cache_lookup lookup_func; /*< this callback is used for lookup of element */ - cache_delete delete_func; /*< this callback is called when an element is deleted */ - cache_steal steal_func; /*< this callback is used to replace duplicates in cache */ - cache_destroy destroy_func; /*< this callback is used for destroying all elements inside cache */ - radix_tree_t *tree; -}; - -/** - * Validate a key for radix - */ -static guint32 -rspamd_kv_radix_validate (gpointer key, guint keylen) -{ - struct in_addr addr; - - if (inet_aton (key, &addr) == 0) { - return 0; - } - - return addr.s_addr; -} - -/** - * Insert an element inside cache - */ -static struct rspamd_kv_element * -rspamd_kv_radix_insert (struct rspamd_kv_cache *c, - gpointer key, - guint keylen, - gpointer value, - gsize len) -{ - struct rspamd_kv_element *elt; - struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; - guint32 rkey = rspamd_kv_radix_validate (key, keylen); - - if (rkey == 0) { - return NULL; - } - - elt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey); - if ((uintptr_t)elt == RADIX_NO_VALUE) { - elt = g_slice_alloc ( - sizeof (struct rspamd_kv_element) + len + keylen + 1); - elt->age = time (NULL); - elt->keylen = keylen; - elt->size = len; - elt->flags = 0; - memcpy (ELT_KEY (elt), key, keylen + 1); - memcpy (ELT_DATA (elt), value, len); - elt->p = &elt->data; - radix32tree_insert (cache->tree, rkey, 0xffffffff, (uintptr_t)elt); - } - else { - radix32tree_delete (cache->tree, rkey, 0xffffffff); - if ((elt->flags & KV_ELT_DIRTY) != 0) { - elt->flags |= KV_ELT_NEED_FREE; - } - else { - /* Free it by self */ - g_slice_free1 (ELT_SIZE (elt), elt); - } - elt = g_slice_alloc ( - sizeof (struct rspamd_kv_element) + len + keylen + 1); - elt->age = time (NULL); - elt->keylen = keylen; - elt->size = len; - elt->flags = 0; - memcpy (ELT_KEY (elt), key, keylen + 1); - memcpy (ELT_DATA (elt), value, len); - elt->p = &elt->data; - radix32tree_insert (cache->tree, rkey, 0xffffffff, (uintptr_t)elt); - } - - return elt; -} - -/** - * Lookup an item inside radix - */ -static struct rspamd_kv_element * -rspamd_kv_radix_lookup (struct rspamd_kv_cache *c, gpointer key, guint keylen) -{ - struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; - guint32 rkey = rspamd_kv_radix_validate (key, keylen); - struct rspamd_kv_element *elt; - - elt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey); - if ((uintptr_t)elt == RADIX_NO_VALUE) { - return NULL; - } - - return elt; -} - -/** - * Replace an element inside cache - */ -static gboolean -rspamd_kv_radix_replace (struct rspamd_kv_cache *c, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt) -{ - struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; - guint32 rkey = rspamd_kv_radix_validate (key, keylen); - struct rspamd_kv_element *oldelt; - - oldelt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey); - if ((uintptr_t)oldelt != RADIX_NO_VALUE) { - radix32tree_delete (cache->tree, rkey, 0xffffffff); - - if ((oldelt->flags & KV_ELT_DIRTY) != 0) { - oldelt->flags |= KV_ELT_NEED_FREE; - } - else { - /* Free it by self */ - g_slice_free1 (ELT_SIZE (oldelt), oldelt); - } - radix32tree_insert (cache->tree, rkey, 0xffffffff, (uintptr_t)elt); - return TRUE; - } - - return FALSE; -} - -/** - * Delete an element from cache - */ -static struct rspamd_kv_element * -rspamd_kv_radix_delete (struct rspamd_kv_cache *c, gpointer key, guint keylen) -{ - struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; - struct rspamd_kv_element *elt; - guint32 rkey = rspamd_kv_radix_validate (key, keylen); - - elt = (struct rspamd_kv_element *)radix32tree_find (cache->tree, rkey); - if ((uintptr_t)elt != RADIX_NO_VALUE) { - radix32tree_delete (cache->tree, rkey, 0xffffffff); - } - else { - return NULL; - } - return elt; -} - -/** - * Delete an element from cache - */ -static void -rspamd_kv_radix_steal (struct rspamd_kv_cache *c, struct rspamd_kv_element *elt) -{ - struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; - guint32 rkey = rspamd_kv_radix_validate (ELT_KEY (elt), elt->keylen); - - - radix32tree_delete (cache->tree, rkey, 0xffffffff); -} - -/** - * Destroy the whole cache - */ -static void -rspamd_kv_radix_destroy (struct rspamd_kv_cache *c) -{ - struct rspamd_kv_radix_cache *cache = (struct rspamd_kv_radix_cache *)c; - - radix_tree_free (cache->tree); - g_slice_free1 (sizeof (struct rspamd_kv_radix_cache), cache); -} - -/** - * Create new radix kv cache - */ -struct rspamd_kv_cache * -rspamd_kv_radix_new (void) -{ - struct rspamd_kv_radix_cache *new; - - new = g_slice_alloc (sizeof (struct rspamd_kv_radix_cache)); - new->tree = radix_tree_create (); - new->init_func = NULL; - new->insert_func = rspamd_kv_radix_insert; - new->lookup_func = rspamd_kv_radix_lookup; - new->replace_func = rspamd_kv_radix_replace; - new->delete_func = rspamd_kv_radix_delete; - new->steal_func = rspamd_kv_radix_steal; - new->destroy_func = rspamd_kv_radix_destroy; - - return (struct rspamd_kv_cache *)new; -} - - -#ifdef WITH_JUDY -/* - * KV cache hash table - */ -struct rspamd_kv_judy_cache { - cache_init init_func; /*< this callback is called on kv storage initialization */ - cache_insert insert_func; /*< this callback is called when element is inserted */ - cache_replace replace_func; /*< this callback is called when element is replace */ - cache_lookup lookup_func; /*< this callback is used for lookup of element */ - cache_delete delete_func; /*< this callback is called when an element is deleted */ - cache_steal steal_func; /*< this callback is used to replace duplicates in cache */ - cache_destroy destroy_func; /*< this callback is used for destroying all elements inside cache */ - Pvoid_t judy; -}; - - -/** - * Lookup an item inside judy - */ -static struct rspamd_kv_element * -rspamd_kv_judy_lookup (struct rspamd_kv_cache *c, gpointer key, guint keylen) -{ - struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c; - struct rspamd_kv_element *elt = NULL, **pelt; - - JHSG (pelt, cache->judy, key, keylen); - if (pelt != NULL) { - elt = *pelt; - } - return elt; -} - -/** - * Delete an element from cache - */ -static struct rspamd_kv_element * -rspamd_kv_judy_delete (struct rspamd_kv_cache *c, gpointer key, guint keylen) -{ - struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c; - struct rspamd_kv_element *elt; - gint rc; - - elt = rspamd_kv_judy_lookup (c, key, keylen); - if (elt) { - JHSD (rc, cache->judy, ELT_KEY (elt), elt->keylen); - } - return elt; -} - -/** - * Steal an element from cache - */ -static void -rspamd_kv_judy_steal (struct rspamd_kv_cache *c, struct rspamd_kv_element *elt) -{ - struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c; - gint rc; - - JHSD (rc, cache->judy, ELT_KEY (elt), elt->keylen); -} - -/** - * Insert an element inside cache - */ -static struct rspamd_kv_element * -rspamd_kv_judy_insert (struct rspamd_kv_cache *c, - gpointer key, - guint keylen, - gpointer value, - gsize len) -{ - struct rspamd_kv_element *elt, **pelt; - struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c; - - if ((elt = rspamd_kv_judy_lookup (c, key, keylen)) == NULL) { - elt = g_slice_alloc ( - sizeof (struct rspamd_kv_element) + len + keylen + 1); - elt->age = time (NULL); - elt->keylen = keylen; - elt->size = len; - elt->flags = 0; - memcpy (ELT_KEY (elt), key, keylen); - memcpy (ELT_DATA (elt), value, len); - JHSI (pelt, cache->judy, ELT_KEY (elt), elt->keylen); - elt->p = &elt->data; - *pelt = elt; - } - else { - rspamd_kv_judy_steal (c, elt); - if ((elt->flags & KV_ELT_DIRTY) != 0) { - elt->flags |= KV_ELT_NEED_FREE; - } - else { - /* Free it by self */ - g_slice_free1 (ELT_SIZE (elt), elt); - } - elt = g_slice_alloc0 ( - sizeof (struct rspamd_kv_element) + len + keylen + 1); - elt->age = time (NULL); - elt->keylen = keylen; - elt->size = len; - elt->flags = 0; - memcpy (ELT_KEY (elt), key, keylen); - memcpy (ELT_DATA (elt), value, len); - elt->p = &elt->data; - JHSI (pelt, cache->judy, ELT_KEY (elt), elt->keylen); - *pelt = elt; - } - - return elt; -} - -/** - * Replace an element inside cache - */ -static gboolean -rspamd_kv_judy_replace (struct rspamd_kv_cache *c, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt) -{ - struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c; - struct rspamd_kv_element *oldelt, **pelt; - - if ((oldelt = rspamd_kv_judy_lookup (c, key, keylen)) != NULL) { - rspamd_kv_judy_steal (c, elt); - - if ((oldelt->flags & KV_ELT_DIRTY) != 0) { - oldelt->flags |= KV_ELT_NEED_FREE; - } - else { - /* Free it by self */ - g_slice_free1 (ELT_SIZE (oldelt), oldelt); - } - JHSI (pelt, cache->judy, ELT_KEY (elt), elt->keylen); - *pelt = elt; - return TRUE; - } - - return FALSE; -} - -/** - * Destroy the whole cache - */ -static void -rspamd_kv_judy_destroy (struct rspamd_kv_cache *c) -{ - struct rspamd_kv_judy_cache *cache = (struct rspamd_kv_judy_cache *)c; - glong bytes; - - JHSFA (bytes, cache->judy); - g_slice_free1 (sizeof (struct rspamd_kv_judy_cache), cache); -} - -/** - * Judy tree - */ -struct rspamd_kv_cache * -rspamd_kv_judy_new (void) -{ - struct rspamd_kv_judy_cache *new; - - new = g_slice_alloc (sizeof (struct rspamd_kv_judy_cache)); - new->judy = NULL; - new->init_func = NULL; - new->insert_func = rspamd_kv_judy_insert; - new->lookup_func = rspamd_kv_judy_lookup; - new->replace_func = rspamd_kv_judy_replace; - new->delete_func = rspamd_kv_judy_delete; - new->steal_func = rspamd_kv_judy_steal; - new->destroy_func = rspamd_kv_judy_destroy; - - return (struct rspamd_kv_cache *)new; -} -#endif diff --git a/src/kvstorage.h b/src/kvstorage.h deleted file mode 100644 index 91db10df3..000000000 --- a/src/kvstorage.h +++ /dev/null @@ -1,283 +0,0 @@ -/* Copyright (c) 2011, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef KVSTORAGE_H_ -#define KVSTORAGE_H_ - -#include "config.h" - -struct rspamd_kv_cache; -struct rspamd_kv_backend; -struct rspamd_kv_storage; -struct rspamd_kv_expire; -struct rspamd_kv_element; - -/* Locking definitions */ -#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30)) -#define RW_R_LOCK g_rw_lock_reader_lock -#define RW_R_UNLOCK g_rw_lock_reader_unlock -#define RW_W_LOCK g_rw_lock_writer_lock -#define RW_W_UNLOCK g_rw_lock_writer_unlock -#else -#define RW_R_LOCK g_static_rw_lock_reader_lock -#define RW_R_UNLOCK g_static_rw_lock_reader_unlock -#define RW_W_LOCK g_static_rw_lock_writer_lock -#define RW_W_UNLOCK g_static_rw_lock_writer_unlock -#endif - -/* Callbacks for cache */ -typedef void (*cache_init)(struct rspamd_kv_cache *cache); -typedef struct rspamd_kv_element * (*cache_insert)(struct rspamd_kv_cache *cache, - gpointer key, guint keylen, gpointer value, gsize len); -typedef gboolean (*cache_replace)(struct rspamd_kv_cache *cache, gpointer key, - guint keylen, - struct rspamd_kv_element *elt); -typedef struct rspamd_kv_element * (*cache_lookup)(struct rspamd_kv_cache *cache, - gpointer key, guint keylen); -typedef struct rspamd_kv_element * (*cache_delete)(struct rspamd_kv_cache *cache, - gpointer key, guint keylen); -typedef void (*cache_steal)(struct rspamd_kv_cache *cache, - struct rspamd_kv_element * elt); -typedef void (*cache_destroy)(struct rspamd_kv_cache *cache); - -/* Callbacks for backend */ -typedef void (*backend_init)(struct rspamd_kv_backend *backend); -typedef gboolean (*backend_insert)(struct rspamd_kv_backend *backend, - gpointer key, guint keylen, - struct rspamd_kv_element *elt); -typedef gboolean (*backend_replace)(struct rspamd_kv_backend *backend, - gpointer key, guint keylen, - struct rspamd_kv_element *elt); -typedef struct rspamd_kv_element * (*backend_lookup)(struct rspamd_kv_backend * - backend, gpointer key, - guint keylen); -typedef void (*backend_delete)(struct rspamd_kv_backend *backend, gpointer key, - guint keylen); -typedef gboolean (*backend_sync)(struct rspamd_kv_backend *backend); -typedef gboolean (*backend_incref)(struct rspamd_kv_backend *backend, - gpointer key, guint keylen); -typedef void (*backend_destroy)(struct rspamd_kv_backend *backend); - -/* Callbacks for expire */ -typedef void (*expire_init)(struct rspamd_kv_expire *expire); -typedef void (*expire_insert)(struct rspamd_kv_expire *expire, - struct rspamd_kv_element *elt); -typedef void (*expire_delete)(struct rspamd_kv_expire *expire, - struct rspamd_kv_element *elt); -typedef gboolean (*expire_step)(struct rspamd_kv_expire *expire, - struct rspamd_kv_storage *storage, - time_t now, gboolean forced); -typedef void (*expire_destroy)(struct rspamd_kv_expire *expire); - - -/* Flags of element */ -enum rspamd_kv_flags { - KV_ELT_ARRAY = 1 << 0, - KV_ELT_PERSISTENT = 1 << 1, - KV_ELT_DIRTY = 1 << 2, - KV_ELT_OUSTED = 1 << 3, - KV_ELT_NEED_FREE = 1 << 4, - KV_ELT_INTEGER = 1 << 5, - KV_ELT_NEED_INSERT = 1 << 6, - KV_ELT_NEED_EXPIRE = 1 << 7 -}; - -#define ELT_DATA(elt) (gchar *)(elt)->data + (elt)->keylen + 1 -#define ELT_LONG(elt) *((glong *)((elt)->data + (elt)->keylen + 1)) -#define ELT_KEY(elt) (gchar *)(elt)->data -#define ELT_SIZE(elt) elt->size + sizeof(struct rspamd_kv_element) + \ - elt->keylen + 1 - -/* Common structures description */ - -struct rspamd_kv_element { - time_t age; /*< age of element */ - guint32 expire; /*< expire of element */ - gint flags; /*< element flags */ - gsize size; /*< size of element */ - TAILQ_ENTRY (rspamd_kv_element) entry; /*< list entry */ - guint keylen; /*< length of key */ - - gpointer p; /*< pointer to data */ - gchar data[1]; /*< expandable data */ -}; - -struct rspamd_kv_cache { - cache_init init_func; /*< this callback is called on kv storage initialization */ - cache_insert insert_func; /*< this callback is called when element is inserted */ - cache_replace replace_func; /*< this callback is called when element is replace */ - cache_lookup lookup_func; /*< this callback is used for lookup of element */ - cache_delete delete_func; /*< this callback is called when an element is deleted */ - cache_steal steal_func; /*< this callback is used to replace duplicates in cache */ - cache_destroy destroy_func; /*< this callback is used for destroying all elements inside cache */ -}; -struct rspamd_kv_backend { - backend_init init_func; /*< this callback is called on kv storage initialization */ - backend_insert insert_func; /*< this callback is called when element is inserted */ - backend_replace replace_func; /*< this callback is called when element is replaced */ - backend_lookup lookup_func; /*< this callback is used for lookup of element */ - backend_delete delete_func; /*< this callback is called when an element is deleted */ - backend_sync sync_func; /*< this callback is called when backend need to be synced */ - backend_incref incref_func; /*< this callback is called when element must be ref'd */ - backend_destroy destroy_func; /*< this callback is used for destroying all elements inside backend */ -}; -struct rspamd_kv_expire { - expire_init init_func; /*< this callback is called on kv storage initialization */ - expire_insert insert_func; /*< this callback is called when element is inserted */ - expire_step step_func; /*< this callback is used when cache is full */ - expire_delete delete_func; /*< this callback is called when an element is deleted */ - expire_destroy destroy_func; /*< this callback is used for destroying all elements inside expire */ -}; - -/* Main kv storage structure */ - -struct rspamd_kv_storage { - struct rspamd_kv_cache *cache; - struct rspamd_kv_backend *backend; - struct rspamd_kv_expire *expire; - - gsize elts; /*< current elements count in a storage */ - gsize max_elts; /*< maximum number of elements in a storage */ - - gsize memory; /*< memory eaten */ - gsize max_memory; /*< memory limit */ - - gint id; /* char ID */ - gchar *name; /* numeric ID */ - - gboolean no_overwrite; /* do not overwrite data with the same keys */ -#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30)) - GRWLock rwlock; /* rwlock in new glib */ -#else - GStaticRWLock rwlock; /* rwlock for threaded access */ -#endif -}; - -/** Create new kv storage */ -struct rspamd_kv_storage * rspamd_kv_storage_new (gint id, const gchar *name, - struct rspamd_kv_cache *cache, struct rspamd_kv_backend *backend, - struct rspamd_kv_expire *expire, - gsize max_elts, gsize max_memory, gboolean no_overwrite); - -/** Insert new element to the kv storage */ -gboolean rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - gpointer data, - gsize len, - gint flags, - guint expire); - -/** Insert element only in cache */ -gboolean rspamd_kv_storage_insert_cache (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - gpointer data, - gsize len, - gint flags, - guint expire, - struct rspamd_kv_element **pelt); - -/** Replace an element in the kv storage */ -gboolean rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt); - -/** Increment value in kvstorage */ -gboolean rspamd_kv_storage_increment (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - glong *value); - -/** Lookup an element inside kv storage */ -struct rspamd_kv_element * rspamd_kv_storage_lookup ( - struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - time_t now); - -/** Expire an element from kv storage */ -struct rspamd_kv_element * rspamd_kv_storage_delete ( - struct rspamd_kv_storage *storage, - gpointer key, - guint keylen); - -/** Destroy kv storage */ -void rspamd_kv_storage_destroy (struct rspamd_kv_storage *storage); - -/** Insert array */ -gboolean rspamd_kv_storage_insert_array (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - guint elt_size, - gpointer data, - gsize len, - gint flags, - guint expire); - -/** Set element inside array */ -gboolean rspamd_kv_storage_set_array (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - guint elt_num, - gpointer data, - gsize len, - time_t now); - -/** Get element inside array */ -gboolean rspamd_kv_storage_get_array (struct rspamd_kv_storage *storage, - gpointer key, - guint keylen, - guint elt_num, - gpointer *data, - gsize *len, - time_t now); - -/* Hash table functions */ -guint kv_elt_hash_func (gconstpointer e); -gboolean kv_elt_compare_func (gconstpointer e1, gconstpointer e2); - -/** - * LRU expire - */ -struct rspamd_kv_expire * rspamd_lru_expire_new (void); - -/** - * Ordinary hash - */ -struct rspamd_kv_cache * rspamd_kv_hash_new (void); - -/** - * Radix tree - */ -struct rspamd_kv_cache * rspamd_kv_radix_new (void); - -#ifdef WITH_JUDY -/** - * Judy tree - */ -struct rspamd_kv_cache * rspamd_kv_judy_new (void); -#endif - -#endif /* KVSTORAGE_H_ */ diff --git a/src/kvstorage_bdb.c b/src/kvstorage_bdb.c deleted file mode 100644 index df2a5ba19..000000000 --- a/src/kvstorage_bdb.c +++ /dev/null @@ -1,392 +0,0 @@ -/* Copyright (c) 2010, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - -#include "config.h" -#include "kvstorage.h" -#include "kvstorage_bdb.h" -#include "util.h" -#include "main.h" -#include - -struct bdb_op { - struct rspamd_kv_element *elt; - enum { - BDB_OP_INSERT, - BDB_OP_DELETE, - BDB_OP_REPLACE - } op; -}; - -/* Main bdb structure */ -struct rspamd_bdb_backend { - backend_init init_func; /*< this callback is called on kv storage initialization */ - backend_insert insert_func; /*< this callback is called when element is inserted */ - backend_replace replace_func; /*< this callback is called when element is replaced */ - backend_lookup lookup_func; /*< this callback is used for lookup of element */ - backend_delete delete_func; /*< this callback is called when an element is deleted */ - backend_sync sync_func; /*< this callback is called when backend need to be synced */ - backend_destroy destroy_func; /*< this callback is used for destroying all elements inside backend */ - DB_ENV *envp; /*< db environment */ - DB *dbp; /*< db pointer */ - gchar *filename; - gchar *dirname; - guint sync_ops; - GQueue *ops_queue; - GHashTable *ops_hash; - gboolean initialized; -}; - -/* Process single bdb operation */ -static gboolean -bdb_process_single_op (struct rspamd_bdb_backend *db, - DB_TXN *txn, - struct bdb_op *op) -{ - DBT db_key, db_data; - - memset (&db_key, 0, sizeof(DBT)); - memset (&db_data, 0, sizeof(DBT)); - - db_key.size = op->elt->keylen; - db_key.data = ELT_KEY (op->elt); - db_data.size = op->elt->size + sizeof (struct rspamd_kv_element) + - op->elt->keylen + 1; - db_data.data = op->elt; - - switch (op->op) { - case BDB_OP_INSERT: - case BDB_OP_REPLACE: - db_data.flags = DB_DBT_USERMEM; - if (db->dbp->put (db->dbp, NULL, &db_key, &db_data, 0) != 0) { - return FALSE; - } - break; - case BDB_OP_DELETE: - db_data.flags = DB_DBT_USERMEM; - /* Set cursor */ - if (db->dbp->del (db->dbp, NULL, &db_key, 0) != 0) { - return FALSE; - } - break; - } - - op->elt->flags &= ~KV_ELT_DIRTY; - return TRUE; -} - -/* Process operations queue */ -static gboolean -bdb_process_queue (struct rspamd_kv_backend *backend) -{ - struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; - struct bdb_op *op; - GList *cur; - - cur = db->ops_queue->head; - while (cur) { - op = cur->data; - if (!bdb_process_single_op (db, NULL, op)) { - return FALSE; - } - cur = g_list_next (cur); - } - - /* Clean the queue */ - cur = db->ops_queue->head; - while (cur) { - op = cur->data; - if (op->op == BDB_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != - 0) { - /* Also clean memory */ - g_slice_free1 (ELT_SIZE (op->elt), op->elt); - } - g_slice_free1 (sizeof (struct bdb_op), op); - cur = g_list_next (cur); - } - - g_hash_table_remove_all (db->ops_hash); - g_queue_clear (db->ops_queue); - - return TRUE; - -} - -/* Backend callbacks */ -static void -rspamd_bdb_init (struct rspamd_kv_backend *backend) -{ - struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; - guint32 flags; - gint ret; - - if ((ret = db_env_create (&db->envp, 0)) != 0) { - /* Cannot create environment */ - goto err; - } - - flags = DB_INIT_MPOOL | - DB_CREATE | /* Create the environment if it does not already exist. */ - DB_INIT_LOCK | /* Initialize locking. */ - DB_THREAD; /* Use threads */ - - if ((ret = db->envp->open (db->envp, db->dirname, flags, 0)) != 0) { - /* Cannot open environment */ - goto err; - } - /* - * Configure db to perform deadlock detection internally, and to - * choose the transaction that has performed the least amount of - * writing to break the deadlock in the event that one is detected. - */ - db->envp->set_lk_detect (db->envp, DB_LOCK_DEFAULT); - - /* - * Avoid explicit sync on committing - */ - db->envp->set_flags (db->envp, DB_TXN_NOSYNC, 1); - - flags = DB_CREATE | DB_THREAD; - /* Create and open db pointer */ - if ((ret = db_create (&db->dbp, db->envp, 0)) != 0) { - goto err; - } - - if ((ret = - db->dbp->open (db->dbp, NULL, db->filename, NULL, DB_HASH, flags, - 0)) != 0) { - goto err; - } - - db->initialized = TRUE; - - return; -err: - if (db->dbp != NULL) { - msg_err ("error opening bdb database: %s", db_strerror (ret)); - db->dbp->close (db->dbp, 0); - } - if (db->envp != NULL) { - msg_err ("error opening bdb environment: %s", db_strerror (ret)); - db->envp->close (db->envp, 0); - } -} - -static gboolean -rspamd_bdb_insert (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt) -{ - struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; - struct bdb_op *op; - - if (!db->initialized) { - return FALSE; - } - - op = g_slice_alloc (sizeof (struct bdb_op)); - op->op = BDB_OP_INSERT; - op->elt = elt; - elt->flags |= KV_ELT_DIRTY; - - g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, elt, op); - - if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= - db->sync_ops) { - return bdb_process_queue (backend); - } - - return TRUE; -} - -static gboolean -rspamd_bdb_replace (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt) -{ - struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; - struct bdb_op *op; - - if (!db->initialized) { - return FALSE; - } - - op = g_slice_alloc (sizeof (struct bdb_op)); - op->op = BDB_OP_REPLACE; - op->elt = elt; - elt->flags |= KV_ELT_DIRTY; - - g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, elt, op); - - if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= - db->sync_ops) { - return bdb_process_queue (backend); - } - - return TRUE; -} - -static struct rspamd_kv_element * -rspamd_bdb_lookup (struct rspamd_kv_backend *backend, gpointer key, - guint keylen) -{ - struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; - struct bdb_op *op; - DBT db_key, db_data; - struct rspamd_kv_element *elt = NULL; - struct rspamd_kv_element search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - if (!db->initialized) { - return NULL; - } - /* First search in ops queue */ - if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { - if (op->op == BDB_OP_DELETE) { - /* To delete, so assume it as not found */ - return NULL; - } - return op->elt; - } - - memset (&db_key, 0, sizeof(DBT)); - memset (&db_data, 0, sizeof(DBT)); - db_key.size = keylen; - db_key.data = key; - db_data.flags = DB_DBT_MALLOC; - - if (db->dbp->get (db->dbp, NULL, &db_key, &db_data, 0) == 0) { - elt = db_data.data; - elt->flags &= ~KV_ELT_DIRTY; - } - - return elt; -} - -static void -rspamd_bdb_delete (struct rspamd_kv_backend *backend, gpointer key, - guint keylen) -{ - struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; - struct bdb_op *op; - struct rspamd_kv_element *elt; - struct rspamd_kv_element search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - if (!db->initialized) { - return; - } - - if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { - op->op = BDB_OP_DELETE; - return; - } - - elt = rspamd_bdb_lookup (backend, key, keylen); - if (elt == NULL) { - return; - } - op = g_slice_alloc (sizeof (struct bdb_op)); - op->op = BDB_OP_DELETE; - op->elt = elt; - elt->flags |= KV_ELT_DIRTY; - - g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, elt, op); - - if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= - db->sync_ops) { - bdb_process_queue (backend); - } - - return; -} - -static void -rspamd_bdb_destroy (struct rspamd_kv_backend *backend) -{ - struct rspamd_bdb_backend *db = (struct rspamd_bdb_backend *)backend; - - if (db->initialized) { - bdb_process_queue (backend); - if (db->dbp != NULL) { - db->dbp->close (db->dbp, 0); - } - if (db->envp != NULL) { - db->envp->close (db->envp, 0); - } - g_free (db->filename); - g_free (db->dirname); - g_queue_free (db->ops_queue); - g_hash_table_unref (db->ops_hash); - g_slice_free1 (sizeof (struct rspamd_bdb_backend), db); - } -} - -/* Create new bdb backend */ -struct rspamd_kv_backend * -rspamd_kv_bdb_new (const gchar *filename, guint sync_ops) -{ - struct rspamd_bdb_backend *new; - struct stat st; - gchar *dirname; - - if (filename == NULL) { - return NULL; - } - dirname = g_path_get_dirname (filename); - if (dirname == NULL || stat (dirname, &st) == -1 || !S_ISDIR (st.st_mode)) { - /* Inaccessible path */ - if (dirname != NULL) { - g_free (dirname); - } - msg_err ("invalid file: %s", filename); - return NULL; - } - - new = g_slice_alloc0 (sizeof (struct rspamd_bdb_backend)); - new->dirname = dirname; - new->filename = g_strdup (filename); - new->sync_ops = sync_ops; - new->ops_queue = g_queue_new (); - new->ops_hash = g_hash_table_new (kv_elt_hash_func, kv_elt_compare_func); - - /* Init callbacks */ - new->init_func = rspamd_bdb_init; - new->insert_func = rspamd_bdb_insert; - new->lookup_func = rspamd_bdb_lookup; - new->delete_func = rspamd_bdb_delete; - new->replace_func = rspamd_bdb_replace; - new->sync_func = bdb_process_queue; - new->destroy_func = rspamd_bdb_destroy; - - return (struct rspamd_kv_backend *)new; -} diff --git a/src/kvstorage_bdb.h b/src/kvstorage_bdb.h deleted file mode 100644 index d71f1f65f..000000000 --- a/src/kvstorage_bdb.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (c) 2010, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - -#ifndef KVSTORAGE_BDB_H_ -#define KVSTORAGE_BDB_H_ - -#include "config.h" -#include "kvstorage.h" - -#ifdef WITH_DB - -/* Create new bdb backend */ -struct rspamd_kv_backend * rspamd_kv_bdb_new (const gchar *filename, - guint sync_ops); - -#endif - -#endif /* KVSTORAGE_BDB_H_ */ diff --git a/src/kvstorage_config.c b/src/kvstorage_config.c deleted file mode 100644 index f236c2b4b..000000000 --- a/src/kvstorage_config.c +++ /dev/null @@ -1,603 +0,0 @@ -/* Copyright (c) 2010, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "kvstorage_config.h" -#include "main.h" -#include "cfg_xml.h" -#ifdef WITH_DB -#include "kvstorage_bdb.h" -#endif -#ifdef WITH_SQLITE -#include "kvstorage_sqlite.h" -#endif -#include "kvstorage_file.h" - -#define FILE_STORAGE_LEVELS 3 - - -/* Global hash of storages indexed by id */ -GHashTable *storages = NULL; -/* Last used id for explicit numbering */ -gint last_id = 0; - -struct kvstorage_config_parser { - enum { - KVSTORAGE_STATE_INIT, - KVSTORAGE_STATE_PARAM, - KVSTORAGE_STATE_BACKEND, - KVSTORAGE_STATE_EXPIRE, - KVSTORAGE_STATE_ID, - KVSTORAGE_STATE_NAME, - KVSTORAGE_STATE_CACHE_TYPE, - KVSTORAGE_STATE_CACHE_MAX_ELTS, - KVSTORAGE_STATE_CACHE_MAX_MEM, - KVSTORAGE_STATE_CACHE_NO_OVERWRITE, - KVSTORAGE_STATE_BACKEND_TYPE, - KVSTORAGE_STATE_BACKEND_FILENAME, - KVSTORAGE_STATE_BACKEND_SYNC_OPS, - KVSTORAGE_STATE_BACKEND_DO_FSYNC, - KVSTORAGE_STATE_BACKEND_DO_REF, - KVSTORAGE_STATE_EXPIRE_TYPE, - KVSTORAGE_STATE_ERROR - } state; - struct kvstorage_config *current_storage; - rspamd_mempool_t *pool; - gchar *cur_elt; -}; - -static void -kvstorage_config_destroy (gpointer k) -{ - struct kvstorage_config *kconf = k; - - if (kconf->name) { - g_free (kconf->name); - } - - if (kconf->storage) { - rspamd_kv_storage_destroy (kconf->storage); - } - - g_free (kconf); -} - -/* Init kvstorage */ -static void -kvstorage_init_callback (const gpointer key, - const gpointer value, - gpointer unused) -{ - struct kvstorage_config *kconf = value; - struct rspamd_kv_cache *cache; - struct rspamd_kv_backend *backend = NULL; - struct rspamd_kv_expire *expire = NULL; - - switch (kconf->cache.type) { - case KVSTORAGE_TYPE_CACHE_HASH: - cache = rspamd_kv_hash_new (); - break; - case KVSTORAGE_TYPE_CACHE_RADIX: - cache = rspamd_kv_radix_new (); - break; -#ifdef WITH_JUDY - case KVSTORAGE_TYPE_CACHE_JUDY: - cache = rspamd_kv_judy_new (); - break; -#endif - default: - msg_err ("unknown cache type, internal error"); - return; - } - - switch (kconf->backend.type) { - case KVSTORAGE_TYPE_BACKEND_NULL: - case KVSTORAGE_TYPE_BACKEND_MAX: - backend = NULL; - break; - case KVSTORAGE_TYPE_BACKEND_FILE: - backend = rspamd_kv_file_new (kconf->backend.filename, - kconf->backend.sync_ops, - FILE_STORAGE_LEVELS, - kconf->backend.do_fsync, - kconf->backend.do_ref); - break; -#ifdef WITH_DB - case KVSTORAGE_TYPE_BACKEND_BDB: - backend = rspamd_kv_bdb_new (kconf->backend.filename, - kconf->backend.sync_ops); - break; -#endif -#ifdef WITH_SQLITE - case KVSTORAGE_TYPE_BACKEND_SQLITE: - backend = rspamd_kv_sqlite_new (kconf->backend.filename, - kconf->backend.sync_ops); - break; -#endif - } - - switch (kconf->expire.type) { - case KVSTORAGE_TYPE_EXPIRE_LRU: - expire = rspamd_lru_expire_new (); - break; - } - - kconf->storage = rspamd_kv_storage_new (kconf->id, - kconf->name, - cache, - backend, - expire, - kconf->cache.max_elements, - kconf->cache.max_memory, - kconf->cache.no_overwrite); -} - -/* XML parse callbacks */ -/* Called for open tags */ -void -kvstorage_xml_start_element (GMarkupParseContext *context, - const gchar *element_name, - const gchar **attribute_names, - const gchar **attribute_values, - gpointer user_data, - GError **error) -{ - struct kvstorage_config_parser *kv_parser = user_data; - - switch (kv_parser->state) { - case KVSTORAGE_STATE_INIT: - /* XXX: never get this state */ - break; - case KVSTORAGE_STATE_PARAM: - if (kv_parser->current_storage == NULL) { - /* Make temporary pool */ - if (kv_parser->pool != NULL) { - rspamd_mempool_delete (kv_parser->pool); - } - kv_parser->pool = - rspamd_mempool_new (rspamd_mempool_suggest_size ()); - - /* Create new kvstorage_config */ - kv_parser->current_storage = - g_malloc0 (sizeof (struct kvstorage_config)); - kv_parser->current_storage->id = last_id++; - } - if (g_ascii_strcasecmp (element_name, "type") == 0) { - kv_parser->state = KVSTORAGE_STATE_CACHE_TYPE; - kv_parser->cur_elt = "type"; - } - else if (g_ascii_strcasecmp (element_name, "max_elements") == 0) { - kv_parser->state = KVSTORAGE_STATE_CACHE_MAX_ELTS; - kv_parser->cur_elt = "max_elements"; - } - else if (g_ascii_strcasecmp (element_name, "max_memory") == 0) { - kv_parser->state = KVSTORAGE_STATE_CACHE_MAX_MEM; - kv_parser->cur_elt = "max_memory"; - } - else if (g_ascii_strcasecmp (element_name, "no_overwrite") == 0) { - kv_parser->state = KVSTORAGE_STATE_CACHE_NO_OVERWRITE; - kv_parser->cur_elt = "no_overwrite"; - } - else if (g_ascii_strcasecmp (element_name, "id") == 0) { - kv_parser->state = KVSTORAGE_STATE_ID; - kv_parser->cur_elt = "id"; - } - else if (g_ascii_strcasecmp (element_name, "name") == 0) { - kv_parser->state = KVSTORAGE_STATE_NAME; - kv_parser->cur_elt = "name"; - } - else if (g_ascii_strcasecmp (element_name, "backend") == 0) { - kv_parser->state = KVSTORAGE_STATE_BACKEND; - kv_parser->cur_elt = "backend"; - } - else if (g_ascii_strcasecmp (element_name, "expire") == 0) { - kv_parser->state = KVSTORAGE_STATE_EXPIRE; - kv_parser->cur_elt = "expire"; - } - else { - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected", - element_name); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - } - break; - case KVSTORAGE_STATE_BACKEND: - if (g_ascii_strcasecmp (element_name, "type") == 0) { - kv_parser->state = KVSTORAGE_STATE_BACKEND_TYPE; - kv_parser->cur_elt = "type"; - } - else if (g_ascii_strcasecmp (element_name, "filename") == 0) { - kv_parser->state = KVSTORAGE_STATE_BACKEND_FILENAME; - kv_parser->cur_elt = "filename"; - } - else if (g_ascii_strcasecmp (element_name, "sync_ops") == 0) { - kv_parser->state = KVSTORAGE_STATE_BACKEND_SYNC_OPS; - kv_parser->cur_elt = "sync_ops"; - } - else if (g_ascii_strcasecmp (element_name, "fsync") == 0) { - kv_parser->state = KVSTORAGE_STATE_BACKEND_DO_FSYNC; - kv_parser->cur_elt = "fsync"; - } - else if (g_ascii_strcasecmp (element_name, "ref") == 0) { - kv_parser->state = KVSTORAGE_STATE_BACKEND_DO_REF; - kv_parser->cur_elt = "ref"; - } - else { - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected in backend definition", - element_name); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - } - break; - case KVSTORAGE_STATE_EXPIRE: - if (g_ascii_strcasecmp (element_name, "type") == 0) { - kv_parser->state = KVSTORAGE_STATE_EXPIRE_TYPE; - kv_parser->cur_elt = "type"; - } - else { - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected in expire definition", - element_name); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - } - break; - default: - /* Do nothing at other states */ - break; - } - -} - -#define CHECK_TAG(s) \ - do { \ - if (g_ascii_strcasecmp (element_name, kv_parser->cur_elt) == 0) { \ - kv_parser->state = (s); \ - } \ - else { \ - if (*error == NULL) *error = g_error_new ( \ - xml_error_quark (), XML_UNMATCHED_TAG, "element %s is unexpected in this state, expected %s", element_name, \ - kv_parser->cur_elt); \ - kv_parser->state = KVSTORAGE_STATE_ERROR; \ - } \ - } while (0) - -/* Called for close tags */ -void -kvstorage_xml_end_element (GMarkupParseContext *context, - const gchar *element_name, - gpointer user_data, - GError **error) -{ - struct kvstorage_config_parser *kv_parser = user_data; - - switch (kv_parser->state) { - case KVSTORAGE_STATE_INIT: - case KVSTORAGE_STATE_PARAM: - if (g_ascii_strcasecmp (element_name, "keystorage") == 0) { - /* XXX: Init actual storage */ - return; - } - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, "end element %s is unexpected, expected start element", - element_name); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - break; - case KVSTORAGE_STATE_ID: - case KVSTORAGE_STATE_NAME: - case KVSTORAGE_STATE_CACHE_TYPE: - case KVSTORAGE_STATE_CACHE_MAX_ELTS: - case KVSTORAGE_STATE_CACHE_MAX_MEM: - case KVSTORAGE_STATE_CACHE_NO_OVERWRITE: - CHECK_TAG (KVSTORAGE_STATE_PARAM); - break; - case KVSTORAGE_STATE_BACKEND_TYPE: - case KVSTORAGE_STATE_BACKEND_FILENAME: - case KVSTORAGE_STATE_BACKEND_SYNC_OPS: - case KVSTORAGE_STATE_BACKEND_DO_FSYNC: - case KVSTORAGE_STATE_BACKEND_DO_REF: - CHECK_TAG (KVSTORAGE_STATE_BACKEND); - break; - case KVSTORAGE_STATE_EXPIRE_TYPE: - CHECK_TAG (KVSTORAGE_STATE_EXPIRE); - break; - case KVSTORAGE_STATE_BACKEND: - if (g_ascii_strcasecmp (element_name, "backend") == 0) { - kv_parser->state = KVSTORAGE_STATE_PARAM; - } - else { - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected", - element_name); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - } - break; - case KVSTORAGE_STATE_EXPIRE: - if (g_ascii_strcasecmp (element_name, "expire") == 0) { - kv_parser->state = KVSTORAGE_STATE_PARAM; - } - else { - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected", - element_name); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - } - break; - default: - /* Do nothing at other states */ - break; - } -} -#undef CHECK_TAG - -/* text is not nul-terminated */ -void -kvstorage_xml_text (GMarkupParseContext *context, - const gchar *text, - gsize text_len, - gpointer user_data, - GError **error) -{ - struct kvstorage_config_parser *kv_parser = user_data; - gchar *err_str; - - /* Strip space symbols */ - while (*text && g_ascii_isspace (*text)) { - text++; - } - if (*text == '\0') { - /* Skip empty text */ - return; - } - - switch (kv_parser->state) { - case KVSTORAGE_STATE_INIT: - case KVSTORAGE_STATE_PARAM: - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, - "text is unexpected, expected start element"); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - break; - case KVSTORAGE_STATE_ID: - kv_parser->current_storage->id = strtoul (text, &err_str, 10); - if ((gsize)(err_str - text) != text_len) { - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, "invalid number: %*s", - (int)text_len, text); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - } - else { - last_id++; - last_id = MAX (kv_parser->current_storage->id, last_id); - } - break; - case KVSTORAGE_STATE_NAME: - kv_parser->current_storage->name = g_malloc (text_len + 1); - rspamd_strlcpy (kv_parser->current_storage->name, text, text_len + 1); - break; - case KVSTORAGE_STATE_CACHE_MAX_ELTS: - kv_parser->current_storage->cache.max_elements = - rspamd_config_parse_limit (text, text_len); - break; - case KVSTORAGE_STATE_CACHE_MAX_MEM: - kv_parser->current_storage->cache.max_memory = - rspamd_config_parse_limit (text, text_len); - break; - case KVSTORAGE_STATE_CACHE_NO_OVERWRITE: - kv_parser->current_storage->cache.no_overwrite = - rspamd_config_parse_flag (text); - break; - case KVSTORAGE_STATE_CACHE_TYPE: - if (g_ascii_strncasecmp (text, "hash", - MIN (text_len, sizeof ("hash") - 1)) == 0) { - kv_parser->current_storage->cache.type = KVSTORAGE_TYPE_CACHE_HASH; - } - else if (g_ascii_strncasecmp (text, "radix", - MIN (text_len, sizeof ("radix") - 1)) == 0) { - kv_parser->current_storage->cache.type = KVSTORAGE_TYPE_CACHE_RADIX; - } -#ifdef WITH_JUDY - else if (g_ascii_strncasecmp (text, "judy", - MIN (text_len, sizeof ("judy") - 1)) == 0) { - kv_parser->current_storage->cache.type = KVSTORAGE_TYPE_CACHE_JUDY; - } -#endif - else { - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, "invalid cache type: %*s", - (int)text_len, text); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - } - break; - case KVSTORAGE_STATE_BACKEND_TYPE: - if (g_ascii_strncasecmp (text, "null", - MIN (text_len, sizeof ("null") - 1)) == 0) { - kv_parser->current_storage->backend.type = - KVSTORAGE_TYPE_BACKEND_NULL; - } - else if (g_ascii_strncasecmp (text, "file", - MIN (text_len, sizeof ("file") - 1)) == 0) { - kv_parser->current_storage->backend.type = - KVSTORAGE_TYPE_BACKEND_FILE; - } -#ifdef WITH_DB - else if (g_ascii_strncasecmp (text, "bdb", - MIN (text_len, sizeof ("bdb") - 1)) == 0) { - kv_parser->current_storage->backend.type = - KVSTORAGE_TYPE_BACKEND_BDB; - } -#endif -#ifdef WITH_SQLITE - else if (g_ascii_strncasecmp (text, "sqlite", - MIN (text_len, sizeof ("sqlite") - 1)) == 0) { - kv_parser->current_storage->backend.type = - KVSTORAGE_TYPE_BACKEND_SQLITE; - } -#endif - else { - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, "invalid backend type: %*s", - (int)text_len, text); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - } - break; - case KVSTORAGE_STATE_BACKEND_FILENAME: - kv_parser->current_storage->backend.filename = g_malloc (text_len + 1); - rspamd_strlcpy (kv_parser->current_storage->backend.filename, - text, - text_len + 1); - break; - case KVSTORAGE_STATE_BACKEND_SYNC_OPS: - kv_parser->current_storage->backend.sync_ops = - rspamd_config_parse_limit (text, text_len); - break; - case KVSTORAGE_STATE_BACKEND_DO_FSYNC: - kv_parser->current_storage->backend.do_fsync = - rspamd_config_parse_flag (text); - break; - case KVSTORAGE_STATE_BACKEND_DO_REF: - kv_parser->current_storage->backend.do_ref = rspamd_config_parse_flag ( - text); - break; - case KVSTORAGE_STATE_EXPIRE_TYPE: - if (g_ascii_strncasecmp (text, "lru", - MIN (text_len, sizeof ("lru") - 1)) == 0) { - kv_parser->current_storage->expire.type = KVSTORAGE_TYPE_EXPIRE_LRU; - } - else { - if (*error == NULL) { - *error = g_error_new ( - xml_error_quark (), XML_EXTRA_ELEMENT, "invalid expire type: %*s", - (int)text_len, text); - } - kv_parser->state = KVSTORAGE_STATE_ERROR; - } - break; - default: - /* Do nothing at other states */ - break; - } - -} - -/* Called on error, including one set by other - * methods in the vtable. The GError should not be freed. - */ -void -kvstorage_xml_error (GMarkupParseContext *context, - GError *error, - gpointer user_data) -{ - msg_err ("kvstorage xml parser error: %s", error->message); -} - -/* - * Cleanup kvstorage after end tag was read - */ -static void -kvstorage_cleanup (gpointer ud) -{ - struct kvstorage_config_parser *kv_parser = ud; - - g_hash_table_insert (storages, - &kv_parser->current_storage->id, - kv_parser->current_storage); - kv_parser->state = KVSTORAGE_STATE_INIT; - g_hash_table_foreach (storages, kvstorage_init_callback, NULL); - kv_parser->current_storage = NULL; -} - -/** Public API */ - -/* Init subparser of kvstorage config */ -void -init_kvstorage_config (void) -{ - GMarkupParser *parser; - struct kvstorage_config_parser *kv_parser; - - if (storages == NULL) { - storages = g_hash_table_new_full (g_int_hash, - g_int_equal, - NULL, - kvstorage_config_destroy); - } - else { - /* Create new global table */ - g_hash_table_destroy (storages); - storages = g_hash_table_new_full (g_int_hash, - g_int_equal, - NULL, - kvstorage_config_destroy); - } - - /* Create and register subparser */ - parser = g_malloc0 (sizeof (GMarkupParser)); - parser->start_element = kvstorage_xml_start_element; - parser->end_element = kvstorage_xml_end_element; - parser->error = kvstorage_xml_error; - parser->text = kvstorage_xml_text; - - kv_parser = g_malloc0 (sizeof (struct kvstorage_config_parser)); - kv_parser->state = KVSTORAGE_STATE_PARAM; - kv_parser->pool = rspamd_mempool_new (rspamd_mempool_suggest_size ()); - - register_subparser ("keystorage", 0, parser, kvstorage_cleanup, kv_parser); -} - -/* Get configuration for kvstorage with specified ID */ -struct kvstorage_config * -get_kvstorage_config (gint id) -{ - if (storages == NULL) { - return NULL; - } - return g_hash_table_lookup (storages, &id); -} - -void -destroy_kvstorage_config (void) -{ - if (storages != NULL) { - g_hash_table_destroy (storages); - storages = NULL; - } -} diff --git a/src/kvstorage_config.h b/src/kvstorage_config.h deleted file mode 100644 index f94d934b3..000000000 --- a/src/kvstorage_config.h +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright (c) 2010, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - -#ifndef KVSTORAGE_CONFIG_H_ -#define KVSTORAGE_CONFIG_H_ - -#include "config.h" -#include "kvstorage.h" - -/* Type of kvstorage cache */ -enum kvstorage_cache_type { - KVSTORAGE_TYPE_CACHE_HASH, - KVSTORAGE_TYPE_CACHE_RADIX, -#ifdef WITH_JUDY - KVSTORAGE_TYPE_CACHE_JUDY, -#endif - KVSTORAGE_TYPE_MAX = 255 -}; - -/* Type of kvstorage backend */ -enum kvstorage_backend_type { - KVSTORAGE_TYPE_BACKEND_NULL = 0, - KVSTORAGE_TYPE_BACKEND_FILE, -#ifdef WITH_DB - KVSTORAGE_TYPE_BACKEND_BDB, -#endif -#ifdef WITH_SQLITE - KVSTORAGE_TYPE_BACKEND_SQLITE, -#endif - KVSTORAGE_TYPE_BACKEND_MAX = 255 -}; - -/* Type of kvstorage expire */ -enum kvstorage_expire_type { - KVSTORAGE_TYPE_EXPIRE_LRU -}; - -/* Cache config */ -struct kvstorage_cache_config { - gsize max_elements; - gsize max_memory; - gboolean no_overwrite; - enum kvstorage_cache_type type; -}; - -/* Backend config */ -struct kvstorage_backend_config { - enum kvstorage_backend_type type; - gchar *filename; - guint sync_ops; - gboolean do_fsync; - gboolean do_ref; -}; - - -/* Expire config */ -struct kvstorage_expire_config { - enum kvstorage_expire_type type; -}; - -/* The main keystorage config */ -struct kvstorage_config { - gint id; - gchar *name; - struct kvstorage_cache_config cache; - struct kvstorage_backend_config backend; - struct kvstorage_expire_config expire; - struct rspamd_kv_storage *storage; -}; - -/* Init subparser of kvstorage config */ -void init_kvstorage_config (void); - -/* Get configuration for kvstorage with specified ID */ -struct kvstorage_config * get_kvstorage_config (gint id); - -void destroy_kvstorage_config (void); - -#endif /* KVSTORAGE_CONFIG_H_ */ diff --git a/src/kvstorage_file.c b/src/kvstorage_file.c deleted file mode 100644 index a8550b77e..000000000 --- a/src/kvstorage_file.c +++ /dev/null @@ -1,739 +0,0 @@ -/* Copyright (c) 2010, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - -#include "config.h" -#include "kvstorage.h" -#include "kvstorage_file.h" -#include "util.h" -#include "main.h" - -struct file_op { - struct rspamd_kv_element *elt; - enum { - FILE_OP_INSERT, - FILE_OP_DELETE, - FILE_OP_REPLACE - } op; - guint32 ref; -}; - -/* Main file structure */ -struct rspamd_file_backend { - backend_init init_func; /*< this callback is called on kv storage initialization */ - backend_insert insert_func; /*< this callback is called when element is inserted */ - backend_replace replace_func; /*< this callback is called when element is replaced */ - backend_lookup lookup_func; /*< this callback is used for lookup of element */ - backend_delete delete_func; /*< this callback is called when an element is deleted */ - backend_sync sync_func; /*< this callback is called when backend need to be synced */ - backend_incref incref_func; /*< this callback is called when element must be ref'd */ - backend_destroy destroy_func; /*< this callback is used for destroying all elements inside backend */ - gchar *filename; - gchar *dirname; - guint dirlen; - guint sync_ops; - guint levels; - GQueue *ops_queue; - GHashTable *ops_hash; - gboolean do_fsync; - gboolean do_ref; - gboolean initialized; -}; - -static const gchar hexdigits[] = -{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', - 'f'}; - -/* Generate file name for operation */ -static gboolean -get_file_name (struct rspamd_file_backend *db, - gchar *key, - guint keylen, - gchar *filebuf, - guint buflen) -{ - gchar *p = filebuf, *end = filebuf + buflen, - *k = key, t; - guint i; - - /* First copy backend dirname to file buf */ - if (buflen <= db->dirlen) { - return FALSE; - } - memcpy (p, db->dirname, db->dirlen); - p += db->dirlen; - *p++ = G_DIR_SEPARATOR; - for (i = 0; i < MIN (keylen, db->levels); i++) { - if (p == end) { - /* Filebuf is not large enough */ - return FALSE; - } - t = *k; - *p++ = hexdigits[(t & 0xf) ^ ((t & 0xf0) >> 4)]; - *p++ = G_DIR_SEPARATOR; - k++; - } - /* Now we have directory, append base64 encoded filename */ - k = key; - if (end - p < (gint)keylen * 2 + 1) { - /* Filebuf is not large enough */ - return FALSE; - } - - i = 0; - while (k < key + keylen) { - t = *k; - *p++ = hexdigits[(t >> 4) & 0xf]; - *p++ = hexdigits[t & 0xf]; - k++; - } - *p = '\0'; - - return TRUE; -} - -/* Read reference from specified file */ -static guint32 -file_get_ref (gint fd) -{ - guint32 target; - - if (read (fd, &target, sizeof (guint32)) != sizeof (guint32)) { - return 0; - } - - return target; -} - -/* Set reference to specified file */ -static gboolean -file_set_ref (gint fd, guint32 ref) -{ - if (write (fd, &ref, sizeof (guint32)) != sizeof (guint32)) { - return FALSE; - } - - return TRUE; -} - -/* - * Open file, set posix_fadvise and all necessary flags - */ -static gint -file_open_fd (const gchar *path, gsize *len, gint flags) -{ - gint fd; - struct stat st; - - if ((flags & O_CREAT) != 0) { - /* Open file */ - if ((fd = open (path, flags, S_IRUSR | S_IWUSR | S_IRGRP)) != -1) { - rspamd_fallocate (fd, 0, *len); -#ifdef HAVE_FADVISE - posix_fadvise (fd, 0, *len, POSIX_FADV_SEQUENTIAL); -#endif - } - } - else { - /* Open file */ - if ((fd = open (path, flags)) == -1) { - return -1; - } - - if (fstat (fd, &st) == -1) { - close (fd); - return -1; - } - -#ifdef HAVE_FADVISE - posix_fadvise (fd, 0, st.st_size, POSIX_FADV_SEQUENTIAL); -#endif - *len = st.st_size; - } - - return fd; -} - -/* Process single file operation */ -static gboolean -file_process_single_op (struct rspamd_file_backend *db, - struct file_op *op, - gint *pfd) -{ - gchar filebuf[PATH_MAX]; - gint fd; - gsize len; - struct iovec iov[2]; - guint32 ref; - - /* Get filename */ - if (!get_file_name (db, ELT_KEY (op->elt), op->elt->keylen, filebuf, - sizeof (filebuf))) { - return FALSE; - } - - if (db->do_ref) { - len = ELT_SIZE (op->elt) + sizeof (guint32); - } - else { - len = ELT_SIZE (op->elt); - } - - if (op->op == FILE_OP_DELETE) { - if (db->do_ref) { - if ((fd = file_open_fd (filebuf, &len, O_RDWR)) == -1) { - *pfd = -1; - return FALSE; - } - if ((ref = file_get_ref (fd)) <= 1) { - /* Refcount is not enough, remove file */ - close (fd); - *pfd = -1; - return unlink (filebuf) != -1; - } - else { - /* Decrease ref */ - lseek (fd, 0, SEEK_SET); - if (!file_set_ref (fd, --ref)) { - *pfd = fd; - return FALSE; - } - } - } - else { - *pfd = -1; - return unlink (filebuf) != -1; - } - } - else { - if ((fd = - file_open_fd (filebuf, &len, O_CREAT | O_WRONLY | O_TRUNC)) == -1) { - *pfd = -1; - return FALSE; - } - if (db->do_ref) { - iov[0].iov_base = &op->ref; - iov[0].iov_len = sizeof (guint32); - iov[1].iov_base = op->elt; - iov[1].iov_len = ELT_SIZE (op->elt); - if (writev (fd, iov, G_N_ELEMENTS (iov)) == -1) { - msg_info ("%d: %s", errno, strerror (errno)); - *pfd = fd; - return FALSE; - } - } - else { - if (write (fd, op->elt, ELT_SIZE (op->elt)) == -1) { - msg_info ("%d: %s", errno, strerror (errno)); - *pfd = fd; - return FALSE; - } - } - } - - *pfd = fd; - return TRUE; -} - -/* Sync vector of descriptors */ -static void -file_sync_fds (gint *fds, gint len, gboolean do_fsync) -{ - gint i, fd; - - for (i = 0; i < len; i++) { - fd = fds[i]; - if (fd != -1) { - if (do_fsync) { -#ifdef HAVE_FDATASYNC - fdatasync (fd); -#else - fsync (fd); -#endif - } - close (fd); - } - } -} - -/* Process operations queue */ -static gboolean -file_process_queue (struct rspamd_kv_backend *backend) -{ - struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; - struct file_op *op; - GList *cur; - gint *fds, i = 0, len; - - len = g_queue_get_length (db->ops_queue); - if (len == 0) { - /* Nothing to process */ - return TRUE; - } - - fds = g_slice_alloc (len * sizeof (gint)); - cur = db->ops_queue->head; - while (cur) { - op = cur->data; - if (!file_process_single_op (db, op, &fds[i])) { - file_sync_fds (fds, i, db->do_fsync); - g_slice_free1 (len * sizeof (gint), fds); - return FALSE; - } - i++; - cur = g_list_next (cur); - } - - file_sync_fds (fds, i, db->do_fsync); - g_slice_free1 (len * sizeof (gint), fds); - - /* Clean the queue */ - g_hash_table_remove_all (db->ops_hash); - cur = db->ops_queue->head; - while (cur) { - op = cur->data; - if (op->op == FILE_OP_DELETE || - ((op->elt->flags & KV_ELT_NEED_FREE) != 0 && - (op->elt->flags & KV_ELT_NEED_INSERT) == 0)) { - /* Also clean memory */ - g_slice_free1 (ELT_SIZE (op->elt), op->elt); - } - else { - /* Unset dirty flag */ - op->elt->flags &= ~KV_ELT_DIRTY; - } - g_slice_free1 (sizeof (struct file_op), op); - cur = g_list_next (cur); - } - - g_queue_clear (db->ops_queue); - - return TRUE; - -} - - -/* Make 16 directories for each level */ -static gboolean -rspamd_recursive_mkdir (guint levels) -{ - guint i; - gchar nbuf[5]; - - /* Create directories for backend */ - if (levels > 0) { - /* Create 16 directories */ - for (i = 0; i < 16; i++) { - rspamd_snprintf (nbuf, sizeof (nbuf), "./%c", hexdigits[i]); - if (mkdir (nbuf, 0755) != 0 && errno != EEXIST) { - msg_info ("cannot create directory %s: %s", nbuf, - strerror (errno)); - return FALSE; - } - else if (levels > 1) { - if (chdir (nbuf) == -1) { - msg_err ("chdir to %s failed: %s", nbuf, strerror (errno)); - return FALSE; - } - if (!rspamd_recursive_mkdir (levels - 1)) { - return FALSE; - } - if (chdir ("../") == -1) { - msg_err ("chdir to ../ failed: %s", strerror (errno)); - return FALSE; - } - } - } - } - return TRUE; - -} - -/* Backend callbacks */ -static void -rspamd_file_init (struct rspamd_kv_backend *backend) -{ - struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; - gchar pathbuf[PATH_MAX]; - - /* Save current directory */ - if (getcwd (pathbuf, sizeof (pathbuf) - 1) == NULL) { - pathbuf[0] = '\0'; - msg_err ("getcwd failed: %s", strerror (errno)); - goto err; - } - - /* Chdir to the working dir */ - if (chdir (db->dirname) == -1) { - msg_err ("chdir failed: %s", strerror (errno)); - goto err; - } - - /* Create directories for backend */ - if (!rspamd_recursive_mkdir (db->levels)) { - goto err; - } - - db->initialized = TRUE; - - if (chdir (pathbuf) == -1) { - msg_err ("chdir to %s failed: %s", pathbuf, strerror (errno)); - } - return; -err: - if (pathbuf[0] != '\0') { - if (chdir (pathbuf) == -1) { - msg_err ("chdir to %s failed: %s", pathbuf, strerror (errno)); - } - } -} - -static gboolean -rspamd_file_insert (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt) -{ - struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; - struct file_op *op; - struct rspamd_kv_element search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - if (!db->initialized) { - return FALSE; - } - - if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { - /* We found another op with such key in this queue */ - if (op->op == FILE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != - 0) { - /* Also clean memory */ - g_hash_table_steal (db->ops_hash, &search_elt); - g_slice_free1 (ELT_SIZE (op->elt), op->elt); - } - op->op = FILE_OP_INSERT; - op->ref++; - op->elt = elt; - elt->flags |= KV_ELT_DIRTY; - g_hash_table_insert (db->ops_hash, elt, op); - } - else { - op = g_slice_alloc (sizeof (struct file_op)); - op->op = FILE_OP_INSERT; - op->elt = elt; - op->ref = 1; - elt->flags |= KV_ELT_DIRTY; - - g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, elt, op); - } - - if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= - db->sync_ops) { - return file_process_queue (backend); - } - - return TRUE; -} - -static gboolean -rspamd_file_replace (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt) -{ - struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; - struct file_op *op; - struct rspamd_kv_element search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - if (!db->initialized) { - return FALSE; - } - if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { - /* We found another op with such key in this queue */ - if (op->op == FILE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != - 0) { - /* Also clean memory */ - g_hash_table_steal (db->ops_hash, &search_elt); - g_slice_free1 (ELT_SIZE (op->elt), op->elt); - } - op->op = FILE_OP_REPLACE; - op->elt = elt; - elt->flags |= KV_ELT_DIRTY; - g_hash_table_insert (db->ops_hash, elt, op); - } - else { - op = g_slice_alloc (sizeof (struct file_op)); - op->op = FILE_OP_REPLACE; - op->elt = elt; - op->ref = 1; - elt->flags |= KV_ELT_DIRTY; - - g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, elt, op); - } - - if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= - db->sync_ops) { - return file_process_queue (backend); - } - - return TRUE; -} - -static struct rspamd_kv_element * -rspamd_file_lookup (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen) -{ - struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; - struct file_op *op; - struct rspamd_kv_element *elt = NULL; - gchar filebuf[PATH_MAX]; - gint fd; - struct rspamd_kv_element search_elt; - gsize len; - - search_elt.keylen = keylen; - search_elt.p = key; - - if (!db->initialized) { - return NULL; - } - /* First search in ops queue */ - if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { - if (op->op == FILE_OP_DELETE) { - /* To delete, so assume it as not found */ - return NULL; - } - return op->elt; - } - - /* Get filename */ - if (!get_file_name (db, key, keylen, filebuf, sizeof (filebuf))) { - return NULL; - } - - if ((fd = file_open_fd (filebuf, &len, O_RDONLY)) == -1) { - return NULL; - } - - /* Read element */ - if (db->do_ref) { - lseek (fd, sizeof (guint32), SEEK_CUR); - len -= sizeof (guint32); - } - elt = g_malloc (len); - if (read (fd, elt, len) == -1) { - g_free (elt); - close (fd); - return NULL; - } - - close (fd); - - elt->flags &= ~(KV_ELT_DIRTY | KV_ELT_NEED_FREE); - - return elt; -} - -static void -rspamd_file_delete (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen) -{ - struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; - gchar filebuf[PATH_MAX]; - struct rspamd_kv_element search_elt; - struct file_op *op; - gsize len; - gint fd; - guint32 ref; - - if (!db->initialized) { - return; - } - - search_elt.keylen = keylen; - search_elt.p = key; - /* First search in ops queue */ - if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { - op->op = FILE_OP_DELETE; - if (op->ref > 0) { - op->ref--; - } - return; - } - /* Get filename */ - if (!get_file_name (db, key, keylen, filebuf, sizeof (filebuf))) { - return; - } - - if (db->do_ref) { - if ((fd = file_open_fd (filebuf, &len, O_RDWR)) == -1) { - return; - } - if ((ref = file_get_ref (fd)) <= 1) { - /* Refcount is not enough, remove file */ - close (fd); - unlink (filebuf); - } - else { - /* Decrease ref */ - lseek (fd, 0, SEEK_SET); - file_set_ref (fd, --ref); - } - return; - } - - unlink (filebuf); -} - -static gboolean -rspamd_file_incref (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen) -{ - struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; - gchar filebuf[PATH_MAX]; - struct rspamd_kv_element search_elt; - struct file_op *op; - gsize len; - gint fd; - guint32 ref; - - if (!db->initialized) { - return FALSE; - } - if (!db->do_ref) { - return TRUE; - } - - search_elt.keylen = keylen; - search_elt.p = key; - /* First search in ops queue */ - if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { - op->ref++; - if (op->op == FILE_OP_DELETE) { - op->op = FILE_OP_INSERT; - } - return TRUE; - } - - /* Get filename */ - if (!get_file_name (db, key, keylen, filebuf, sizeof (filebuf))) { - return FALSE; - } - - if ((fd = file_open_fd (filebuf, &len, O_RDWR)) == -1) { - return FALSE; - } - - ref = file_get_ref (fd); - - /* Decrease ref */ - lseek (fd, 0, SEEK_SET); - - if (file_set_ref (fd, ++ref)) { - close (fd); - return TRUE; - } - else { - close (fd); - return FALSE; - } -} - -static void -rspamd_file_destroy (struct rspamd_kv_backend *backend) -{ - struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; - - if (db->initialized) { - file_process_queue (backend); - g_free (db->filename); - g_free (db->dirname); - g_queue_free (db->ops_queue); - g_hash_table_unref (db->ops_hash); - g_slice_free1 (sizeof (struct rspamd_file_backend), db); - - /* Sync again */ - sync (); - } -} - -/* Create new file backend */ -struct rspamd_kv_backend * -rspamd_kv_file_new (const gchar *filename, - guint sync_ops, - guint levels, - gboolean do_fsync, - gboolean do_ref) -{ - struct rspamd_file_backend *new; - struct stat st; - gchar *dirname; - - if (filename == NULL) { - return NULL; - } - - dirname = g_path_get_dirname (filename); - if (dirname == NULL || stat (dirname, &st) == -1 || !S_ISDIR (st.st_mode)) { - /* Inaccessible path */ - if (dirname != NULL) { - g_free (dirname); - } - msg_err ("invalid file: %s", filename); - return NULL; - } - - new = g_slice_alloc0 (sizeof (struct rspamd_file_backend)); - new->dirname = dirname; - new->dirlen = strlen (dirname); - new->filename = g_strdup (filename); - new->sync_ops = sync_ops; - new->levels = levels; - new->do_fsync = do_fsync; - new->do_ref = do_ref; - new->ops_queue = g_queue_new (); - new->ops_hash = g_hash_table_new (kv_elt_hash_func, kv_elt_compare_func); - - /* Init callbacks */ - new->init_func = rspamd_file_init; - new->insert_func = rspamd_file_insert; - new->lookup_func = rspamd_file_lookup; - new->delete_func = rspamd_file_delete; - new->replace_func = rspamd_file_replace; - new->sync_func = file_process_queue; - new->incref_func = rspamd_file_incref; - new->destroy_func = rspamd_file_destroy; - - return (struct rspamd_kv_backend *)new; -} - diff --git a/src/kvstorage_file.h b/src/kvstorage_file.h deleted file mode 100644 index 4e97f467b..000000000 --- a/src/kvstorage_file.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (c) 2010, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Rambler BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - -#ifndef KVSTORAGE_FILE_H_ -#define KVSTORAGE_FILE_H_ - -#include "config.h" -#include "kvstorage.h" - -/* Create new file backend */ -struct rspamd_kv_backend * rspamd_kv_file_new (const gchar *filename, - guint sync_ops, - guint levels, - gboolean do_fsync, - gboolean do_ref); - - -#endif /* KVSTORAGE_FILE_H_ */ diff --git a/src/kvstorage_server.c b/src/kvstorage_server.c deleted file mode 100644 index 25dbc05a1..000000000 --- a/src/kvstorage_server.c +++ /dev/null @@ -1,1413 +0,0 @@ -/* Copyright (c) 2010, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - -#include "config.h" -#include "kvstorage.h" -#include "kvstorage_config.h" -#include "kvstorage_server.h" -#include "cfg_file.h" -#include "cfg_xml.h" -#include "main.h" - -#define ERROR_COMMON "ERROR" CRLF -#define ERROR_UNKNOWN_COMMAND "CLIENT_ERROR unknown command" CRLF -#define ERROR_NOT_STORED "NOT_STORED" CRLF -#define ERROR_EXISTS "EXISTS" CRLF -#define ERROR_NOT_FOUND "NOT_FOUND" CRLF -#define ERROR_INVALID_KEYSTORAGE "CLIENT_ERROR storage does not exists" CRLF - -#define ERROR_REDIS_OK "+OK" CRLF - - -static sig_atomic_t wanna_die = 0; -static sig_atomic_t do_reopen_log = 0; -static sig_atomic_t soft_wanna_die = 0; - -/* Logging functions */ -#define thr_err(...) do { \ - g_mutex_lock (thr->log_mtx); \ - rspamd_common_log_function (rspamd_main->logger, \ - G_LOG_LEVEL_CRITICAL, \ - __FUNCTION__, \ - __VA_ARGS__); \ - g_mutex_unlock (thr->log_mtx); \ -} while (0) - -#define thr_warn(...) do { \ - g_mutex_lock (thr->log_mtx); \ - rspamd_common_log_function (rspamd_main->logger, \ - G_LOG_LEVEL_WARNING, \ - __FUNCTION__, \ - __VA_ARGS__); \ - g_mutex_unlock (thr->log_mtx); \ -} while (0) - -#define thr_info(...) do { \ - g_mutex_lock (thr->log_mtx); \ - rspamd_common_log_function (rspamd_main->logger, \ - G_LOG_LEVEL_INFO, \ - __FUNCTION__, \ - __VA_ARGS__); \ - g_mutex_unlock (thr->log_mtx); \ -} while (0) - -/* Init functions */ -gpointer init_keystorage (void); -void start_keystorage (struct rspamd_worker *worker); - -worker_t keystorage_worker = { - "keystorage", /* Name */ - init_keystorage, /* Init function */ - start_keystorage, /* Start function */ - TRUE, /* Has socket */ - FALSE, /* Non unique */ - TRUE, /* Non threaded */ - FALSE, /* Non killable */ - SOCK_STREAM /* TCP socket */ -}; - -#ifndef HAVE_SA_SIGINFO -static void -sig_handler (gint signo) -#else -static void -sig_handler (gint signo, siginfo_t *info, void *unused) -#endif -{ - switch (signo) { - case SIGUSR1: - do_reopen_log = 1; - break; - case SIGINT: - case SIGTERM: - wanna_die = 1; - break; - case SIGUSR2: - soft_wanna_die = 1; - break; - } -} - -gpointer -init_keystorage (void) -{ - struct kvstorage_worker_ctx *ctx; - GQuark type; - - type = g_quark_try_string ("keystorage"); - ctx = g_malloc0 (sizeof (struct kvstorage_worker_ctx)); - ctx->pool = rspamd_mempool_new (rspamd_mempool_suggest_size ()); - - /* Set default values */ - ctx->timeout_raw = 300000; - - register_worker_opt (type, "timeout", xml_handle_seconds, ctx, - G_STRUCT_OFFSET (struct kvstorage_worker_ctx, timeout_raw)); - register_worker_opt (type, "redis", xml_handle_boolean, ctx, - G_STRUCT_OFFSET (struct kvstorage_worker_ctx, is_redis)); - return ctx; -} - -/* Make post-init configuration */ -static gboolean -config_kvstorage_worker (struct rspamd_worker *worker) -{ - struct kvstorage_worker_ctx *ctx = worker->ctx; - - /* Init timeval */ - msec_to_tv (ctx->timeout_raw, &ctx->io_timeout); - - return TRUE; -} - -/* - * Free kvstorage session - */ -static void -free_kvstorage_session (struct kvstorage_session *session) -{ - rspamd_remove_dispatcher (session->dispather); - rspamd_mempool_delete (session->pool); - close (session->sock); - g_slice_free1 (sizeof (struct kvstorage_session), session); -} - -/* - * Parse kvstorage command - */ -static gboolean -parse_kvstorage_command (struct kvstorage_session *session, gchar *c, guint len) -{ - if (len == 3) { - /* Set or get command */ - if ((c[0] == 'g' || c[0] == 'G') && - (c[1] == 'e' || c[1] == 'E') && - (c[2] == 't' || c[2] == 'T')) { - session->command = KVSTORAGE_CMD_GET; - } - else if ((c[0] == 's' || c[0] == 'S') && - (c[1] == 'e' || c[1] == 'E') && - (c[2] == 't' || c[2] == 'T')) { - session->command = KVSTORAGE_CMD_SET; - } - else if ((c[0] == 'd' || c[0] == 'D') && - (c[1] == 'e' || c[1] == 'E') && - (c[2] == 'l' || c[2] == 'L')) { - session->command = KVSTORAGE_CMD_DELETE; - } - else { - /* Error */ - return FALSE; - } - } - else if (len == 4) { - if ((c[0] == 'i' || c[0] == 'I') && - (c[1] == 'n' || c[1] == 'N') && - (c[2] == 'c' || c[2] == 'C') && - (c[3] == 'r' || c[3] == 'R')) { - session->command = KVSTORAGE_CMD_INCR; - session->arg_data.value = 1; - } - else if ((c[0] == 'd' || c[0] == 'D') && - (c[1] == 'e' || c[1] == 'E') && - (c[2] == 'c' || c[2] == 'C') && - (c[3] == 'r' || c[3] == 'R')) { - session->command = KVSTORAGE_CMD_DECR; - session->arg_data.value = -1; - } - else if (g_ascii_strncasecmp (c, "quit", 4) == 0) { - session->command = KVSTORAGE_CMD_QUIT; - } - else if (g_ascii_strncasecmp (c, "sync", - 4) == 0 || g_ascii_strncasecmp (c, "save", 4) == 0) { - session->command = KVSTORAGE_CMD_SYNC; - } - } - else if (len == 6) { - if ((c[0] == 'i' || c[0] == 'I') && - (c[1] == 'n' || c[1] == 'N') && - (c[2] == 'c' || c[2] == 'C') && - (c[3] == 'r' || c[3] == 'R') && - (c[4] == 'b' || c[4] == 'B') && - (c[5] == 'y' || c[5] == 'Y')) { - session->command = KVSTORAGE_CMD_INCR; - session->arg_data.value = 1; - } - else if ((c[0] == 'd' || c[0] == 'D') && - (c[1] == 'e' || c[1] == 'E') && - (c[2] == 'c' || c[2] == 'C') && - (c[3] == 'r' || c[3] == 'R') && - (c[4] == 'b' || c[4] == 'B') && - (c[5] == 'y' || c[5] == 'Y')) { - session->command = KVSTORAGE_CMD_DECR; - session->arg_data.value = -1; - } - else if ((c[0] == 'd' || c[0] == 'D') && - (c[1] == 'e' || c[1] == 'E') && - (c[2] == 'l' || c[2] == 'L') && - (c[3] == 'e' || c[3] == 'E') && - (c[4] == 't' || c[4] == 'T') && - (c[5] == 'e' || c[5] == 'E')) { - session->command = KVSTORAGE_CMD_DELETE; - } - else if (g_ascii_strncasecmp (c, "select", 6) == 0) { - session->command = KVSTORAGE_CMD_SELECT; - } - else { - return FALSE; - } - } - - return TRUE; -} - -/** - * Parse kvstorage line - */ -static gboolean -parse_kvstorage_line (struct kvstorage_session *session, rspamd_fstring_t *in) -{ - gchar *p, *c, *end; - gint state = 0, next_state = 0; - gboolean is_redis; - - p = in->begin; - end = in->begin + in->len; - c = p; - is_redis = session->thr->ctx->is_redis; - - /* State machine for parsing */ - while (p <= end) { - switch (state) { - case 0: - /* At this state we try to read identifier of storage */ - if (g_ascii_isdigit (*p)) { - p++; - } - else { - if (g_ascii_isspace (*p) && p != c) { - /* We have some digits, so parse id */ - session->id = strtoul (c, NULL, 10); - state = 99; - next_state = 1; - } - else if (c == p) { - if (*p != '*') { - /* We have some character, so assume id as 0 and parse command */ - session->id = 0; - state = 1; - } - else { - /* In fact it is redis number of commands */ - c = ++p; - state = 7; - session->id = 0; - } - } - else { - /* We have something wrong here (like some digits and then come non-digits) */ - return FALSE; - } - } - break; - case 1: - /* At this state we parse command */ - if (g_ascii_isalpha (*p) && p != end) { - p++; - } - else { - if (parse_kvstorage_command (session, c, p - c)) { - switch (session->command) { - - case KVSTORAGE_CMD_QUIT: - case KVSTORAGE_CMD_SYNC: - /* Single argument command */ - state = 100; - break; - case KVSTORAGE_CMD_SELECT: - /* Select command, read id next */ - state = 99; - next_state = 6; - break; - default: - /* Normal command, read key */ - state = 99; - next_state = 2; - } - } - else { - /* Some error */ - return FALSE; - } - } - break; - case 2: - /* Read and store key */ - if (!g_ascii_isspace (*p) && end != p) { - p++; - } - else { - if (p == c) { - return FALSE; - } - else { - session->key = rspamd_mempool_alloc (session->pool, - p - c + 1); - rspamd_strlcpy (session->key, c, p - c + 1); - session->keylen = p - c; - /* Now we must select next state based on command */ - if (session->command == KVSTORAGE_CMD_SET || - session->command == KVSTORAGE_CMD_INCR || - session->command == KVSTORAGE_CMD_DECR) { - /* Read flags */ - state = 99; - if (is_redis) { - next_state = 5; - session->flags = 0; - session->expire = 0; - } - else { - if (session->command == KVSTORAGE_CMD_SET) { - next_state = 3; - } - else { - next_state = 5; - } - } - } - else { - /* Nothing to read for other commands */ - state = 100; - } - } - } - break; - case 3: - /* Read flags */ - if (g_ascii_isdigit (*p)) { - p++; - } - else { - if (g_ascii_isspace (*p)) { - session->flags = strtoul (c, NULL, 10); - state = 99; - if (session->command == KVSTORAGE_CMD_SET) { - next_state = 4; - } - else { - /* INCR and DECR */ - next_state = 5; - } - } - else { - return FALSE; - } - } - break; - case 4: - /* Read exptime */ - if (g_ascii_isdigit (*p)) { - p++; - } - else { - if (g_ascii_isspace (*p)) { - session->expire = strtoul (c, NULL, 10); - state = 99; - next_state = 5; - } - else { - return FALSE; - } - } - break; - case 5: - /* Read size or incr/decr values */ - if (g_ascii_isdigit (*p)) { - p++; - } - else { - if (g_ascii_isspace (*p) || p >= end - 1) { - if (session->command == KVSTORAGE_CMD_SET) { - session->arg_data.length = strtoul (c, NULL, 10); - } - else { - if (p != c) { - session->arg_data.value = strtoul (c, NULL, 10); - if (session->command == KVSTORAGE_CMD_DECR) { - session->arg_data.value = - -session->arg_data.value; - } - } - else if (session->command == KVSTORAGE_CMD_INCR) { - session->arg_data.value = 1; - } - else { - session->arg_data.value = -1; - } - } - state = 100; - } - else { - return FALSE; - } - } - break; - case 6: - /* Read index of storage */ - if (g_ascii_isdigit (*p)) { - p++; - } - else { - if (g_ascii_isspace (*p) || end == p) { - session->id = strtoul (c, NULL, 10); - state = 100; - } - else { - return FALSE; - } - } - break; - case 7: - /* Read arguments count */ - if (g_ascii_isdigit (*p)) { - p++; - } - else { - if (g_ascii_isspace (*p) || end == p) { - session->argc = strtoul (c, NULL, 10); - session->argnum = 0; - state = 100; - /* Switch to arglen state */ - session->state = KVSTORAGE_STATE_READ_ARGLEN; - } - else { - return FALSE; - } - } - break; - case 99: - /* Skip spaces state */ - if (g_ascii_isspace (*p)) { - p++; - } - else { - c = p; - state = next_state; - } - break; - case 100: - /* Successful state */ - return TRUE; - break; - } - } - - return state == 100; -} - -/* Process normal kvstorage command */ -static gboolean -kvstorage_process_command (struct kvstorage_session *session, gboolean is_redis) -{ - gint r; - gchar outbuf[BUFSIZ], intbuf[sizeof ("9223372036854775807")]; - gboolean res; - struct rspamd_kv_element *elt; - guint eltlen; - glong longval; - - if (session->command == KVSTORAGE_CMD_SET) { - session->state = KVSTORAGE_STATE_READ_DATA; - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_CHARACTER, - session->arg_data.length); - } - else if (session->command == KVSTORAGE_CMD_GET) { - elt = rspamd_kv_storage_lookup (session->cf->storage, - session->key, - session->keylen, - session->now); - if (elt == NULL) { - RW_R_UNLOCK (&session->cf->storage->rwlock); - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - ERROR_NOT_FOUND, - sizeof (ERROR_NOT_FOUND) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, "$-1" CRLF, - sizeof ("$-1" CRLF) - 1, FALSE, TRUE); - } - } - else { - if (elt->flags & KV_ELT_INTEGER) { - eltlen = rspamd_snprintf (intbuf, - sizeof (intbuf), - "%l", - ELT_LONG (elt)); - - } - else { - eltlen = elt->size; - } - - if (!is_redis) { - r = rspamd_snprintf (outbuf, - sizeof (outbuf), - "VALUE %s %ud %ud" CRLF, - ELT_KEY (elt), - elt->flags, - eltlen); - } - else { - r = rspamd_snprintf (outbuf, sizeof (outbuf), "$%ud" CRLF, - eltlen); - } - if (!rspamd_dispatcher_write (session->dispather, outbuf, - r, TRUE, FALSE)) { - RW_R_UNLOCK (&session->cf->storage->rwlock); - return FALSE; - } - if (elt->flags & KV_ELT_INTEGER) { - if (!rspamd_dispatcher_write (session->dispather, intbuf, - eltlen, TRUE, TRUE)) { - RW_R_UNLOCK (&session->cf->storage->rwlock); - return FALSE; - } - } - else { - if (!rspamd_dispatcher_write (session->dispather, - ELT_DATA (elt), eltlen, TRUE, TRUE)) { - RW_R_UNLOCK (&session->cf->storage->rwlock); - return FALSE; - } - } - session->elt = elt; - if (!is_redis) { - res = rspamd_dispatcher_write (session->dispather, - CRLF "END" CRLF, - sizeof (CRLF "END" CRLF) - 1, - FALSE, - TRUE); - } - else { - res = rspamd_dispatcher_write (session->dispather, CRLF, - sizeof (CRLF) - 1, FALSE, TRUE); - } - if (!res) { - RW_R_UNLOCK (&session->cf->storage->rwlock); - } - - return res; - } - } - else if (session->command == KVSTORAGE_CMD_DELETE) { - elt = rspamd_kv_storage_delete (session->cf->storage, - session->key, - session->keylen); - if (elt != NULL) { - if ((elt->flags & KV_ELT_DIRTY) == 0) { - /* Free memory if backend has deleted this element */ - g_slice_free1 (ELT_SIZE (elt), elt); - } - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - "DELETED" CRLF, - sizeof ("DELETED" CRLF) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, ":1" CRLF, - sizeof (":1" CRLF) - 1, FALSE, TRUE); - } - } - else { - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - ERROR_NOT_FOUND, - sizeof (ERROR_NOT_FOUND) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, ":0" CRLF, - sizeof (":0" CRLF) - 1, FALSE, TRUE); - } - } - } - else if (session->command == KVSTORAGE_CMD_INCR || session->command == - KVSTORAGE_CMD_DECR) { - longval = session->arg_data.value; - if (!rspamd_kv_storage_increment (session->cf->storage, session->key, - session->keylen, &longval)) { - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - ERROR_NOT_FOUND, - sizeof (ERROR_NOT_FOUND) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, - "-ERR not found" CRLF, - sizeof ("-ERR not found" CRLF) - 1, - FALSE, - TRUE); - } - } - else { - if (!is_redis) { - r = rspamd_snprintf (outbuf, sizeof (outbuf), "%l" CRLF, - longval); - } - else { - r = rspamd_snprintf (outbuf, sizeof (outbuf), ":%l" CRLF, - longval); - } - if (!rspamd_dispatcher_write (session->dispather, outbuf, - r, FALSE, FALSE)) { - return FALSE; - } - } - } - else if (session->command == KVSTORAGE_CMD_SYNC) { - if (session->cf->storage->backend == NULL || - session->cf->storage->backend->sync_func == NULL) { - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - ERROR_COMMON, - sizeof (ERROR_COMMON) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, - "-ERR unsupported" CRLF, - sizeof ("-ERR unsupported" CRLF) - 1, - FALSE, - TRUE); - } - } - else { - if (session->cf->storage->backend->sync_func (session->cf->storage-> - backend)) { - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - "SYNCED" CRLF, - sizeof ("SYNCED" CRLF) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, - "+OK" CRLF, - sizeof ("+OK" CRLF) - 1, - FALSE, - TRUE); - } - } - else { - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - "NOT_SYNCED" CRLF, - sizeof ("NOT_SYNCED" CRLF) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, - "-ERR not synced" CRLF, - sizeof ("-ERR not synced" CRLF) - 1, - FALSE, - TRUE); - } - } - } - } - else if (session->command == KVSTORAGE_CMD_SELECT) { - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, "SELECTED" CRLF, - sizeof ("SELECTED" CRLF) - 1, FALSE, TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, "+OK" CRLF, - sizeof ("+OK" CRLF) - 1, FALSE, TRUE); - } - } - else if (session->command == KVSTORAGE_CMD_QUIT) { - /* Quit session */ - free_kvstorage_session (session); - return FALSE; - } - - return TRUE; -} - -static gboolean -kvstorage_read_arglen (rspamd_fstring_t *in, guint *len) -{ - gchar *p = in->begin, *end = in->begin + in->len, *c; - gint state = 0; - - c = p; - while (p < end) { - switch (state) { - case 0: - if (*p != '$') { - return FALSE; - } - else { - p++; - c = p; - state = 1; - } - break; - case 1: - if (g_ascii_isdigit (*p) && p != end - 1) { - p++; - } - else { - if (p != end - 1) { - return FALSE; - } - else { - *len = strtoul (c, NULL, 10); - return TRUE; - } - } - break; - } - } - - return TRUE; -} - -/* - * Check number of arguments for a command - */ -static gboolean -kvstorage_check_argnum (struct kvstorage_session *session) -{ - switch (session->command) { - case KVSTORAGE_CMD_QUIT: - case KVSTORAGE_CMD_SYNC: - return session->argc == 1; - case KVSTORAGE_CMD_SET: - return session->argc == 3 || session->argc == 4; - case KVSTORAGE_CMD_INCR: - case KVSTORAGE_CMD_DECR: - return session->argc == 2 || session->argc == 3; - default: - return session->argc == 2; - } - - /* Unreachable */ - return FALSE; -} - -/** - * Dispatcher callbacks - */ - -/* - * Callback that is called when there is data to read in buffer - */ -static gboolean -kvstorage_read_socket (rspamd_fstring_t * in, void *arg) -{ - struct kvstorage_session *session = (struct kvstorage_session *) arg; - struct kvstorage_worker_thread *thr; - gint r; - guint arglen = 0; - gchar outbuf[BUFSIZ]; - gboolean is_redis; - - if (in->len == 0) { - /* Skip empty commands */ - return TRUE; - } - thr = session->thr; - is_redis = thr->ctx->is_redis; - - switch (session->state) { - case KVSTORAGE_STATE_READ_CMD: - /* Update timestamp */ - session->now = time (NULL); - if (!parse_kvstorage_line (session, in)) { - thr_info ("%ud: unknown command: %V", thr->id, in); - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - ERROR_UNKNOWN_COMMAND, - sizeof (ERROR_UNKNOWN_COMMAND) - 1, - FALSE, - TRUE); - } - else { - r = rspamd_snprintf (outbuf, - sizeof (outbuf), - "-ERR unknown command '%V'" CRLF, - in); - return rspamd_dispatcher_write (session->dispather, outbuf, - r, FALSE, TRUE); - } - } - else { - session->cf = get_kvstorage_config (session->id); - if (session->cf == NULL) { - thr_info ("%ud: bad keystorage: %ud", thr->id, session->id); - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - ERROR_INVALID_KEYSTORAGE, - sizeof (ERROR_INVALID_KEYSTORAGE) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, - "-ERR unknown keystorage" CRLF, - sizeof ("-ERR unknown keystorage" CRLF) - 1, - FALSE, - TRUE); - } - } - if (session->state != KVSTORAGE_STATE_READ_ARGLEN) { - return kvstorage_process_command (session, is_redis); - } - } - break; - case KVSTORAGE_STATE_READ_ARGLEN: - if (!kvstorage_read_arglen (in, &arglen)) { - session->state = KVSTORAGE_STATE_READ_CMD; - r = rspamd_snprintf (outbuf, - sizeof (outbuf), - "-ERR unknown arglen '%V'" CRLF, - in); - return rspamd_dispatcher_write (session->dispather, outbuf, - r, FALSE, TRUE); - } - else { - session->state = KVSTORAGE_STATE_READ_ARG; - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_CHARACTER, - arglen); - } - break; - case KVSTORAGE_STATE_READ_ARG: - if (session->argnum == 0) { - /* Read command */ - if (!parse_kvstorage_command (session, in->begin, in->len)) { - session->state = KVSTORAGE_STATE_READ_CMD; - r = rspamd_snprintf (outbuf, - sizeof (outbuf), - "-ERR unknown command '%V'" CRLF, - in); - return rspamd_dispatcher_write (session->dispather, outbuf, - r, FALSE, TRUE); - } - else { - if (!kvstorage_check_argnum (session)) { - session->state = KVSTORAGE_STATE_READ_CMD; - r = rspamd_snprintf (outbuf, - sizeof (outbuf), - "-ERR invalid argnum for command '%V': %ud" CRLF, - in, - session->argc); - return rspamd_dispatcher_write (session->dispather, outbuf, - r, FALSE, TRUE); - } - else { - if (session->argnum == session->argc - 1) { - session->state = KVSTORAGE_STATE_READ_CMD; - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_LINE, - -1); - return kvstorage_process_command (session, TRUE); - } - else { - session->argnum++; - session->state = KVSTORAGE_STATE_READ_ARGLEN; - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_LINE, - -1); - } - } - } - } - else if (session->argnum == 1) { - if (session->command != KVSTORAGE_CMD_SELECT) { - /* This argument is a key for normal command */ - session->key = rspamd_mempool_fstrdup (session->pool, in); - session->keylen = in->len; - if (session->argnum == session->argc - 1) { - session->state = KVSTORAGE_STATE_READ_CMD; - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_LINE, - -1); - return kvstorage_process_command (session, TRUE); - } - else { - session->argnum++; - session->state = KVSTORAGE_STATE_READ_ARGLEN; - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_LINE, - -1); - } - } - else { - /* Special case for select command */ - session->state = KVSTORAGE_STATE_READ_CMD; - rspamd_strlcpy (outbuf, in->begin, MIN (sizeof (outbuf), - in->len)); - session->id = strtoul (outbuf, NULL, 10); - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_LINE, - -1); - return kvstorage_process_command (session, TRUE); - } - } - else if (session->argnum == 2) { - /* We get datablock for set command */ - if (session->command == KVSTORAGE_CMD_SET && session->argc == 3) { - session->state = KVSTORAGE_STATE_READ_CMD; - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_LINE, - -1); - if (rspamd_kv_storage_insert (session->cf->storage, - session->key, session->keylen, - in->begin, in->len, - session->flags, session->expire)) { - return rspamd_dispatcher_write (session->dispather, - "+OK" CRLF, - sizeof ("+OK" CRLF) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, - "-ERR not stored" CRLF, - sizeof ("-ERR not stored" CRLF) - 1, - FALSE, - TRUE); - } - } - else if (session->command == KVSTORAGE_CMD_SET && session->argc == - 4) { - /* It is expire argument */ - session->state = KVSTORAGE_STATE_READ_CMD; - rspamd_strtol (in->begin, in->len, (glong *)&session->expire); - session->argnum++; - session->state = KVSTORAGE_STATE_READ_ARGLEN; - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_LINE, - -1); - } - else { - session->state = KVSTORAGE_STATE_READ_CMD; - rspamd_strtol (in->begin, in->len, &session->arg_data.value); - if (session->command == KVSTORAGE_CMD_DECR) { - session->arg_data.value = -session->arg_data.value; - } - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_LINE, - -1); - return kvstorage_process_command (session, TRUE); - } - } - else if (session->argnum == 3) { - /* We get datablock for set command */ - if (session->command == KVSTORAGE_CMD_SET && session->argc == 4) { - session->state = KVSTORAGE_STATE_READ_CMD; - rspamd_set_dispatcher_policy (session->dispather, - BUFFER_LINE, - -1); - if (rspamd_kv_storage_insert (session->cf->storage, - session->key, session->keylen, - in->begin, in->len, - session->flags, session->expire)) { - return rspamd_dispatcher_write (session->dispather, - "+OK" CRLF, - sizeof ("+OK" CRLF) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, - "-ERR not stored" CRLF, - sizeof ("-ERR not stored" CRLF) - 1, - FALSE, - TRUE); - } - } - } - break; - case KVSTORAGE_STATE_READ_DATA: - session->state = KVSTORAGE_STATE_READ_CMD; - rspamd_set_dispatcher_policy (session->dispather, BUFFER_LINE, -1); - if (rspamd_kv_storage_insert (session->cf->storage, session->key, - session->keylen, - in->begin, in->len, - session->flags, session->expire)) { - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - "STORED" CRLF, - sizeof ("STORED" CRLF) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, "+OK" CRLF, - sizeof ("+OK" CRLF) - 1, FALSE, TRUE); - } - } - else { - if (!is_redis) { - return rspamd_dispatcher_write (session->dispather, - ERROR_NOT_STORED, - sizeof (ERROR_NOT_STORED) - 1, - FALSE, - TRUE); - } - else { - return rspamd_dispatcher_write (session->dispather, - "-ERR not stored" CRLF, - sizeof ("-ERR not stored" CRLF) - 1, - FALSE, - TRUE); - } - } - - break; - } - - return TRUE; -} - -/* - * Called if buffers were written - */ -static gboolean -kvstorage_write_socket (void *arg) -{ - struct kvstorage_session *session = (struct kvstorage_session *) arg; - - if (session->elt) { - if ((session->elt->flags & KV_ELT_NEED_INSERT) != 0) { - /* Insert to cache and free element */ - session->elt->flags &= ~KV_ELT_NEED_INSERT; - RW_R_UNLOCK (&session->cf->storage->rwlock); - rspamd_kv_storage_insert_cache (session->cf->storage, - ELT_KEY (session->elt), - session->elt->keylen, ELT_DATA (session->elt), - session->elt->size, session->elt->flags, - session->elt->expire, NULL); - g_free (session->elt); - session->elt = NULL; - return TRUE; - } - RW_R_UNLOCK (&session->cf->storage->rwlock); - session->elt = NULL; - - } - - return TRUE; -} - -/* - * Called if something goes wrong - */ -static void -kvstorage_err_socket (GError * err, void *arg) -{ - struct kvstorage_session *session = (struct kvstorage_session *) arg; - struct kvstorage_worker_thread *thr; - - thr = session->thr; - if (err->code != -1) { - thr_info ("%ud: abnormally closing connection from: %s, error: %s", - thr->id, inet_ntoa (session->client_addr), err->message); - } - - if (session->elt) { - RW_R_UNLOCK (&session->cf->storage->rwlock); - session->elt = NULL; - } - - g_error_free (err); - free_kvstorage_session (session); -} - -/** - * Accept function - */ -static void -thr_accept_socket (gint fd, short what, void *arg) -{ - struct kvstorage_worker_thread *thr = (struct kvstorage_worker_thread *)arg; - union sa_union su; - socklen_t addrlen = sizeof (su.ss); - gint nfd; - struct kvstorage_session *session; - - g_mutex_lock (thr->accept_mtx); - if ((nfd = - accept_from_socket (fd, (struct sockaddr *)&su.ss, &addrlen)) == -1) { - thr_warn ("%ud: accept failed: %s", thr->id, strerror (errno)); - g_mutex_unlock (thr->accept_mtx); - return; - } - - /* Check for EAGAIN */ - if (nfd == 0) { - g_mutex_unlock (thr->accept_mtx); - return; - } - - session = g_slice_alloc0 (sizeof (struct kvstorage_session)); - session->pool = rspamd_mempool_new (rspamd_mempool_suggest_size ()); - session->state = KVSTORAGE_STATE_READ_CMD; - session->thr = thr; - session->sock = nfd; - session->dispather = rspamd_create_dispatcher (thr->ev_base, - nfd, - BUFFER_LINE, - kvstorage_read_socket, - kvstorage_write_socket, - kvstorage_err_socket, - thr->tv, - session); - - g_mutex_unlock (thr->accept_mtx); - session->elt = NULL; - - if (su.ss.ss_family == AF_UNIX) { - session->client_addr.s_addr = INADDR_NONE; - } - else if (su.ss.ss_family == AF_INET) { - memcpy (&session->client_addr, &su.s4.sin_addr, - sizeof (struct in_addr)); - } -} - -/** - * Handle termination - */ -static void -thr_term_socket (gint fd, short what, void *arg) -{ - struct kvstorage_worker_thread *thr = (struct kvstorage_worker_thread *)arg; - struct timeval tv; - - if (read (fd, &tv, sizeof (struct timeval)) != sizeof (struct timeval)) { - thr_err ("cannot read data from socket: %s", strerror (errno)); - tv.tv_sec = 0; - tv.tv_usec = 0; - } - - event_base_loopexit (thr->ev_base, &tv); - event_del (&thr->bind_ev); -} - -/** - * Thread main worker function - */ -static gpointer -kvstorage_thread (gpointer ud) -{ - struct kvstorage_worker_thread *thr = ud; - - /* Block signals as it is dispatcher deity */ - sigprocmask (SIG_BLOCK, thr->signals, NULL); - /* Init thread specific events */ - thr->ev_base = event_init (); - - event_set (&thr->bind_ev, - thr->worker->cf->listen_sock, - EV_READ | EV_PERSIST, - thr_accept_socket, - (void *)thr); - event_base_set (thr->ev_base, &thr->bind_ev); - event_add (&thr->bind_ev, NULL); - - event_set (&thr->term_ev, - thr->term_sock[0], - EV_READ | EV_PERSIST, - thr_term_socket, - (void *)thr); - event_base_set (thr->ev_base, &thr->term_ev); - event_add (&thr->term_ev, NULL); - - event_base_loop (thr->ev_base, 0); - - return NULL; -} - -/** - * Create new thread, set it detached - */ -static struct kvstorage_worker_thread * -create_kvstorage_thread (struct rspamd_worker *worker, - struct kvstorage_worker_ctx *ctx, - guint id, - sigset_t *signals) -{ - struct kvstorage_worker_thread *new; - GError *err = NULL; - - new = - rspamd_mempool_alloc (ctx->pool, - sizeof (struct kvstorage_worker_thread)); - new->ctx = ctx; - new->worker = worker; - new->tv = &ctx->io_timeout; - new->log_mtx = ctx->log_mtx; - new->accept_mtx = ctx->accept_mtx; - new->id = id; - - /* Create and setup terminating socket */ - if (rspamd_socketpair (new->term_sock) == -1) { - msg_err ("socket failed: %s", strerror (errno)); - return NULL; - } - rspamd_socket_nonblocking (new->term_sock[0]); - -#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30)) - new->thr = g_thread_create (kvstorage_thread, new, FALSE, &err); -#else - gchar *name; - - name = rspamd_mempool_alloc (ctx->pool, - sizeof ("kvstorage_thread") + sizeof ("4294967296") - 1); - rspamd_snprintf (name, - sizeof ("kvstorage_thread") + sizeof ("4294967296") - 1, - "kvstorage_thread%d", - id); - - new->thr = g_thread_new (name, kvstorage_thread, new); -#endif - new->ev_base = NULL; - new->signals = signals; - - if (new->thr == NULL) { - msg_err ("cannot create thread: %s", err->message); - } - - return new; -} - -/* - * Start worker process - */ -void -start_keystorage (struct rspamd_worker *worker) -{ - struct sigaction signals; - struct kvstorage_worker_ctx *ctx = worker->ctx; - guint i; - struct kvstorage_worker_thread *thr; - struct timeval tv; - GList *cur; - - gperf_profiler_init (worker->srv->cfg, "kvstorage"); - - if (!g_thread_supported ()) { - msg_err ( - "threads support is not supported on your system so kvstorage is not functionable"); - exit (EXIT_SUCCESS); - } - /* Create socketpair */ - if (rspamd_socketpair (ctx->s_pair) == -1) { - msg_err ("cannot create socketpair, exiting"); - exit (EXIT_SUCCESS); - } - worker->srv->pid = getpid (); - ctx->threads = NULL; - -#if _EVENT_NUMERIC_VERSION > 0x02000000 - if (evthread_use_pthreads () == -1) { - msg_err ( - "threads support is not supported in your libevent so kvstorage is not functionable"); - exit (EXIT_SUCCESS); - } -#endif - - /* Set kvstorage options */ - if ( !config_kvstorage_worker (worker)) { - msg_err ("cannot configure kvstorage worker, exiting"); - exit (EXIT_SUCCESS); - } - - rspamd_signals_init (&signals, sig_handler); - - /* Set umask */ - umask (S_IWGRP | S_IWOTH | S_IROTH | S_IRGRP); - - /* Init mutexes */ -#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30)) - ctx->log_mtx = g_mutex_new (); - ctx->accept_mtx = g_mutex_new (); -#else - ctx->log_mtx = rspamd_mempool_alloc (ctx->pool, sizeof (GMutex)); - ctx->accept_mtx = rspamd_mempool_alloc (ctx->pool, sizeof (GMutex)); - g_mutex_init (ctx->log_mtx); - g_mutex_init (ctx->accept_mtx); -#endif - - /* Start workers threads */ - for (i = 0; i < worker->cf->count; i++) { - thr = create_kvstorage_thread (worker, ctx, i, &signals.sa_mask); - if (thr != NULL) { - ctx->threads = g_list_prepend (ctx->threads, thr); - } - } - - sigprocmask (SIG_BLOCK, &signals.sa_mask, NULL); - /* Signal processing cycle */ - for (;; ) { - msg_debug ("calling sigsuspend"); - sigemptyset (&signals.sa_mask); - sigsuspend (&signals.sa_mask); - if (wanna_die == 1) { - wanna_die = 0; - tv.tv_sec = 0; - tv.tv_usec = 0; - msg_info ("worker's immediately shutdown is requested"); - cur = ctx->threads; - while (cur) { - thr = cur->data; - while (write (thr->term_sock[1], &tv, - sizeof (struct timeval)) == -1) { - if (errno != EAGAIN) { - msg_err ("write to term socket failed: %s", - strerror (errno)); - abort (); - } - } - cur = g_list_next (cur); - } - break; - } - else if (soft_wanna_die == 1) { - soft_wanna_die = 0; - tv.tv_sec = SOFT_SHUTDOWN_TIME; - tv.tv_usec = 0; - msg_info ("worker's shutdown is pending in %d sec", - SOFT_SHUTDOWN_TIME); - cur = ctx->threads; - while (cur) { - thr = cur->data; - while (write (thr->term_sock[1], &tv, - sizeof (struct timeval)) == -1) { - if (errno != EAGAIN) { - msg_err ("write to term socket failed: %s", - strerror (errno)); - abort (); - } - } - cur = g_list_next (cur); - } - break; - } - else if (do_reopen_log == 1) { - do_reopen_log = 0; - rspamd_log_reopen (rspamd_main->logger); - } - } - - msg_info ("syncing storages"); - /* Wait for threads in the recent glib */ -#if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION > 30)) - cur = ctx->threads; - while (cur) { - thr = cur->data; - (void)g_thread_join (thr->thr); - cur = g_list_next (cur); - } -#endif - destroy_kvstorage_config (); - rspamd_log_close (rspamd_main->logger); - exit (EXIT_SUCCESS); -} diff --git a/src/kvstorage_server.h b/src/kvstorage_server.h deleted file mode 100644 index 7af9d79ba..000000000 --- a/src/kvstorage_server.h +++ /dev/null @@ -1,98 +0,0 @@ -/* Copyright (c) 2010, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - -#ifndef KVSTORAGE_SERVER_H_ -#define KVSTORAGE_SERVER_H_ - -#include "config.h" -#include "mem_pool.h" -#include "buffer.h" - -/* Configuration context for kvstorage worker */ -struct kvstorage_worker_ctx { - struct timeval io_timeout; - guint32 timeout_raw; - GList *threads; - gint s_pair[2]; - gboolean is_redis; - rspamd_mempool_t *pool; - struct event_base *ev_base; - GMutex *log_mtx; - GMutex *accept_mtx; -}; - -struct kvstorage_worker_thread { - struct event bind_ev; - struct event term_ev; - struct timeval *tv; - struct kvstorage_worker_ctx *ctx; - struct rspamd_worker *worker; - GThread *thr; - struct event_base *ev_base; - GMutex *log_mtx; - GMutex *accept_mtx; - guint id; - sigset_t *signals; - gint term_sock[2]; -}; - -struct kvstorage_session { - rspamd_io_dispatcher_t *dispather; - enum { - KVSTORAGE_STATE_READ_CMD, - KVSTORAGE_STATE_READ_ARGLEN, - KVSTORAGE_STATE_READ_ARG, - KVSTORAGE_STATE_READ_DATA - } state; - enum { - KVSTORAGE_CMD_SET, - KVSTORAGE_CMD_GET, - KVSTORAGE_CMD_DELETE, - KVSTORAGE_CMD_SYNC, - KVSTORAGE_CMD_SELECT, - KVSTORAGE_CMD_INCR, - KVSTORAGE_CMD_DECR, - KVSTORAGE_CMD_QUIT - } command; - guint id; - guint argc; - guint argnum; - rspamd_mempool_t *pool; - gchar *key; - guint keylen; - struct kvstorage_config *cf; - struct kvstorage_worker_thread *thr; - struct rspamd_kv_element *elt; - struct in_addr client_addr; - gint sock; - guint flags; - guint expire; - union { - glong value; - guint length; - } arg_data; - time_t now; -}; - -#endif /* KVSTORAGE_SERVER_H_ */ diff --git a/src/kvstorage_sqlite.c b/src/kvstorage_sqlite.c deleted file mode 100644 index c9b504974..000000000 --- a/src/kvstorage_sqlite.c +++ /dev/null @@ -1,507 +0,0 @@ -/* Copyright (c) 2010, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - -#include "config.h" -#include "kvstorage.h" -#include "kvstorage_sqlite.h" -#include "util.h" -#include "main.h" -#include - -#define TABLE_NAME "kvstorage" -#define CREATE_TABLE_SQL "CREATE TABLE " TABLE_NAME \ - " (key TEXT CONSTRAINT _key PRIMARY KEY, data BLOB)" -#define SET_SQL "INSERT OR REPLACE INTO " TABLE_NAME \ - " (key, data) VALUES (?1, ?2)" -#define GET_SQL "SELECT data FROM " TABLE_NAME " WHERE key = ?1" -#define DELETE_SQL "DELETE FROM " TABLE_NAME " WHERE key = ?1" - -struct sqlite_op { - struct rspamd_kv_element *elt; - enum { - SQLITE_OP_INSERT, - SQLITE_OP_DELETE, - SQLITE_OP_REPLACE - } op; -}; - -/* Main sqlite structure */ -struct rspamd_sqlite_backend { - backend_init init_func; /*< this callback is called on kv storage initialization */ - backend_insert insert_func; /*< this callback is called when element is inserted */ - backend_replace replace_func; /*< this callback is called when element is replaced */ - backend_lookup lookup_func; /*< this callback is used for lookup of element */ - backend_delete delete_func; /*< this callback is called when an element is deleted */ - backend_sync sync_func; /*< this callback is called when backend need to be synced */ - backend_destroy destroy_func; /*< this callback is used for destroying all elements inside backend */ - sqlite3 *dbp; - gchar *filename; - gchar *dirname; - guint sync_ops; - GQueue *ops_queue; - GHashTable *ops_hash; - gboolean initialized; - sqlite3_stmt *get_stmt; - sqlite3_stmt *set_stmt; - sqlite3_stmt *delete_stmt; -}; - -/* Process single sqlite operation */ -static gboolean -sqlite_process_single_op (struct rspamd_sqlite_backend *db, - struct sqlite_op *op) -{ - gboolean res = FALSE; - - op->elt->flags &= ~KV_ELT_DIRTY; - switch (op->op) { - case SQLITE_OP_INSERT: - case SQLITE_OP_REPLACE: - if (sqlite3_bind_text (db->set_stmt, 1, ELT_KEY (op->elt), - op->elt->keylen, SQLITE_STATIC) == SQLITE_OK && - sqlite3_bind_blob (db->set_stmt, 2, op->elt, ELT_SIZE (op->elt), - SQLITE_STATIC) == SQLITE_OK) { - if (sqlite3_step (db->set_stmt) == SQLITE_DONE) { - res = TRUE; - } - } - sqlite3_reset (db->set_stmt); - break; - case SQLITE_OP_DELETE: - if (sqlite3_bind_text (db->delete_stmt, 1, ELT_KEY (op->elt), - op->elt->keylen, SQLITE_STATIC) == SQLITE_OK) { - if (sqlite3_step (db->delete_stmt) == SQLITE_DONE) { - res = TRUE; - } - } - sqlite3_reset (db->delete_stmt); - break; - } - - if (!res) { - op->elt->flags |= KV_ELT_DIRTY; - } - return res; -} - -/* Process operations queue */ -static gboolean -sqlite_process_queue (struct rspamd_kv_backend *backend) -{ - struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; - struct sqlite_op *op; - GList *cur; - - cur = db->ops_queue->head; - while (cur) { - op = cur->data; - if (!sqlite_process_single_op (db, op)) { - return FALSE; - } - cur = g_list_next (cur); - } - - /* Clean the queue */ - cur = db->ops_queue->head; - while (cur) { - op = cur->data; - if (op->op == SQLITE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != - 0) { - /* Also clean memory */ - g_slice_free1 (ELT_SIZE (op->elt), op->elt); - } - g_slice_free1 (sizeof (struct sqlite_op), op); - cur = g_list_next (cur); - } - - g_hash_table_remove_all (db->ops_hash); - g_queue_clear (db->ops_queue); - - return TRUE; - -} - -/* Create table for kvstorage */ -static gboolean -rspamd_sqlite_create_table (struct rspamd_sqlite_backend *db) -{ - gint ret; - sqlite3_stmt *stmt = NULL; - - ret = - sqlite3_prepare_v2 (db->dbp, - CREATE_TABLE_SQL, - sizeof (CREATE_TABLE_SQL) - 1, - &stmt, - NULL); - if (ret != SQLITE_OK) { - if (stmt != NULL) { - sqlite3_finalize (stmt); - } - return FALSE; - } - - ret = sqlite3_step (stmt); - if (ret != SQLITE_DONE) { - sqlite3_finalize (stmt); - return FALSE; - } - - sqlite3_finalize (stmt); - return TRUE; -} - -/* Backend callbacks */ -static void -rspamd_sqlite_init (struct rspamd_kv_backend *backend) -{ - struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; - guint32 flags; - gint ret, r; - gchar sqlbuf[BUFSIZ]; - sqlite3_stmt *stmt = NULL; - - /* Set multi-threaded mode */ - if (sqlite3_config (SQLITE_CONFIG_MULTITHREAD) != SQLITE_OK) { - goto err; - } - - flags = SQLITE_OPEN_READWRITE | - SQLITE_OPEN_CREATE | - SQLITE_OPEN_NOMUTEX; - - ret = sqlite3_open_v2 (db->filename, &db->dbp, flags, NULL); - - if (ret != 0) { - goto err; - } - /* Now check if we have table */ - r = rspamd_snprintf (sqlbuf, - sizeof (sqlbuf), - "SELECT * FROM " TABLE_NAME " LIMIT 1"); - ret = sqlite3_prepare_v2 (db->dbp, sqlbuf, r, &stmt, NULL); - - if (ret == SQLITE_ERROR) { - /* Try to create table */ - if (!rspamd_sqlite_create_table (db)) { - goto err; - } - } - else if (ret != SQLITE_OK) { - goto err; - } - /* We have table here, perform vacuum */ - sqlite3_finalize (stmt); - r = rspamd_snprintf (sqlbuf, sizeof (sqlbuf), "VACUUM"); - ret = sqlite3_prepare_v2 (db->dbp, sqlbuf, r, &stmt, NULL); - if (ret != SQLITE_OK) { - goto err; - } - /* Perform VACUUM */ - sqlite3_step (stmt); - sqlite3_finalize (stmt); - - /* Prepare required statements */ - ret = sqlite3_prepare_v2 (db->dbp, - GET_SQL, - sizeof (GET_SQL) - 1, - &db->get_stmt, - NULL); - if (ret != SQLITE_OK) { - goto err; - } - ret = sqlite3_prepare_v2 (db->dbp, - SET_SQL, - sizeof (SET_SQL) - 1, - &db->set_stmt, - NULL); - if (ret != SQLITE_OK) { - goto err; - } - ret = sqlite3_prepare_v2 (db->dbp, - DELETE_SQL, - sizeof (DELETE_SQL) - 1, - &db->delete_stmt, - NULL); - if (ret != SQLITE_OK) { - goto err; - } - - db->initialized = TRUE; - - return; -err: - if (db->dbp != NULL) { - msg_err ("error opening sqlite database: %d", ret); - } - if (stmt != NULL) { - msg_err ("error executing statement: %d", ret); - sqlite3_finalize (stmt); - } - if (db->get_stmt != NULL) { - sqlite3_finalize (db->get_stmt); - } - if (db->set_stmt != NULL) { - sqlite3_finalize (db->set_stmt); - } - if (db->delete_stmt != NULL) { - sqlite3_finalize (db->delete_stmt); - } -} - -static gboolean -rspamd_sqlite_insert (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt) -{ - struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; - struct sqlite_op *op; - struct rspamd_kv_element search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - if (!db->initialized) { - return FALSE; - } - - if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { - /* We found another op with such key in this queue */ - if (op->op == SQLITE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != - 0) { - /* Also clean memory */ - g_slice_free1 (ELT_SIZE (op->elt), op->elt); - } - op->op = SQLITE_OP_INSERT; - op->elt = elt; - } - else { - op = g_slice_alloc (sizeof (struct sqlite_op)); - op->op = SQLITE_OP_INSERT; - op->elt = elt; - elt->flags |= KV_ELT_DIRTY; - - g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, elt, op); - } - - if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= - db->sync_ops) { - return sqlite_process_queue (backend); - } - - return TRUE; -} - -static gboolean -rspamd_sqlite_replace (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen, - struct rspamd_kv_element *elt) -{ - struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; - struct sqlite_op *op; - - if (!db->initialized) { - return FALSE; - } - if ((op = g_hash_table_lookup (db->ops_hash, elt)) != NULL) { - /* We found another op with such key in this queue */ - if (op->op == SQLITE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != - 0) { - /* Also clean memory */ - g_slice_free1 (ELT_SIZE (op->elt), op->elt); - } - op->op = SQLITE_OP_REPLACE; - op->elt = elt; - } - else { - op = g_slice_alloc (sizeof (struct sqlite_op)); - op->op = SQLITE_OP_REPLACE; - op->elt = elt; - elt->flags |= KV_ELT_DIRTY; - - g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, elt, op); - } - - if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= - db->sync_ops) { - return sqlite_process_queue (backend); - } - - return TRUE; -} - -static struct rspamd_kv_element * -rspamd_sqlite_lookup (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen) -{ - struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; - struct sqlite_op *op; - struct rspamd_kv_element *elt = NULL; - gint l; - gconstpointer d; - struct rspamd_kv_element search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - if (!db->initialized) { - return NULL; - } - /* First search in ops queue */ - if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { - if (op->op == SQLITE_OP_DELETE) { - /* To delete, so assume it as not found */ - return NULL; - } - return op->elt; - } - - if (sqlite3_bind_text (db->get_stmt, 1, key, keylen, - SQLITE_STATIC) == SQLITE_OK) { - if (sqlite3_step (db->get_stmt) == SQLITE_ROW) { - l = sqlite3_column_bytes (db->get_stmt, 0); - elt = g_malloc (l); - d = sqlite3_column_blob (db->get_stmt, 0); - /* Make temporary copy */ - memcpy (elt, d, l); - } - } - - sqlite3_reset (db->get_stmt); - return elt; -} - -static void -rspamd_sqlite_delete (struct rspamd_kv_backend *backend, - gpointer key, - guint keylen) -{ - struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; - struct sqlite_op *op; - struct rspamd_kv_element *elt; - struct rspamd_kv_element search_elt; - - search_elt.keylen = keylen; - search_elt.p = key; - - if (!db->initialized) { - return; - } - - if ((op = g_hash_table_lookup (db->ops_hash, &search_elt)) != NULL) { - op->op = SQLITE_OP_DELETE; - return; - } - - elt = rspamd_sqlite_lookup (backend, key, keylen); - if (elt == NULL) { - return; - } - op = g_slice_alloc (sizeof (struct sqlite_op)); - op->op = SQLITE_OP_DELETE; - op->elt = elt; - elt->flags |= KV_ELT_DIRTY; - - g_queue_push_head (db->ops_queue, op); - g_hash_table_insert (db->ops_hash, elt, op); - - if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= - db->sync_ops) { - sqlite_process_queue (backend); - } - - return; -} - -static void -rspamd_sqlite_destroy (struct rspamd_kv_backend *backend) -{ - struct rspamd_sqlite_backend *db = (struct rspamd_sqlite_backend *)backend; - - if (db->initialized) { - sqlite_process_queue (backend); - if (db->get_stmt != NULL) { - sqlite3_finalize (db->get_stmt); - } - if (db->set_stmt != NULL) { - sqlite3_finalize (db->set_stmt); - } - if (db->delete_stmt != NULL) { - sqlite3_finalize (db->delete_stmt); - } - sqlite3_close (db->dbp); - g_free (db->filename); - g_free (db->dirname); - g_queue_free (db->ops_queue); - g_hash_table_unref (db->ops_hash); - g_slice_free1 (sizeof (struct rspamd_sqlite_backend), db); - } -} - -/* Create new sqlite backend */ -struct rspamd_kv_backend * -rspamd_kv_sqlite_new (const gchar *filename, guint sync_ops) -{ - struct rspamd_sqlite_backend *new; - struct stat st; - gchar *dirname; - - if (filename == NULL) { - return NULL; - } - - dirname = g_path_get_dirname (filename); - if (dirname == NULL || stat (dirname, &st) == -1 || !S_ISDIR (st.st_mode)) { - /* Inaccessible path */ - if (dirname != NULL) { - g_free (dirname); - } - msg_err ("invalid file: %s", filename); - return NULL; - } - - new = g_slice_alloc0 (sizeof (struct rspamd_sqlite_backend)); - new->dirname = dirname; - new->filename = g_strdup (filename); - new->sync_ops = sync_ops; - new->ops_queue = g_queue_new (); - new->ops_hash = g_hash_table_new (kv_elt_hash_func, kv_elt_compare_func); - - /* Init callbacks */ - new->init_func = rspamd_sqlite_init; - new->insert_func = rspamd_sqlite_insert; - new->lookup_func = rspamd_sqlite_lookup; - new->delete_func = rspamd_sqlite_delete; - new->replace_func = rspamd_sqlite_replace; - new->sync_func = sqlite_process_queue; - new->destroy_func = rspamd_sqlite_destroy; - - return (struct rspamd_kv_backend *)new; -} - diff --git a/src/kvstorage_sqlite.h b/src/kvstorage_sqlite.h deleted file mode 100644 index 0ed0fd1ea..000000000 --- a/src/kvstorage_sqlite.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (c) 2010, Vsevolod Stakhov - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Rambler BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - - -#ifndef KVSTORAGE_SQLITE_H_ -#define KVSTORAGE_SQLITE_H_ - -#include "config.h" -#include "kvstorage.h" - -#ifdef WITH_SQLITE - -/* Create new sqlite backend */ -struct rspamd_kv_backend * rspamd_kv_sqlite_new (const gchar *filename, - guint sync_ops); - -#endif - -#endif /* KVSTORAGE_SQLITE_H_ */ diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index 776d00670..068ced265 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -30,7 +30,6 @@ #include "uthash_strcase.h" #include "filter.h" #include "lua/lua_common.h" -#include "kvstorage_config.h" #include "map.h" #include "dynamic_cfg.h" #include "utlist.h" diff --git a/src/main.c b/src/main.c index 8191982b5..1686c39b4 100644 --- a/src/main.c +++ b/src/main.c @@ -28,7 +28,6 @@ #include "smtp.h" #include "libutil/map.h" #include "fuzzy_storage.h" -#include "kvstorage_server.h" #include "libserver/symbols_cache.h" #include "lua/lua_common.h" #include "libserver/worker_util.h" -- 2.39.5