diff options
| -rw-r--r-- | lib/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/kvstorage_config.c | 9 | ||||
| -rw-r--r-- | src/kvstorage_config.h | 1 | ||||
| -rw-r--r-- | src/kvstorage_file.c | 353 | ||||
| -rw-r--r-- | src/kvstorage_file.h | 35 | ||||
| -rw-r--r-- | src/kvstorage_sqlite.h | 2 |
6 files changed, 400 insertions, 1 deletions
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 2b762b6c0..f2aa2e5f2 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -49,6 +49,7 @@ SET(RSPAMDLIBSRC ../src/binlog.c ../src/images.c ../src/kvstorage.c ../src/kvstorage_config.c + ../src/kvstorage_file.c ../src/lmtp_proto.c ../src/logger.c ../src/map.c diff --git a/src/kvstorage_config.c b/src/kvstorage_config.c index 8097078a4..218d26953 100644 --- a/src/kvstorage_config.c +++ b/src/kvstorage_config.c @@ -30,6 +30,9 @@ #ifdef WITH_SQLITE #include "kvstorage_sqlite.h" #endif +#include "kvstorage_file.h" + +#define FILE_STORAGE_LEVELS 3 /* Global hash of storages indexed by id */ @@ -106,6 +109,9 @@ kvstorage_init_callback (const gpointer key, const gpointer value, gpointer unus case KVSTORAGE_TYPE_BACKEND_MAX: backend = NULL; break; + case KVSTORAGE_TYPE_BACKEND_FILE: + backend = rspamd_kv_file_new (kconf->backend.filename, kconf->backend.sync_ops, FILE_STORAGE_LEVELS); + break; #ifdef WITH_DB case KVSTORAGE_TYPE_BACKEND_BDB: backend = rspamd_kv_bdb_new (kconf->backend.filename, kconf->backend.sync_ops); @@ -383,6 +389,9 @@ void kvstorage_xml_text (GMarkupParseContext *context, if (g_ascii_strncasecmp (text, "null", MIN (text_len, sizeof ("null") - 1)) == 0) { kv_parser->current_storage->backend.type = KVSTORAGE_TYPE_BACKEND_NULL; } + else if (g_ascii_strncasecmp (text, "file", MIN (text_len, sizeof ("file") - 1)) == 0) { + kv_parser->current_storage->backend.type = KVSTORAGE_TYPE_BACKEND_FILE; + } #ifdef WITH_DB else if (g_ascii_strncasecmp (text, "bdb", MIN (text_len, sizeof ("bdb") - 1)) == 0) { kv_parser->current_storage->backend.type = KVSTORAGE_TYPE_BACKEND_BDB; diff --git a/src/kvstorage_config.h b/src/kvstorage_config.h index 74adc0642..11b682506 100644 --- a/src/kvstorage_config.h +++ b/src/kvstorage_config.h @@ -41,6 +41,7 @@ enum kvstorage_cache_type { /* Type of kvstorage backend */ enum kvstorage_backend_type { KVSTORAGE_TYPE_BACKEND_NULL = 0, + KVSTORAGE_TYPE_BACKEND_FILE, #ifdef WITH_DB KVSTORAGE_TYPE_BACKEND_BDB, #endif diff --git a/src/kvstorage_file.c b/src/kvstorage_file.c new file mode 100644 index 000000000..a7343aab9 --- /dev/null +++ b/src/kvstorage_file.c @@ -0,0 +1,353 @@ +/* Copyright (c) 2010, Vsevolod Stakhov + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#include "config.h" +#include "kvstorage.h" +#include "kvstorage_file.h" +#include "util.h" +#include "main.h" + +struct file_op { + struct rspamd_kv_element *elt; + enum { + FILE_OP_INSERT, + FILE_OP_DELETE, + FILE_OP_REPLACE + } op; +}; + +/* Main file structure */ +struct rspamd_file_backend { + backend_init init_func; /*< this callback is called on kv storage initialization */ + backend_insert insert_func; /*< this callback is called when element is inserted */ + backend_replace replace_func; /*< this callback is called when element is replaced */ + backend_lookup lookup_func; /*< this callback is used for lookup of element */ + backend_delete delete_func; /*< this callback is called when an element is deleted */ + backend_sync sync_func; /*< this callback is called when backend need to be synced */ + backend_destroy destroy_func; /*< this callback is used for destroying all elements inside backend */ + gchar *filename; + gchar *dirname; + guint sync_ops; + guint levels; + GQueue *ops_queue; + GHashTable *ops_hash; + gboolean initialized; +}; + +static const gchar hexdigits[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + +/* Process single file operation */ +static gboolean +file_process_single_op (struct rspamd_file_backend *db, struct file_op *op) +{ + gboolean res = FALSE; + + return res; +} + +/* Process operations queue */ +static gboolean +file_process_queue (struct rspamd_kv_backend *backend) +{ + struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; + struct file_op *op; + GList *cur; + + cur = db->ops_queue->head; + while (cur) { + op = cur->data; + if (! file_process_single_op (db, op)) { + return FALSE; + } + cur = g_list_next (cur); + } + + /* Clean the queue */ + cur = db->ops_queue->head; + while (cur) { + op = cur->data; + if (op->op == FILE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != 0) { + /* Also clean memory */ + g_slice_free1 (ELT_SIZE (op->elt), op->elt); + } + g_slice_free1 (sizeof (struct file_op), op); + cur = g_list_next (cur); + } + + g_hash_table_remove_all (db->ops_hash); + g_queue_clear (db->ops_queue); + + return TRUE; + +} + + +/* Make 16 directories for each level */ +static gboolean +rspamd_recursive_mkdir (guint levels) +{ + guint i, j; + gchar nbuf[5]; + + /* Create directories for backend */ + if (levels > 0) { + /* Create 16 directories */ + for (j = 0; j < 16; j ++) { + rspamd_snprintf (nbuf, sizeof (nbuf), "./%c", hexdigits[j]); + if (mkdir (nbuf, 0755) != 0 && errno != EEXIST) { + return FALSE; + } + else if (levels > 1) { + chdir (nbuf); + if (! rspamd_recursive_mkdir (levels - 1)) { + return FALSE; + } + chdir ("../"); + } + } + } + return TRUE; + +} + +/* Backend callbacks */ +static void +rspamd_file_init (struct rspamd_kv_backend *backend) +{ + struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; + gint ret; + gchar pathbuf[PATH_MAX]; + + /* Save current directory */ + if (getcwd (pathbuf, sizeof (pathbuf) - 1) == NULL) { + pathbuf[0] = '\0'; + msg_err ("getcwd failed: %s", strerror (errno)); + goto err; + } + + /* Chdir to the working dir */ + if (chdir (db->dirname) == -1) { + msg_err ("chdir failed: %s", strerror (errno)); + goto err; + } + + /* Create directories for backend */ + rspamd_recursive_mkdir (db->levels); + + db->initialized = TRUE; + + chdir (pathbuf); + return; +err: + if (pathbuf[0] != '\0') { + chdir (pathbuf); + } +} + +static gboolean +rspamd_file_insert (struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt) +{ + struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; + struct file_op *op; + + if (!db->initialized) { + return FALSE; + } + + if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + /* We found another op with such key in this queue */ + if (op->op == FILE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != 0) { + /* Also clean memory */ + g_slice_free1 (ELT_SIZE (op->elt), op->elt); + } + op->op = FILE_OP_INSERT; + op->elt = elt; + } + else { + op = g_slice_alloc (sizeof (struct file_op)); + op->op = FILE_OP_INSERT; + op->elt = elt; + elt->flags |= KV_ELT_DIRTY; + + g_queue_push_head (db->ops_queue, op); + g_hash_table_insert (db->ops_hash, ELT_KEY (elt), op); + } + + if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { + return file_process_queue (backend); + } + + return TRUE; +} + +static gboolean +rspamd_file_replace (struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt) +{ + struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; + struct file_op *op; + + if (!db->initialized) { + return FALSE; + } + if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + /* We found another op with such key in this queue */ + if (op->op == FILE_OP_DELETE || (op->elt->flags & KV_ELT_NEED_FREE) != 0) { + /* Also clean memory */ + g_slice_free1 (ELT_SIZE (op->elt), op->elt); + } + op->op = FILE_OP_REPLACE; + op->elt = elt; + } + else { + op = g_slice_alloc (sizeof (struct file_op)); + op->op = FILE_OP_REPLACE; + op->elt = elt; + elt->flags |= KV_ELT_DIRTY; + + g_queue_push_head (db->ops_queue, op); + g_hash_table_insert (db->ops_hash, ELT_KEY (elt), op); + } + + if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { + return file_process_queue (backend); + } + + return TRUE; +} + +static struct rspamd_kv_element* +rspamd_file_lookup (struct rspamd_kv_backend *backend, gpointer key) +{ + struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; + struct file_op *op; + struct rspamd_kv_element *elt = NULL; + gint l; + gconstpointer d; + + if (!db->initialized) { + return NULL; + } + /* First search in ops queue */ + if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + if (op->op == FILE_OP_DELETE) { + /* To delete, so assume it as not found */ + return NULL; + } + return op->elt; + } + return elt; +} + +static void +rspamd_file_delete (struct rspamd_kv_backend *backend, gpointer key) +{ + struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; + struct file_op *op; + struct rspamd_kv_element *elt; + + if (!db->initialized) { + return; + } + + if ((op = g_hash_table_lookup (db->ops_hash, key)) != NULL) { + op->op = FILE_OP_DELETE; + return; + } + + elt = rspamd_file_lookup (backend, key); + if (elt == NULL) { + return; + } + op = g_slice_alloc (sizeof (struct file_op)); + op->op = FILE_OP_DELETE; + op->elt = elt; + elt->flags |= KV_ELT_DIRTY; + + g_queue_push_head (db->ops_queue, op); + g_hash_table_insert (db->ops_hash, ELT_KEY(elt), op); + + if (db->sync_ops > 0 && g_queue_get_length (db->ops_queue) >= db->sync_ops) { + file_process_queue (backend); + } + + return; +} + +static void +rspamd_file_destroy (struct rspamd_kv_backend *backend) +{ + struct rspamd_file_backend *db = (struct rspamd_file_backend *)backend; + + if (db->initialized) { + file_process_queue (backend); + g_free (db->filename); + g_free (db->dirname); + g_queue_free (db->ops_queue); + g_hash_table_unref (db->ops_hash); + g_slice_free1 (sizeof (struct rspamd_file_backend), db); + } +} + +/* Create new file backend */ +struct rspamd_kv_backend * +rspamd_kv_file_new (const gchar *filename, guint sync_ops, guint levels) +{ + struct rspamd_file_backend *new; + struct stat st; + gchar *dirname; + + if (filename == NULL) { + return NULL; + } + + dirname = g_path_get_dirname (filename); + if (dirname == NULL || stat (dirname, &st) == -1 || !S_ISDIR (st.st_mode)) { + /* Inaccessible path */ + if (dirname != NULL) { + g_free (dirname); + } + msg_err ("invalid file: %s", filename); + return NULL; + } + + new = g_slice_alloc0 (sizeof (struct rspamd_file_backend)); + new->dirname = dirname; + new->filename = g_strdup (filename); + new->sync_ops = sync_ops; + new->levels = levels; + new->ops_queue = g_queue_new (); + new->ops_hash = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); + + /* Init callbacks */ + new->init_func = rspamd_file_init; + new->insert_func = rspamd_file_insert; + new->lookup_func = rspamd_file_lookup; + new->delete_func = rspamd_file_delete; + new->replace_func = rspamd_file_replace; + new->sync_func = file_process_queue; + new->destroy_func = rspamd_file_destroy; + + return (struct rspamd_kv_backend *)new; +} + diff --git a/src/kvstorage_file.h b/src/kvstorage_file.h new file mode 100644 index 000000000..fab392894 --- /dev/null +++ b/src/kvstorage_file.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2010, Vsevolod Stakhov + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Rambler BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef KVSTORAGE_FILE_H_ +#define KVSTORAGE_FILE_H_ + +#include "config.h" +#include "kvstorage.h" + +/* Create new file backend */ +struct rspamd_kv_backend* rspamd_kv_file_new (const gchar *filename, guint sync_ops, guint levels); + + +#endif /* KVSTORAGE_FILE_H_ */ diff --git a/src/kvstorage_sqlite.h b/src/kvstorage_sqlite.h index 59d23a3c6..cea14df26 100644 --- a/src/kvstorage_sqlite.h +++ b/src/kvstorage_sqlite.h @@ -30,7 +30,7 @@ #ifdef WITH_SQLITE -/* Create new bdb backend */ +/* Create new sqlite backend */ struct rspamd_kv_backend* rspamd_kv_sqlite_new (const gchar *filename, guint sync_ops); #endif |