* Initial implementation of key-value storage

Detect and link against bdb
Change version to 0.4.5

2 files changed, 340 insertions, 0 deletions

diff --git a/src/kvstorage.c b/src/kvstorage.c
new file mode 100644
index 000000000..1371df9f5
--- /dev/null
+++ b/src/kvstorage.c
@@ -0,0 +1,203 @@
+/* Copyright (c) 2011, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above copyright
+ *         notice, this list of conditions and the following disclaimer in the
+ *         documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "kvstorage.h"
+#include "main.h"
+
+#define MAX_EXPIRE_STEPS 10
+
+/** Create new kv storage */
+struct rspamd_kv_storage *
+rspamd_kv_storage_new (gint id, const gchar *name, struct rspamd_kv_cache *cache, struct rspamd_kv_backend *backend, struct rspamd_kv_expire *expire,
+		gsize max_elts, gsize max_memory)
+{
+	struct rspamd_kv_storage 			*new;
+
+	new = g_slice_alloc (sizeof (struct rspamd_kv_storage));
+	new->elts = 0;
+	new->memory = 0;
+
+	new->cache = cache;
+	new->backend = backend;
+	new->expire = expire;
+
+	new->max_elts = max_elts;
+	new->max_memory = max_memory;
+
+	new->id = id;
+
+	if (name != NULL) {
+		new->name = g_strdup (name);
+	}
+	else {
+		/* Name is absent, use ID as name */
+		new->name = g_malloc (sizeof ("18446744073709551616"));
+		rspamd_snprintf (new->name, sizeof ("18446744073709551616"), "%d", id);
+	}
+
+	return new;
+}
+
+/** Insert new element to the kv storage */
+gboolean
+rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, gpointer data, gsize len, gint flags)
+{
+	gint 								steps = 0;
+	struct rspamd_kv_element           *elt;
+	gboolean							res = TRUE;
+
+	/* Hard limit */
+	if (len > storage->max_memory) {
+		msg_info ("<%s>: trying to insert value of length %z while limit is %z", len, storage->max_memory);
+		return FALSE;
+	}
+
+	/* Now check limits */
+	while (storage->memory + len > storage->max_memory || storage->elts >= storage->max_elts) {
+		if (storage->expire) {
+			storage->expire->step_func (storage->expire, storage);
+		}
+		else {
+			msg_warn ("<%s>: storage %s is full and no expire function is defined", storage->name);
+		}
+		if (++steps > MAX_EXPIRE_STEPS) {
+			msg_warn ("<%s>: cannot expire enough keys in storage", storage->name);
+			return FALSE;
+		}
+	}
+
+	/* Insert elt to the cache */
+	elt = storage->cache->insert_func (storage->cache, key, data, len);
+	if (elt == NULL) {
+		return FALSE;
+	}
+	elt->flags = flags;
+
+	/* Place to the backend */
+	if (storage->backend) {
+		res = storage->backend->insert_func (storage->backend, key, elt);
+	}
+
+	/* Insert to the expire */
+	if (storage->expire) {
+		storage->expire->insert_func (storage->expire, elt);
+	}
+
+	storage->elts ++;
+	storage->memory += len + sizeof (struct rspamd_kv_element);
+
+	return res;
+}
+
+/** Replace an element in the kv storage */
+gboolean
+rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, struct rspamd_kv_element *elt)
+{
+	gboolean						res = TRUE;
+	gint							steps = 0;
+
+	/* Hard limit */
+	if (elt->size > storage->max_memory) {
+		msg_info ("<%s>: trying to replace value of length %z while limit is %z", elt->size, storage->max_memory);
+		return FALSE;
+	}
+
+	/* Now check limits */
+	while (storage->memory + elt->size > storage->max_memory) {
+		if (storage->expire) {
+			storage->expire->step_func (storage->expire, storage);
+		}
+		else {
+			msg_warn ("<%s>: storage %s is full and no expire function is defined", storage->name);
+		}
+		if (++steps > MAX_EXPIRE_STEPS) {
+			msg_warn ("<%s>: cannot expire enough keys in storage", storage->name);
+			return FALSE;
+		}
+	}
+
+	/* Insert elt to the cache */
+	res = storage->cache->replace_func (storage->cache, key, elt);
+
+	/* Place to the backend */
+	if (res && storage->backend) {
+		res = storage->backend->replace_func (storage->backend, key, elt);
+	}
+
+	return res;
+}
+
+/** Lookup an element inside kv storage */
+struct rspamd_kv_element*
+rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage, gpointer key)
+{
+	struct rspamd_kv_element			*elt = NULL;
+
+	/* First try to look at cache */
+	elt = storage->cache->lookup_func (storage->cache, key);
+
+	/* Next look at the backend */
+	if (storage->backend) {
+		elt = storage->backend->lookup_func (storage->backend, key);
+	}
+
+	return elt;
+}
+
+/** Expire an element from kv storage */
+gboolean
+rspamd_kv_storage_delete (struct rspamd_kv_storage *storage, gpointer key)
+{
+	gboolean							res = TRUE;
+
+	/* First delete key from cache */
+	res = storage->cache->delete_func (storage->cache, key);
+
+	/* Now delete from backend */
+	if (storage->backend) {
+		res = storage->backend->delete_func (storage->backend, key);
+	}
+	/* Notify expire */
+	/* XXX: implement this */
+
+	return res;
+}
+
+/** Destroy kv storage */
+void
+rspamd_kv_storage_destroy (struct rspamd_kv_storage *storage)
+{
+	if (storage->cache) {
+		storage->cache->destroy_func (storage->cache);
+	}
+	if (storage->backend) {
+		storage->backend->destroy_func (storage->backend);
+	}
+	if (storage->expire) {
+		storage->expire->destroy_func (storage->expire);
+	}
+
+	g_free (storage->name);
+	g_slice_free1 (sizeof (struct rspamd_kv_storage), storage);
+}
diff --git a/src/kvstorage.h b/src/kvstorage.h
new file mode 100644
index 000000000..6caa3f4a1
--- /dev/null
+++ b/src/kvstorage.h
@@ -0,0 +1,137 @@
+/* Copyright (c) 2011, Vsevolod Stakhov
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *       * Redistributions of source code must retain the above copyright
+ *         notice, this list of conditions and the following disclaimer.
+ *       * Redistributions in binary form must reproduce the above copyright
+ *         notice, this list of conditions and the following disclaimer in the
+ *         documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef KVSTORAGE_H_
+#define KVSTORAGE_H_
+
+#include "config.h"
+
+struct rspamd_kv_cache;
+struct rspamd_kv_backend;
+struct rspamd_kv_storage;
+struct rspamd_kv_expire;
+struct rspamd_kv_element;
+
+/* Callbacks for cache */
+typedef void (*cache_init)(struct rspamd_kv_cache *cache);
+typedef struct rspamd_kv_element* (*cache_insert)(struct rspamd_kv_cache *cache, gpointer key, gpointer value, gsize len);
+typedef gboolean (*cache_replace)(struct rspamd_kv_cache *cache, gpointer key, struct rspamd_kv_element *elt);
+typedef struct rspamd_kv_element* (*cache_lookup)(struct rspamd_kv_cache *cache, gpointer key);
+typedef struct rspamd_kv_element* (*cache_delete)(struct rspamd_kv_cache *cache, gpointer key);
+typedef void (*cache_destroy)(struct rspamd_kv_cache *cache);
+
+/* Callbacks for backend */
+typedef void (*backend_init)(struct rspamd_kv_backend *backend);
+typedef gboolean (*backend_insert)(struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt);
+typedef gboolean (*backend_replace)(struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt);
+typedef struct rspamd_kv_element* (*backend_lookup)(struct rspamd_kv_backend *backend, gpointer key);
+typedef gboolean (*backend_delete)(struct rspamd_kv_backend *backend, gpointer key);
+typedef void (*backend_destroy)(struct rspamd_kv_backend *backend);
+
+/* Callbacks for expire */
+typedef void (*expire_init)(struct rspamd_kv_expire *expire);
+typedef void (*expire_insert)(struct rspamd_kv_expire *expire, struct rspamd_kv_element *elt);
+typedef void (*expire_delete)(struct rspamd_kv_expire *expire, struct rspamd_kv_element *elt);
+typedef gboolean (*expire_step)(struct rspamd_kv_expire *expire, struct rspamd_kv_storage *storage);
+typedef void (*expire_destroy)(struct rspamd_kv_expire *expire);
+
+
+/* Flags of element */
+enum rspamd_kv_flags {
+	KV_ELT_ARRAY = 1 << 0,
+	KV_ELT_PERSISTEN = 1 << 1
+};
+
+/* Common structures description */
+
+struct rspamd_kv_element {
+	time_t age;									/*< age of element */
+	enum rspamd_kv_flags flags;					/*< element flags  */
+	gsize size;									/*< size of element */
+	TAILQ_ENTRY(rspamd_kv_element) entry;		/*< list entry */
+
+	gchar data[1];								/*< expandable data */
+};
+
+struct rspamd_kv_cache {
+	cache_init init_func;						/*< this callback is called on kv storage initialization */
+	cache_insert insert_func;					/*< this callback is called when element is inserted */
+	cache_replace replace_func;					/*< this callback is called when element is replace */
+	cache_lookup lookup_func;					/*< this callback is used for lookup of element */
+	cache_delete delete_func;					/*< this callback is called when an element is deleted */
+	cache_destroy destroy_func;					/*< this callback is used for destroying all elements inside cache */
+};
+struct rspamd_kv_backend {
+	backend_init init_func;						/*< this callback is called on kv storage initialization */
+	backend_insert insert_func;					/*< this callback is called when element is inserted */
+	backend_replace replace_func;				/*< this callback is called when element is replaced */
+	backend_lookup lookup_func;					/*< this callback is used for lookup of element */
+	backend_delete delete_func;					/*< this callback is called when an element is deleted */
+	backend_destroy destroy_func;				/*< this callback is used for destroying all elements inside backend */
+};
+struct rspamd_kv_expire {
+	expire_init init_func;						/*< this callback is called on kv storage initialization */
+	expire_insert insert_func;					/*< this callback is called when element is inserted */
+	expire_step step_func;						/*< this callback is used when cache is full */
+	expire_delete delete_func;					/*< this callback is called when an element is deleted */
+	expire_destroy destroy_func;				/*< this callback is used for destroying all elements inside expire */
+};
+
+/* Main kv storage structure */
+
+struct rspamd_kv_storage {
+	struct rspamd_kv_cache *cache;
+	struct rspamd_kv_backend *backend;
+	struct rspamd_kv_expire *expire;
+
+	gsize elts;									/*< current elements count in a storage */
+	gsize max_elts;								/*< maximum number of elements in a storage */
+
+	gsize memory;								/*< memory eaten */
+	gsize max_memory;							/*< memory limit */
+
+	gint id;									/* char ID */
+	gchar *name;								/* numeric ID */
+};
+
+/** Create new kv storage */
+struct rspamd_kv_storage *rspamd_kv_storage_new (gint id, const gchar *name,
+		struct rspamd_kv_cache *cache, struct rspamd_kv_backend *backend, struct rspamd_kv_expire *expire,
+		gsize max_elts, gsize max_memory);
+
+/** Insert new element to the kv storage */
+gboolean rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, gpointer data, gsize len, gint flags);
+
+/** Replace an element in the kv storage */
+gboolean rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, struct rspamd_kv_element *elt);
+
+/** Lookup an element inside kv storage */
+struct rspamd_kv_element* rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage, gpointer key);
+
+/** Expire an element from kv storage */
+gboolean rspamd_kv_storage_delete (struct rspamd_kv_storage *storage, gpointer key);
+
+/** Destroy kv storage */
+void rspamd_kv_storage_destroy (struct rspamd_kv_storage *storage);
+
+#endif /* KVSTORAGE_H_ */