From 62cbfbe0f96fb936c26de62d734c32ddb8ae4135 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Tue, 11 Oct 2011 19:39:38 +0400 Subject: [PATCH] * Initial implementation of key-value storage Detect and link against bdb Change version to 0.4.5 --- CMakeLists.txt | 85 +++++++++++++++---- config.h.in | 2 + lib/CMakeLists.txt | 1 + src/kvstorage.c | 203 +++++++++++++++++++++++++++++++++++++++++++++ src/kvstorage.h | 137 ++++++++++++++++++++++++++++++ 5 files changed, 411 insertions(+), 17 deletions(-) create mode 100644 src/kvstorage.c create mode 100644 src/kvstorage.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f27580d0e..ae042706a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ PROJECT(rspamd C) SET(RSPAMD_VERSION_MAJOR 0) SET(RSPAMD_VERSION_MINOR 4) -SET(RSPAMD_VERSION_PATCH 4) +SET(RSPAMD_VERSION_PATCH 5) SET(RSPAMD_VERSION "${RSPAMD_VERSION_MAJOR}.${RSPAMD_VERSION_MINOR}.${RSPAMD_VERSION_PATCH}") @@ -361,10 +361,16 @@ LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS}) # Check for libevent -FIND_LIBRARY(LIBEVENT_LIBRARY NAMES event PATHS /lib - /opt/lib - /usr/lib - /usr/local/lib +FIND_LIBRARY(LIBEVENT_LIBRARY NAMES event PATH_SUFFIXES lib64 lib + PATHS + ~/Library/Frameworks + /Library/Frameworks + /usr/local + /usr + /sw + /opt/local + /opt/csw + /opt DOC "Path where the libevent library can be found") IF(NOT LIBEVENT_LIBRARY) MESSAGE(FATAL_ERROR "libevent is required for building rspamd") @@ -380,10 +386,16 @@ INCLUDE_DIRECTORIES("${LIBEVENT_INCLUDE}") LINK_DIRECTORIES("${LIBEVENT_PATH}") # Find libjudy -FIND_LIBRARY(LIBJUDY_LIBRARY NAMES Judy PATHS /lib - /opt/lib - /usr/lib - /usr/local/lib +FIND_LIBRARY(LIBJUDY_LIBRARY NAMES Judy PATH_SUFFIXES lib64 lib + PATHS + ~/Library/Frameworks + /Library/Frameworks + /usr/local + /usr + /sw + /opt/local + /opt/csw + /opt DOC "Path where the libjudy library can be found") IF(LIBJUDY_LIBRARY) FIND_PATH(LIBJUDY_INCLUDE Judy.h PATHS /opt/include @@ -397,6 +409,30 @@ IF(LIBJUDY_LIBRARY) SET(WITH_JUDY 1) ENDIF(LIBJUDY_LIBRARY) +# Find libbd +FIND_LIBRARY(LIBDB_LIBRARY NAMES db-4 PATHS PATH_SUFFIXES lib64 lib + PATHS + ~/Library/Frameworks + /Library/Frameworks + /usr/local + /usr + /sw + /opt/local + /opt/csw + /opt + DOC "Path where the libdb library can be found") +IF(LIBDB_LIBRARY) + FIND_PATH(LIBDB_INCLUDE db.h PATHS /opt/include + /usr/include + /usr/local/include + DOC "Path where the bdb header files can be found") + + GET_FILENAME_COMPONENT(LIBDB_PATH "${LIBDB_LIBRARY}" PATH) + INCLUDE_DIRECTORIES("${LIBDB_INCLUDE}") + LINK_DIRECTORIES("${LIBDB_PATH}") + SET(WITH_DB 1) +ENDIF(LIBDB_LIBRARY) + IF(ENABLE_PROFILING MATCHES "ON") SET(WITH_PROFILER 1) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pg") @@ -419,10 +455,16 @@ IF(ENABLE_GPERF_TOOLS MATCHES "ON") /usr/local/include DOC "Path where google perftools includes can be found") INCLUDE_DIRECTORIES("${GPERF_INCLUDE}") - FIND_LIBRARY(GPERF_LIBRARY NAMES profiler PATHS /lib - /opt/lib - /usr/lib - /usr/local/lib + FIND_LIBRARY(GPERF_LIBRARY NAMES profiler PATH_SUFFIXES lib64 lib + PATHS + ~/Library/Frameworks + /Library/Frameworks + /usr/local + /usr + /sw + /opt/local + /opt/csw + /opt DOC "Path where the gperf library can be found") IF(NOT GPERF_LIBRARY) MESSAGE(FATAL_ERROR "gperf tools support is enabled but not found in system") @@ -435,10 +477,16 @@ ENDIF(ENABLE_GPERF_TOOLS MATCHES "ON") # Find util library -FIND_LIBRARY(LIBUTIL_LIBRARY NAMES util PATHS /lib - /opt/lib - /usr/lib - /usr/local/lib +FIND_LIBRARY(LIBUTIL_LIBRARY NAMES util PATHS PATH_SUFFIXES lib64 lib + PATHS + ~/Library/Frameworks + /Library/Frameworks + /usr/local + /usr + /sw + /opt/local + /opt/csw + /opt DOC "Path where the libutil library can be found") IF(LIBUTIL_LIBRARY) LIST(APPEND CMAKE_REQUIRED_LIBRARIES util) @@ -713,6 +761,9 @@ ENDIF(ENABLE_LUAJIT MATCHES "ON") IF(LIBJUDY_LIBRARY) TARGET_LINK_LIBRARIES(rspamd Judy) ENDIF(LIBJUDY_LIBRARY) +IF(LIBDB_LIBRARY) + TARGET_LINK_LIBRARIES(rspamd db-4) +ENDIF(LIBDB_LIBRARY) TARGET_LINK_LIBRARIES(rspamd event) TARGET_LINK_LIBRARIES(rspamd pcre) diff --git a/config.h.in b/config.h.in index 81b797334..804ef08b5 100644 --- a/config.h.in +++ b/config.h.in @@ -121,6 +121,8 @@ #cmakedefine WITH_JUDY 1 +#cmakedefine WITH_DB 1 + #cmakedefine WITH_GPERF_TOOLS 1 #cmakedefine HAVE_ASM_PAUSE 1 diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 6071c0e78..12ffaf62e 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -44,6 +44,7 @@ SET(RSPAMDLIBSRC ../src/binlog.c ../src/hash.c ../src/html.c ../src/images.c + ../src/kvstorage.c ../src/lmtp_proto.c ../src/logger.c ../src/map.c diff --git a/src/kvstorage.c b/src/kvstorage.c new file mode 100644 index 000000000..1371df9f5 --- /dev/null +++ b/src/kvstorage.c @@ -0,0 +1,203 @@ +/* Copyright (c) 2011, Vsevolod Stakhov + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "kvstorage.h" +#include "main.h" + +#define MAX_EXPIRE_STEPS 10 + +/** Create new kv storage */ +struct rspamd_kv_storage * +rspamd_kv_storage_new (gint id, const gchar *name, struct rspamd_kv_cache *cache, struct rspamd_kv_backend *backend, struct rspamd_kv_expire *expire, + gsize max_elts, gsize max_memory) +{ + struct rspamd_kv_storage *new; + + new = g_slice_alloc (sizeof (struct rspamd_kv_storage)); + new->elts = 0; + new->memory = 0; + + new->cache = cache; + new->backend = backend; + new->expire = expire; + + new->max_elts = max_elts; + new->max_memory = max_memory; + + new->id = id; + + if (name != NULL) { + new->name = g_strdup (name); + } + else { + /* Name is absent, use ID as name */ + new->name = g_malloc (sizeof ("18446744073709551616")); + rspamd_snprintf (new->name, sizeof ("18446744073709551616"), "%d", id); + } + + return new; +} + +/** Insert new element to the kv storage */ +gboolean +rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, gpointer data, gsize len, gint flags) +{ + gint steps = 0; + struct rspamd_kv_element *elt; + gboolean res = TRUE; + + /* Hard limit */ + if (len > storage->max_memory) { + msg_info ("<%s>: trying to insert value of length %z while limit is %z", len, storage->max_memory); + return FALSE; + } + + /* Now check limits */ + while (storage->memory + len > storage->max_memory || storage->elts >= storage->max_elts) { + if (storage->expire) { + storage->expire->step_func (storage->expire, storage); + } + else { + msg_warn ("<%s>: storage %s is full and no expire function is defined", storage->name); + } + if (++steps > MAX_EXPIRE_STEPS) { + msg_warn ("<%s>: cannot expire enough keys in storage", storage->name); + return FALSE; + } + } + + /* Insert elt to the cache */ + elt = storage->cache->insert_func (storage->cache, key, data, len); + if (elt == NULL) { + return FALSE; + } + elt->flags = flags; + + /* Place to the backend */ + if (storage->backend) { + res = storage->backend->insert_func (storage->backend, key, elt); + } + + /* Insert to the expire */ + if (storage->expire) { + storage->expire->insert_func (storage->expire, elt); + } + + storage->elts ++; + storage->memory += len + sizeof (struct rspamd_kv_element); + + return res; +} + +/** Replace an element in the kv storage */ +gboolean +rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, struct rspamd_kv_element *elt) +{ + gboolean res = TRUE; + gint steps = 0; + + /* Hard limit */ + if (elt->size > storage->max_memory) { + msg_info ("<%s>: trying to replace value of length %z while limit is %z", elt->size, storage->max_memory); + return FALSE; + } + + /* Now check limits */ + while (storage->memory + elt->size > storage->max_memory) { + if (storage->expire) { + storage->expire->step_func (storage->expire, storage); + } + else { + msg_warn ("<%s>: storage %s is full and no expire function is defined", storage->name); + } + if (++steps > MAX_EXPIRE_STEPS) { + msg_warn ("<%s>: cannot expire enough keys in storage", storage->name); + return FALSE; + } + } + + /* Insert elt to the cache */ + res = storage->cache->replace_func (storage->cache, key, elt); + + /* Place to the backend */ + if (res && storage->backend) { + res = storage->backend->replace_func (storage->backend, key, elt); + } + + return res; +} + +/** Lookup an element inside kv storage */ +struct rspamd_kv_element* +rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage, gpointer key) +{ + struct rspamd_kv_element *elt = NULL; + + /* First try to look at cache */ + elt = storage->cache->lookup_func (storage->cache, key); + + /* Next look at the backend */ + if (storage->backend) { + elt = storage->backend->lookup_func (storage->backend, key); + } + + return elt; +} + +/** Expire an element from kv storage */ +gboolean +rspamd_kv_storage_delete (struct rspamd_kv_storage *storage, gpointer key) +{ + gboolean res = TRUE; + + /* First delete key from cache */ + res = storage->cache->delete_func (storage->cache, key); + + /* Now delete from backend */ + if (storage->backend) { + res = storage->backend->delete_func (storage->backend, key); + } + /* Notify expire */ + /* XXX: implement this */ + + return res; +} + +/** Destroy kv storage */ +void +rspamd_kv_storage_destroy (struct rspamd_kv_storage *storage) +{ + if (storage->cache) { + storage->cache->destroy_func (storage->cache); + } + if (storage->backend) { + storage->backend->destroy_func (storage->backend); + } + if (storage->expire) { + storage->expire->destroy_func (storage->expire); + } + + g_free (storage->name); + g_slice_free1 (sizeof (struct rspamd_kv_storage), storage); +} diff --git a/src/kvstorage.h b/src/kvstorage.h new file mode 100644 index 000000000..6caa3f4a1 --- /dev/null +++ b/src/kvstorage.h @@ -0,0 +1,137 @@ +/* Copyright (c) 2011, Vsevolod Stakhov + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef KVSTORAGE_H_ +#define KVSTORAGE_H_ + +#include "config.h" + +struct rspamd_kv_cache; +struct rspamd_kv_backend; +struct rspamd_kv_storage; +struct rspamd_kv_expire; +struct rspamd_kv_element; + +/* Callbacks for cache */ +typedef void (*cache_init)(struct rspamd_kv_cache *cache); +typedef struct rspamd_kv_element* (*cache_insert)(struct rspamd_kv_cache *cache, gpointer key, gpointer value, gsize len); +typedef gboolean (*cache_replace)(struct rspamd_kv_cache *cache, gpointer key, struct rspamd_kv_element *elt); +typedef struct rspamd_kv_element* (*cache_lookup)(struct rspamd_kv_cache *cache, gpointer key); +typedef struct rspamd_kv_element* (*cache_delete)(struct rspamd_kv_cache *cache, gpointer key); +typedef void (*cache_destroy)(struct rspamd_kv_cache *cache); + +/* Callbacks for backend */ +typedef void (*backend_init)(struct rspamd_kv_backend *backend); +typedef gboolean (*backend_insert)(struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt); +typedef gboolean (*backend_replace)(struct rspamd_kv_backend *backend, gpointer key, struct rspamd_kv_element *elt); +typedef struct rspamd_kv_element* (*backend_lookup)(struct rspamd_kv_backend *backend, gpointer key); +typedef gboolean (*backend_delete)(struct rspamd_kv_backend *backend, gpointer key); +typedef void (*backend_destroy)(struct rspamd_kv_backend *backend); + +/* Callbacks for expire */ +typedef void (*expire_init)(struct rspamd_kv_expire *expire); +typedef void (*expire_insert)(struct rspamd_kv_expire *expire, struct rspamd_kv_element *elt); +typedef void (*expire_delete)(struct rspamd_kv_expire *expire, struct rspamd_kv_element *elt); +typedef gboolean (*expire_step)(struct rspamd_kv_expire *expire, struct rspamd_kv_storage *storage); +typedef void (*expire_destroy)(struct rspamd_kv_expire *expire); + + +/* Flags of element */ +enum rspamd_kv_flags { + KV_ELT_ARRAY = 1 << 0, + KV_ELT_PERSISTEN = 1 << 1 +}; + +/* Common structures description */ + +struct rspamd_kv_element { + time_t age; /*< age of element */ + enum rspamd_kv_flags flags; /*< element flags */ + gsize size; /*< size of element */ + TAILQ_ENTRY(rspamd_kv_element) entry; /*< list entry */ + + gchar data[1]; /*< expandable data */ +}; + +struct rspamd_kv_cache { + cache_init init_func; /*< this callback is called on kv storage initialization */ + cache_insert insert_func; /*< this callback is called when element is inserted */ + cache_replace replace_func; /*< this callback is called when element is replace */ + cache_lookup lookup_func; /*< this callback is used for lookup of element */ + cache_delete delete_func; /*< this callback is called when an element is deleted */ + cache_destroy destroy_func; /*< this callback is used for destroying all elements inside cache */ +}; +struct rspamd_kv_backend { + backend_init init_func; /*< this callback is called on kv storage initialization */ + backend_insert insert_func; /*< this callback is called when element is inserted */ + backend_replace replace_func; /*< this callback is called when element is replaced */ + backend_lookup lookup_func; /*< this callback is used for lookup of element */ + backend_delete delete_func; /*< this callback is called when an element is deleted */ + backend_destroy destroy_func; /*< this callback is used for destroying all elements inside backend */ +}; +struct rspamd_kv_expire { + expire_init init_func; /*< this callback is called on kv storage initialization */ + expire_insert insert_func; /*< this callback is called when element is inserted */ + expire_step step_func; /*< this callback is used when cache is full */ + expire_delete delete_func; /*< this callback is called when an element is deleted */ + expire_destroy destroy_func; /*< this callback is used for destroying all elements inside expire */ +}; + +/* Main kv storage structure */ + +struct rspamd_kv_storage { + struct rspamd_kv_cache *cache; + struct rspamd_kv_backend *backend; + struct rspamd_kv_expire *expire; + + gsize elts; /*< current elements count in a storage */ + gsize max_elts; /*< maximum number of elements in a storage */ + + gsize memory; /*< memory eaten */ + gsize max_memory; /*< memory limit */ + + gint id; /* char ID */ + gchar *name; /* numeric ID */ +}; + +/** Create new kv storage */ +struct rspamd_kv_storage *rspamd_kv_storage_new (gint id, const gchar *name, + struct rspamd_kv_cache *cache, struct rspamd_kv_backend *backend, struct rspamd_kv_expire *expire, + gsize max_elts, gsize max_memory); + +/** Insert new element to the kv storage */ +gboolean rspamd_kv_storage_insert (struct rspamd_kv_storage *storage, gpointer key, gpointer data, gsize len, gint flags); + +/** Replace an element in the kv storage */ +gboolean rspamd_kv_storage_replace (struct rspamd_kv_storage *storage, gpointer key, struct rspamd_kv_element *elt); + +/** Lookup an element inside kv storage */ +struct rspamd_kv_element* rspamd_kv_storage_lookup (struct rspamd_kv_storage *storage, gpointer key); + +/** Expire an element from kv storage */ +gboolean rspamd_kv_storage_delete (struct rspamd_kv_storage *storage, gpointer key); + +/** Destroy kv storage */ +void rspamd_kv_storage_destroy (struct rspamd_kv_storage *storage); + +#endif /* KVSTORAGE_H_ */ -- 2.39.5