From 73ded9877fc31f8c1ff5818b643edb7d7c4ea788 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Thu, 7 Jun 2012 19:06:09 +0400 Subject: [PATCH] Avoid using global 'counters' hash, include it in struct rspamd_main. Move worker related utils to separate file and include it in rspamd-mime library. Add some cross-dependencies. --- CMakeLists.txt | 6 +- lib/CMakeLists.txt | 5 +- src/cdb/CMakeLists.txt | 1 + src/controller.c | 3 +- src/json/CMakeLists.txt | 1 + src/lua/CMakeLists.txt | 7 +- src/main.c | 4 +- src/main.h | 1 + src/worker.c | 187 ----------------------------------- src/worker_util.c | 213 ++++++++++++++++++++++++++++++++++++++++ 10 files changed, 231 insertions(+), 197 deletions(-) create mode 100644 src/worker_util.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 3279b761f..cde158b15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,6 +98,8 @@ IF(CMAKE_INSTALL_PREFIX) SET(PREFIX ${CMAKE_INSTALL_PREFIX}) ENDIF(CMAKE_INSTALL_PREFIX) +LIST(APPEND CMAKE_REQUIRED_LIBRARIES m) + # Try to detect platform for further configuration CHECK_C_SOURCE_COMPILES ("#if !defined(__x86_64) && !defined(__amd64) && !defined(_M_X64) #error assume 32 bit arch @@ -934,11 +936,12 @@ IF(NOT HIREDIS_FOUND) INCLUDE_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/contrib/hiredis") ENDIF(NOT HIREDIS_FOUND) + ADD_SUBDIRECTORY(src/lua) ADD_SUBDIRECTORY(src/json) ADD_SUBDIRECTORY(src/cdb) - ADD_SUBDIRECTORY(lib) + ADD_SUBDIRECTORY(src/client) ADD_SUBDIRECTORY(utils) @@ -1011,7 +1014,6 @@ TARGET_LINK_LIBRARIES(rspamd hiredis) IF(GLIB_COMPAT) TARGET_LINK_LIBRARIES(rspamd glibadditions) ENDIF(GLIB_COMPAT) -TARGET_LINK_LIBRARIES(rspamd m) ##################### INSTALLATION ########################################## diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 1d2b0c473..39a82b40a 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -39,7 +39,6 @@ SET(LIBRSPAMDSERVERSRC ../src/proxy.c ../src/settings.c ../src/spf.c - ../src/smtp_proto.c ../src/statfile.c ../src/statfile_sync.c ../src/symbols_cache.c @@ -53,7 +52,9 @@ SET(LIBRSPAMDMIMESRC ../src/images.c ../src/message.c ../src/protocol.c - ../src/smtp_utils.c) + ../src/smtp_utils.c + ../src/smtp_proto.c + ../src/worker_util.c) # Add targets # Rspamdutil diff --git a/src/cdb/CMakeLists.txt b/src/cdb/CMakeLists.txt index 920345fff..8c7f6b3be 100644 --- a/src/cdb/CMakeLists.txt +++ b/src/cdb/CMakeLists.txt @@ -4,6 +4,7 @@ SET(CDBSRC cdb_init.c cdb_make.c) ADD_LIBRARY(rspamd-cdb ${LINK_TYPE} ${CDBSRC}) +TARGET_LINK_LIBRARIES(rspamd-cdb rspamd-util) SET_TARGET_PROPERTIES(rspamd-cdb PROPERTIES VERSION ${RSPAMD_VERSION}) SET_TARGET_PROPERTIES(rspamd-cdb PROPERTIES COMPILE_FLAGS "-DRSPAMD_LIB") INSTALL(TARGETS rspamd-cdb diff --git a/src/controller.c b/src/controller.c index 15b22f53e..cf07f1f91 100644 --- a/src/controller.c +++ b/src/controller.c @@ -115,7 +115,6 @@ static time_t start_time; static gchar greetingbuf[1024]; static sig_atomic_t wanna_die = 0; -extern rspamd_hash_t *counters; static gboolean controller_write_socket (void *arg); @@ -793,7 +792,7 @@ process_command (struct controller_command *cmd, gchar **cmd_args, struct contro } break; case COMMAND_COUNTERS: - rspamd_hash_foreach (counters, counter_write_callback, session); + rspamd_hash_foreach (rspamd_main->counters, counter_write_callback, session); break; } return TRUE; diff --git a/src/json/CMakeLists.txt b/src/json/CMakeLists.txt index 267fc0ea8..9b37384bb 100644 --- a/src/json/CMakeLists.txt +++ b/src/json/CMakeLists.txt @@ -7,6 +7,7 @@ SET(JSONSRC dump.c value.c) ADD_LIBRARY(rspamd-json ${LINK_TYPE} ${JSONSRC}) +TARGET_LINK_LIBRARIES(rspamd-json rspamd-util) SET_TARGET_PROPERTIES(rspamd-json PROPERTIES VERSION ${RSPAMD_VERSION}) SET_TARGET_PROPERTIES(rspamd-json PROPERTIES COMPILE_FLAGS "-DRSPAMD_LIB") INSTALL(TARGETS rspamd-json diff --git a/src/lua/CMakeLists.txt b/src/lua/CMakeLists.txt index 7ebd0de96..0c9d93621 100644 --- a/src/lua/CMakeLists.txt +++ b/src/lua/CMakeLists.txt @@ -14,7 +14,12 @@ SET(LUASRC lua_common.c ADD_LIBRARY(rspamd-lua ${LINK_TYPE} ${LUASRC}) SET_TARGET_PROPERTIES(rspamd-lua PROPERTIES VERSION ${RSPAMD_VERSION}) -TARGET_LINK_LIBRARIES(rspamd-lua ${LUALIB}) +TARGET_LINK_LIBRARIES(rspamd-lua rspamd-util) +IF(ENABLE_LUAJIT MATCHES "ON") + TARGET_LINK_LIBRARIES(rspamd-lua "${LUAJIT_LIBRARY}") +ELSE(ENABLE_LUAJIT MATCHES "ON") + TARGET_LINK_LIBRARIES(rspamd-lua "${LUA_LIBRARY}") +ENDIF(ENABLE_LUAJIT MATCHES "ON") SET_TARGET_PROPERTIES(rspamd-lua PROPERTIES COMPILE_FLAGS "-DRSPAMD_LIB") INSTALL(TARGETS rspamd-lua LIBRARY DESTINATION ${LIBDIR} diff --git a/src/main.c b/src/main.c index d61e48bac..e13d387fc 100644 --- a/src/main.c +++ b/src/main.c @@ -41,8 +41,6 @@ /* 10 seconds after getting termination signal to terminate all workers with SIGKILL */ #define HARD_TERMINATION_TIME 10 -extern rspamd_hash_t *counters; - static struct rspamd_worker *fork_worker (struct rspamd_main *, struct worker_conf *); static gboolean load_rspamd_config (struct config_file *cfg, gboolean init_modules); static void init_cfg_cache (struct config_file *cfg); @@ -921,7 +919,7 @@ main (gint argc, gchar **argv, gchar **env) } /* Init counters */ - counters = rspamd_hash_new_shared (rspamd_main->server_pool, g_str_hash, g_str_equal, 64); + rspamd_main->counters = rspamd_hash_new_shared (rspamd_main->server_pool, g_str_hash, g_str_equal, 64); /* Init listen sockets hash */ listen_sockets = g_hash_table_new (g_direct_hash, g_direct_equal); diff --git a/src/main.h b/src/main.h index 1d8fab134..f90edf1d2 100644 --- a/src/main.h +++ b/src/main.h @@ -99,6 +99,7 @@ struct rspamd_main { memory_pool_t *server_pool; /**< server's memory pool */ statfile_pool_t *statfile_pool; /**< shared statfiles pool */ GHashTable *workers; /**< workers pool indexed by pid */ + rspamd_hash_t *counters; /**< symbol cache counters */ rspamd_logger_t *logger; uid_t workers_uid; /**< worker's uid running to */ gid_t workers_gid; /**< worker's gid running to */ diff --git a/src/worker.c b/src/worker.c index 1da4458d7..465104b6e 100644 --- a/src/worker.c +++ b/src/worker.c @@ -115,8 +115,6 @@ static gboolean write_socket (void *arg); static sig_atomic_t wanna_die = 0; -rspamd_hash_t *counters = NULL; - #ifndef HAVE_SA_SIGINFO static void sig_handler (gint signo) @@ -179,191 +177,6 @@ sigusr1_handler (gint fd, short what, void *arg) return; } -/* - * Destructor for recipients list in a task - */ -static void -rcpt_destruct (void *pointer) -{ - struct worker_task *task = (struct worker_task *) pointer; - - if (task->rcpt) { - g_list_free (task->rcpt); - } -} - -/* - * Create new task - */ -struct worker_task * -construct_task (struct rspamd_worker *worker) -{ - struct worker_task *new_task; - - new_task = g_slice_alloc0 (sizeof (struct worker_task)); - - new_task->worker = worker; - new_task->state = READ_COMMAND; - new_task->cfg = worker->srv->cfg; - new_task->view_checked = FALSE; -#ifdef HAVE_CLOCK_GETTIME -# ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID - clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &new_task->ts); -# elif defined(HAVE_CLOCK_VIRTUAL) - clock_gettime (CLOCK_VIRTUAL, &new_task->ts); -# else - clock_gettime (CLOCK_REALTIME, &new_task->ts); -# endif -#endif - if (gettimeofday (&new_task->tv, NULL) == -1) { - msg_warn ("gettimeofday failed: %s", strerror (errno)); - } - - new_task->task_pool = memory_pool_new (memory_pool_get_size ()); - - /* Add destructor for recipients list (it would be better to use anonymous function here */ - memory_pool_add_destructor (new_task->task_pool, - (pool_destruct_func) rcpt_destruct, new_task); - new_task->results = g_hash_table_new (g_str_hash, g_str_equal); - memory_pool_add_destructor (new_task->task_pool, - (pool_destruct_func) g_hash_table_destroy, - new_task->results); - new_task->re_cache = g_hash_table_new (g_str_hash, g_str_equal); - memory_pool_add_destructor (new_task->task_pool, - (pool_destruct_func) g_hash_table_destroy, - new_task->re_cache); - new_task->raw_headers = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); - memory_pool_add_destructor (new_task->task_pool, - (pool_destruct_func) g_hash_table_destroy, - new_task->raw_headers); - new_task->emails = g_tree_new (compare_email_func); - memory_pool_add_destructor (new_task->task_pool, - (pool_destruct_func) g_tree_destroy, - new_task->emails); - new_task->urls = g_tree_new (compare_url_func); - memory_pool_add_destructor (new_task->task_pool, - (pool_destruct_func) g_tree_destroy, - new_task->urls); - new_task->sock = -1; - new_task->is_mime = TRUE; - new_task->pre_result.action = METRIC_ACTION_NOACTION; - - return new_task; -} - -/** - * Return worker's control structure by its type - * @param type - * @return worker's control structure or NULL - */ -worker_t* -get_worker_by_type (GQuark type) -{ - worker_t **cur; - - cur = &workers[0]; - while (*cur) { - if (g_quark_from_string ((*cur)->name) == type) { - return *cur; - } - cur ++; - } - - return NULL; -} - - -/* - * Free all structures of worker_task - */ -void -free_task (struct worker_task *task, gboolean is_soft) -{ - GList *part; - struct mime_part *p; - - if (task) { - debug_task ("free pointer %p", task); - while ((part = g_list_first (task->parts))) { - task->parts = g_list_remove_link (task->parts, part); - p = (struct mime_part *) part->data; - g_byte_array_free (p->content, TRUE); - g_list_free_1 (part); - } - if (task->text_parts) { - g_list_free (task->text_parts); - } - if (task->images) { - g_list_free (task->images); - } - if (task->messages) { - g_list_free (task->messages); - } - if (task->received) { - g_list_free (task->received); - } - memory_pool_delete (task->task_pool); - if (task->dispatcher) { - if (is_soft) { - /* Plan dispatcher shutdown */ - task->dispatcher->wanna_die = 1; - } - else { - rspamd_remove_dispatcher (task->dispatcher); - } - } - if (task->sock != -1) { - close (task->sock); - } - g_slice_free1 (sizeof (struct worker_task), task); - } -} - -void -free_task_hard (gpointer ud) -{ - struct worker_task *task = ud; - - free_task (task, FALSE); -} - -void -free_task_soft (gpointer ud) -{ - struct worker_task *task = ud; - - free_task (task, FALSE); -} - -double -set_counter (const gchar *name, guint32 value) -{ - struct counter_data *cd; - double alpha; - gchar *key; - - cd = rspamd_hash_lookup (counters, (gpointer) name); - - if (cd == NULL) { - cd = memory_pool_alloc_shared (counters->pool, sizeof (struct counter_data)); - cd->value = value; - cd->number = 0; - key = memory_pool_strdup_shared (counters->pool, name); - rspamd_hash_insert (counters, (gpointer) key, (gpointer) cd); - } - else { - /* Calculate new value */ - memory_pool_wlock_rwlock (counters->lock); - - alpha = 2. / (++cd->number + 1); - cd->value = cd->value * (1. - alpha) + value * alpha; - - memory_pool_wunlock_rwlock (counters->lock); - } - - return cd->value; -} - #ifndef BUILD_STATIC static void fin_custom_filters (struct worker_task *task) diff --git a/src/worker_util.c b/src/worker_util.c new file mode 100644 index 000000000..ba4444485 --- /dev/null +++ b/src/worker_util.c @@ -0,0 +1,213 @@ +/* Copyright (c) 2010-2011, Vsevolod Stakhov + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "main.h" +#include "message.h" + +extern struct rspamd_main *rspamd_main; + +/* + * Destructor for recipients list in a task + */ +static void +rcpt_destruct (void *pointer) +{ + struct worker_task *task = (struct worker_task *) pointer; + + if (task->rcpt) { + g_list_free (task->rcpt); + } +} + +/* + * Create new task + */ +struct worker_task * +construct_task (struct rspamd_worker *worker) +{ + struct worker_task *new_task; + + new_task = g_slice_alloc0 (sizeof (struct worker_task)); + + new_task->worker = worker; + new_task->state = READ_COMMAND; + new_task->cfg = worker->srv->cfg; + new_task->view_checked = FALSE; +#ifdef HAVE_CLOCK_GETTIME +# ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID + clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &new_task->ts); +# elif defined(HAVE_CLOCK_VIRTUAL) + clock_gettime (CLOCK_VIRTUAL, &new_task->ts); +# else + clock_gettime (CLOCK_REALTIME, &new_task->ts); +# endif +#endif + if (gettimeofday (&new_task->tv, NULL) == -1) { + msg_warn ("gettimeofday failed: %s", strerror (errno)); + } + + new_task->task_pool = memory_pool_new (memory_pool_get_size ()); + + /* Add destructor for recipients list (it would be better to use anonymous function here */ + memory_pool_add_destructor (new_task->task_pool, + (pool_destruct_func) rcpt_destruct, new_task); + new_task->results = g_hash_table_new (g_str_hash, g_str_equal); + memory_pool_add_destructor (new_task->task_pool, + (pool_destruct_func) g_hash_table_destroy, + new_task->results); + new_task->re_cache = g_hash_table_new (g_str_hash, g_str_equal); + memory_pool_add_destructor (new_task->task_pool, + (pool_destruct_func) g_hash_table_destroy, + new_task->re_cache); + new_task->raw_headers = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); + memory_pool_add_destructor (new_task->task_pool, + (pool_destruct_func) g_hash_table_destroy, + new_task->raw_headers); + new_task->emails = g_tree_new (compare_email_func); + memory_pool_add_destructor (new_task->task_pool, + (pool_destruct_func) g_tree_destroy, + new_task->emails); + new_task->urls = g_tree_new (compare_url_func); + memory_pool_add_destructor (new_task->task_pool, + (pool_destruct_func) g_tree_destroy, + new_task->urls); + new_task->sock = -1; + new_task->is_mime = TRUE; + new_task->pre_result.action = METRIC_ACTION_NOACTION; + + return new_task; +} + +/** + * Return worker's control structure by its type + * @param type + * @return worker's control structure or NULL + */ +worker_t* +get_worker_by_type (GQuark type) +{ + worker_t **cur; + + cur = &workers[0]; + while (*cur) { + if (g_quark_from_string ((*cur)->name) == type) { + return *cur; + } + cur ++; + } + + return NULL; +} + + +/* + * Free all structures of worker_task + */ +void +free_task (struct worker_task *task, gboolean is_soft) +{ + GList *part; + struct mime_part *p; + + if (task) { + debug_task ("free pointer %p", task); + while ((part = g_list_first (task->parts))) { + task->parts = g_list_remove_link (task->parts, part); + p = (struct mime_part *) part->data; + g_byte_array_free (p->content, TRUE); + g_list_free_1 (part); + } + if (task->text_parts) { + g_list_free (task->text_parts); + } + if (task->images) { + g_list_free (task->images); + } + if (task->messages) { + g_list_free (task->messages); + } + if (task->received) { + g_list_free (task->received); + } + memory_pool_delete (task->task_pool); + if (task->dispatcher) { + if (is_soft) { + /* Plan dispatcher shutdown */ + task->dispatcher->wanna_die = 1; + } + else { + rspamd_remove_dispatcher (task->dispatcher); + } + } + if (task->sock != -1) { + close (task->sock); + } + g_slice_free1 (sizeof (struct worker_task), task); + } +} + +void +free_task_hard (gpointer ud) +{ + struct worker_task *task = ud; + + free_task (task, FALSE); +} + +void +free_task_soft (gpointer ud) +{ + struct worker_task *task = ud; + + free_task (task, FALSE); +} + +double +set_counter (const gchar *name, guint32 value) +{ + struct counter_data *cd; + double alpha; + gchar *key; + + cd = rspamd_hash_lookup (rspamd_main->counters, (gpointer) name); + + if (cd == NULL) { + cd = memory_pool_alloc_shared (rspamd_main->counters->pool, sizeof (struct counter_data)); + cd->value = value; + cd->number = 0; + key = memory_pool_strdup_shared (rspamd_main->counters->pool, name); + rspamd_hash_insert (rspamd_main->counters, (gpointer) key, (gpointer) cd); + } + else { + /* Calculate new value */ + memory_pool_wlock_rwlock (rspamd_main->counters->lock); + + alpha = 2. / (++cd->number + 1); + cd->value = cd->value * (1. - alpha) + value * alpha; + + memory_pool_wunlock_rwlock (rspamd_main->counters->lock); + } + + return cd->value; +} -- 2.39.5