From fee5af783ea49c1fd77e335bb9986fe58152af9a Mon Sep 17 00:00:00 2001 From: "cebka@cebka-laptop" Date: Tue, 14 Oct 2008 17:34:56 +0400 Subject: [PATCH] * Configure modules when worker starts for simplifying reconfig procedure * Add two behaviour types of memory allocator: greedy and ungreedy * Adopt surbl module for new configure style --- cfg_utils.c | 2 +- config.h.in | 2 ++ configure | 23 +++++++++++++++++- mem_pool.c | 14 +++++++++++ plugins/surbl.c | 64 +++++++++++++++++++++++++++++++++---------------- url.h | 18 +++++++------- worker.c | 27 +++++++++++---------- 7 files changed, 107 insertions(+), 43 deletions(-) diff --git a/cfg_utils.c b/cfg_utils.c index 7f64e4552..8630a0963 100644 --- a/cfg_utils.c +++ b/cfg_utils.c @@ -409,7 +409,7 @@ parse_regexp (memory_pool_t *pool, char *line) struct rspamd_regexp *result; int regexp_flags = 0; enum rspamd_regexp_type type = REGEXP_NONE; - GError *err; + GError *err = NULL; /* Find begin */ while (*line != '/') { diff --git a/config.h.in b/config.h.in index 36d67d56e..48c7bc129 100644 --- a/config.h.in +++ b/config.h.in @@ -7,6 +7,8 @@ struct config_file; typedef struct module_s { const char *name; int (*module_init_func)(struct config_file *cfg, struct module_ctx **ctx); + int (*module_config_func)(struct config_file *cfg); + int (*module_reconfig_func)(struct config_file *cfg); } module_t; extern module_t modules[]; diff --git a/configure b/configure index 73eab3b06..d7fa8cf79 100755 --- a/configure +++ b/configure @@ -458,8 +458,10 @@ write_modules() echo "module_t modules[] = {" >> modules.c; modules_num=0 for m in $MODULES ; do - echo "{\"${m}\", ${m}_module_init}," >> modules.c + echo "{\"${m}\", ${m}_module_init, ${m}_module_config, ${m}_module_reconfig}," >> modules.c echo "int ${m}_module_init(struct config_file *cfg, struct module_ctx **ctx);" >> modules.h + echo "int ${m}_module_config(struct config_file *cfg);" >> modules.h + echo "int ${m}_module_reconfig(struct config_file *cfg);" >> modules.h modules_num=`expr $modules_num + 1` done echo "};" >> modules.c @@ -658,6 +660,19 @@ do name=`echo $modline | sed 's/.*MODULE:\([^: ]*\).*/\1/'` MODULES="$MODULES $name" ;; + --disable-module) SOURCES=`echo $SOURCES | sed -e "s/${value}//g"` + if [ ! -f $value ] ; then + echo "Cannot find $value, please check path" + exit 1 + fi + modline=`grep '/***MODULE:' $value` + if [ F"$modline" = "F" ] ; then + echo "Cannot find /***MODULE line in $value, please check syntax" + exit 1 + fi + + name=`echo $modline | sed 's/.*MODULE:\([^: ]*\).*/\1/'` + MODULES=`echo $MODULES | sed -e "s/${name}//g"` ;; *) echo "$0: error: invalid option \"$option\"" exit 1 @@ -674,6 +689,7 @@ if [ "F$help" = "Fyes" ] ; then --enable-debug turn on extra debug messages --enable-opt turn on extra optimization --add-module add additional C module + --disable-module do not include C module --user=USER set user to use --group=GROUP set group to use END @@ -740,6 +756,11 @@ if [ $? -eq 0 ] ; then have_opt "SRANDOMDEV" fi +check_function "getpagesize" "unistd.h" +if [ $? -eq 0 ] ; then + have_opt "GETPAGESIZE" +fi + check_function "setproctitle" "unistd.h" if [ $? -eq 0 ] ; then have_opt "SETPROCTITLE" diff --git a/mem_pool.c b/mem_pool.c index 56fc0ef78..00986c869 100644 --- a/mem_pool.c +++ b/mem_pool.c @@ -13,6 +13,16 @@ pthread_mutex_t stat_mtx = PTHREAD_MUTEX_INITIALIZER; #define STAT_UNLOCK() do {} while (0) #endif +/* + * This define specify whether we should check all pools for free space for new object + * or just begin scan from current (recently attached) pool + * If MEMORY_GREEDY is defined, then we scan all pools to find free space (more CPU usage, slower + * but requires less memory). If it is not defined check only current pool and if object is too large + * to place in it allocate new one (this may cause huge CPU usage in some cases too, but generally faster than + * greedy method) + */ +#undef MEMORY_GREEDY + /* Internal statistic */ static size_t bytes_allocated = 0; static size_t chunks_allocated = 0; @@ -54,7 +64,11 @@ memory_pool_alloc (memory_pool_t *pool, size_t size) struct _pool_chain *new, *cur; if (pool) { +#ifdef MEMORY_GREEDY + cur = pool->first_pool; +#else cur = pool->cur_pool; +#endif /* Find free space in pool chain */ while (memory_pool_free (cur) < size && cur->next) { cur = cur->next; diff --git a/plugins/surbl.c b/plugins/surbl.c index d35ab6069..23c258326 100644 --- a/plugins/surbl.c +++ b/plugins/surbl.c @@ -49,7 +49,8 @@ struct surbl_ctx { char *metric; GHashTable *hosters; GHashTable *whitelist; - unsigned use_redirector:1; + unsigned use_redirector; + memory_pool_t *surbl_pool; }; struct redirector_param { @@ -78,11 +79,8 @@ static int surbl_test_url (struct worker_task *task); int surbl_module_init (struct config_file *cfg, struct module_ctx **ctx) { - struct hostent *hent; GError *err = NULL; - char *value, *cur_tok, *str; - surbl_module_ctx = g_malloc (sizeof (struct surbl_ctx)); surbl_module_ctx->header_filter = NULL; @@ -90,9 +88,37 @@ surbl_module_init (struct config_file *cfg, struct module_ctx **ctx) surbl_module_ctx->message_filter = NULL; surbl_module_ctx->url_filter = surbl_test_url; surbl_module_ctx->use_redirector = 0; + surbl_module_ctx->surbl_pool = memory_pool_new (1024); + + surbl_module_ctx->hosters = g_hash_table_new (g_str_hash, g_str_equal); + /* Register destructors */ + memory_pool_add_destructor (surbl_module_ctx->surbl_pool, (pool_destruct_func)g_hash_table_remove_all, surbl_module_ctx->hosters); + + surbl_module_ctx->whitelist = g_hash_table_new (g_str_hash, g_str_equal); + /* Register destructors */ + memory_pool_add_destructor (surbl_module_ctx->surbl_pool, (pool_destruct_func)g_hash_table_remove_all, surbl_module_ctx->whitelist); + + /* Init matching regexps */ + extract_hoster_regexp = g_regex_new ("([^.]+)\\.([^.]+)\\.([^.]+)$", G_REGEX_RAW, 0, &err); + extract_normal_regexp = g_regex_new ("([^.]+)\\.([^.]+)$", G_REGEX_RAW, 0, &err); + extract_numeric_regexp = g_regex_new ("(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})$", G_REGEX_RAW, 0, &err); + + *ctx = (struct module_ctx *)surbl_module_ctx; + + evdns_init (); + + return 0; +} + +int +surbl_module_config (struct config_file *cfg) +{ + struct hostent *hent; + + char *value, *cur_tok, *str; if ((value = get_module_opt (cfg, "surbl", "redirector")) != NULL) { - str = strdup (value); + str = memory_pool_strdup (surbl_module_ctx->surbl_pool, value); cur_tok = strsep (&str, ":"); if (!inet_aton (cur_tok, &surbl_module_ctx->redirector_addr)) { /* Try to call gethostbyname */ @@ -142,26 +168,26 @@ surbl_module_init (struct config_file *cfg, struct module_ctx **ctx) surbl_module_ctx->max_urls = DEFAULT_SURBL_MAX_URLS; } if ((value = get_module_opt (cfg, "surbl", "suffix")) != NULL) { - surbl_module_ctx->suffix = value; + surbl_module_ctx->suffix = memory_pool_strdup (surbl_module_ctx->surbl_pool, value); + g_free (value); } else { surbl_module_ctx->suffix = DEFAULT_SURBL_SUFFIX; } if ((value = get_module_opt (cfg, "surbl", "symbol")) != NULL) { - surbl_module_ctx->symbol = value; + surbl_module_ctx->symbol = memory_pool_strdup (surbl_module_ctx->surbl_pool, value); + g_free (value); } else { surbl_module_ctx->symbol = DEFAULT_SURBL_SYMBOL; } if ((value = get_module_opt (cfg, "surbl", "metric")) != NULL) { - surbl_module_ctx->metric = value; + surbl_module_ctx->metric = memory_pool_strdup (surbl_module_ctx->surbl_pool, value); + g_free (value); } else { surbl_module_ctx->metric = DEFAULT_METRIC; } - - surbl_module_ctx->hosters = g_hash_table_new (g_str_hash, g_str_equal); - surbl_module_ctx->whitelist = g_hash_table_new (g_str_hash, g_str_equal); if ((value = get_module_opt (cfg, "surbl", "hostings")) != NULL) { char comment_flag = 0; str = value; @@ -200,17 +226,15 @@ surbl_module_init (struct config_file *cfg, struct module_ctx **ctx) } } } - - /* Init matching regexps */ - extract_hoster_regexp = g_regex_new ("([^.]+)\\.([^.]+)\\.([^.]+)$", G_REGEX_RAW, 0, &err); - extract_normal_regexp = g_regex_new ("([^.]+)\\.([^.]+)$", G_REGEX_RAW, 0, &err); - extract_numeric_regexp = g_regex_new ("(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})$", G_REGEX_RAW, 0, &err); - - *ctx = (struct module_ctx *)surbl_module_ctx; +} - evdns_init (); +int +surbl_module_reconfig (struct config_file *cfg) +{ + memory_pool_delete (surbl_module_ctx->surbl_pool); + surbl_module_ctx->surbl_pool = memory_pool_new (1024); - return 0; + return surbl_module_config (cfg); } static char * diff --git a/url.h b/url.h index 2d9acaaf5..6987c38d1 100644 --- a/url.h +++ b/url.h @@ -39,17 +39,17 @@ struct uri { /* @protocollen should only be usable if @protocol is either * PROTOCOL_USER or an uri string should be composed. */ - unsigned int protocollen:16; - unsigned int userlen:16; - unsigned int passwordlen:16; - unsigned int hostlen:16; - unsigned int portlen:8; - unsigned int datalen:16; - unsigned int fragmentlen:16; + unsigned int protocollen; + unsigned int userlen; + unsigned int passwordlen; + unsigned int hostlen; + unsigned int portlen; + unsigned int datalen; + unsigned int fragmentlen; /* Flags */ - unsigned int ipv6:1; /* URI contains IPv6 host */ - unsigned int form:1; /* URI originated from form */ + unsigned int ipv6; /* URI contains IPv6 host */ + unsigned int form; /* URI originated from form */ /* Link */ TAILQ_ENTRY(uri) next; diff --git a/worker.c b/worker.c index 0b1594ba7..8e09a84d3 100644 --- a/worker.c +++ b/worker.c @@ -27,15 +27,9 @@ #include "upstream.h" #include "cfg_file.h" #include "url.h" +#include "modules.h" -#define CONTENT_LENGTH_HEADER "Content-Length: " -#define HELO_HEADER "Helo: " -#define FROM_HEADER "From: " -#define IP_ADDR_HEADER "IP: " -#define NRCPT_HEADER "Recipient-Number: " -#define RCPT_HEADER "Rcpt: " - -#define TASK_POOL_SIZE 16384 +#define TASK_POOL_SIZE 4095 const f_str_t CRLF = { /* begin */"\r\n", @@ -77,13 +71,9 @@ free_task (struct worker_task *task) struct mime_part *part; if (task) { - if (task->message) { - g_object_unref (task->message); - } if (task->memc_ctx) { memc_close_ctx (task->memc_ctx); } - while (!TAILQ_EMPTY (&task->parts)) { part = TAILQ_FIRST (&task->parts); g_object_unref (part->type); @@ -178,6 +168,7 @@ process_message (struct worker_task *task) message = g_mime_parser_construct_message (parser); task->message = message; + memory_pool_add_destructor (task->task_pool, (pool_destruct_func)g_object_unref, task->message); /* free the parser (and the stream) */ g_object_unref (parser); @@ -299,7 +290,11 @@ accept_socket (int fd, short what, void *arg) new_task->cfg = worker->srv->cfg; TAILQ_INIT (&new_task->urls); TAILQ_INIT (&new_task->parts); +#ifdef HAVE_GETPAGESIZE + new_task->task_pool = memory_pool_new (getpagesize () - 1); +#else new_task->task_pool = memory_pool_new (TASK_POOL_SIZE); +#endif new_task->memc_ctx = memory_pool_alloc (new_task->task_pool, sizeof (memcached_ctx_t)); if (new_task->memc_ctx == NULL) { msg_err ("accept_socket: cannot allocate memory for memcached ctx, %m"); @@ -319,6 +314,9 @@ void start_worker (struct rspamd_worker *worker, int listen_sock) { struct sigaction signals; + int i; + + worker->srv->pid = getpid (); worker->srv->type = TYPE_WORKER; event_init (); @@ -335,6 +333,11 @@ start_worker (struct rspamd_worker *worker, int listen_sock) event_set(&worker->bind_ev, listen_sock, EV_READ | EV_PERSIST, accept_socket, (void *)worker); event_add(&worker->bind_ev, NULL); + /* Perform modules configuring */ + for (i = 0; i < MODULES_NUM; i ++) { + modules[i].module_config_func (worker->srv->cfg); + } + /* Send SIGUSR2 to parent */ kill (getppid (), SIGUSR2); -- 2.39.5