Rework lists and hashes.

diff --git a/src/libserver/buffer.c b/src/libserver/buffer.c
index 403b3dafd28d4eedad4be96c0368edec99552e5c..5fbf915781233c128b8bb71ac551734404228bea 100644 (file)
--- a/src/libserver/buffer.c
+++ b/src/libserver/buffer.c
@@ -28,6 +28,7 @@
 #ifdef HAVE_SYS_SENDFILE_H
 #include <sys/sendfile.h>
 #endif
+#include "utlist.h"
 
 #define G_DISPATCHER_ERROR dispatcher_error_quark ()
 #define debug_ip(...) rspamd_conditional_debug (rspamd_main->logger, \

diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h
index 1181c126ca4c5de1e9711bd967d6a3de217eac69..0938b22dfe84b9fc662cbc0bef4760eae58654bf 100644 (file)
--- a/src/libserver/cfg_file.h
+++ b/src/libserver/cfg_file.h
@@ -11,7 +11,6 @@
 #include "upstream.h"
 #include "symbols_cache.h"
 #include "cfg_rcl.h"
-#include "utlist.h"
 #include "ucl.h"
 
 #define DEFAULT_BIND_PORT 11333
@@ -211,21 +210,6 @@ struct rspamd_worker_bind_conf {
        struct rspamd_worker_bind_conf *next;
 };
 
-struct rspamd_worker_param_parser {
-       rspamd_rcl_handler_t handler;                   /**< handler function                                                                   */
-       struct rspamd_rcl_struct_parser parser;         /**< parser attributes                                                                  */
-       const gchar *name;                              /**< parameter's name                                                                   */
-       UT_hash_handle hh;                              /**< hash by name                                                                               */
-};
-
-struct rspamd_worker_cfg_parser {
-       struct rspamd_worker_param_parser *parsers;     /**< parsers hash                                                                               */
-       gint type;                                      /**< workers quark                                                                              */
-       gboolean (*def_obj_parser)(const ucl_object_t *obj, gpointer ud);   /**< default object parser                                                          */
-       gpointer def_ud;
-       UT_hash_handle hh;                              /**< hash by type                                                                               */
-};
-
 /**
  * Config params for rspamd worker
  */
@@ -498,6 +482,11 @@ struct rspamd_classifier_config * rspamd_config_find_classifier (
 gboolean rspamd_config_parse_ip_list (const gchar *ip_list,
        radix_tree_t **tree);
 
+void rspamd_ucl_add_conf_macros (struct ucl_parser *parser,
+       struct rspamd_config *cfg);
+
+void rspamd_ucl_add_conf_variables (struct ucl_parser *parser);
+
 #endif /* ifdef CFG_FILE_H */
 /*
  * vi:ts=4

diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c
index 22ce3be50c6b84c1fb39291389854d89f4ccc7d0..2926b21c24881614a78b400601a126b027312126 100644 (file)
--- a/src/libserver/cfg_rcl.c
+++ b/src/libserver/cfg_rcl.c
@@ -23,12 +23,50 @@
 
 #include "cfg_rcl.h"
 #include "main.h"
+#include "uthash_strcase.h"
+#include "utlist.h"
 #include "cfg_file.h"
 #include "lua/lua_common.h"
 #include "expressions.h"
 #include "classifiers/classifiers.h"
 #include "tokenizers/tokenizers.h"
 
+
+struct rspamd_rcl_default_handler_data {
+       struct rspamd_rcl_struct_parser pd;
+       const gchar *key;
+       rspamd_rcl_handler_t handler;
+       UT_hash_handle hh;
+};
+
+struct rspamd_rcl_section {
+       const gchar *name;                  /**< name of section */
+       rspamd_rcl_handler_t handler;       /**< handler of section attributes */
+       enum ucl_type type;         /**< type of attribute */
+       gboolean required;                  /**< whether this param is required */
+       gboolean strict_type;               /**< whether we need strict type */
+       UT_hash_handle hh;                  /** hash handle */
+       struct rspamd_rcl_section *subsections; /**< hash table of subsections */
+       struct rspamd_rcl_default_handler_data *default_parser; /**< generic parsing fields */
+       rspamd_rcl_section_fin_t fin; /** called at the end of section parsing */
+       gpointer fin_ud;
+};
+
+struct rspamd_worker_param_parser {
+       rspamd_rcl_handler_t handler;                   /**< handler function                                                                   */
+       struct rspamd_rcl_struct_parser parser;         /**< parser attributes                                                                  */
+       const gchar *name;                              /**< parameter's name                                                                   */
+       UT_hash_handle hh;                              /**< hash by name                                                                               */
+};
+
+struct rspamd_worker_cfg_parser {
+       struct rspamd_worker_param_parser *parsers;     /**< parsers hash                                                                               */
+       gint type;                                      /**< workers quark                                                                              */
+       gboolean (*def_obj_parser)(const ucl_object_t *obj, gpointer ud);   /**< default object parser                                                          */
+       gpointer def_ud;
+       UT_hash_handle hh;                              /**< hash by type                                                                               */
+};
+
 /*
  * Common section handlers
  */
@@ -1891,3 +1929,69 @@ rspamd_rcl_register_worker_parser (struct rspamd_config *cfg, gint type,
        nparser->def_obj_parser = func;
        nparser->def_ud = ud;
 }
+
+gboolean
+rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
+       const gchar *convert_to, rspamd_rcl_section_fin_t logger_fin,
+       gpointer logger_ud)
+{
+       struct stat st;
+       gint fd;
+       gchar *data;
+       GError *err = NULL;
+       struct rspamd_rcl_section *top, *logger;
+       gboolean res;
+       struct ucl_parser *parser;
+
+       if (stat (filename, &st) == -1) {
+               msg_err ("cannot stat %s: %s", filename, strerror (errno));
+               return FALSE;
+       }
+       if ((fd = open (filename, O_RDONLY)) == -1) {
+               msg_err ("cannot open %s: %s", filename, strerror (errno));
+               return FALSE;
+
+       }
+       /* Now mmap this file to simplify reading process */
+       if ((data =
+               mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
+               msg_err ("cannot mmap %s: %s", filename, strerror (errno));
+               close (fd);
+               return FALSE;
+       }
+       close (fd);
+
+       parser = ucl_parser_new (UCL_PARSER_KEY_LOWERCASE);
+       rspamd_ucl_add_conf_variables (parser);
+       rspamd_ucl_add_conf_macros (parser, cfg);
+       if (!ucl_parser_add_chunk (parser, data, st.st_size)) {
+               msg_err ("ucl parser error: %s", ucl_parser_get_error (parser));
+               ucl_parser_free (parser);
+               munmap (data, st.st_size);
+               return FALSE;
+       }
+       munmap (data, st.st_size);
+       cfg->rcl_obj = ucl_parser_get_object (parser);
+       ucl_parser_free (parser);
+       res = TRUE;
+
+       if (!res) {
+               return FALSE;
+       }
+
+       top = rspamd_rcl_config_init ();
+       err = NULL;
+
+       HASH_FIND_STR (top, "logging", logger);
+       if (logger != NULL) {
+               logger->fin = logger_fin;
+               logger->fin_ud = logger_ud;
+       }
+
+       if (!rspamd_read_rcl_config (top, cfg, cfg->rcl_obj, &err)) {
+               msg_err ("rcl parse error: %s", err->message);
+               return FALSE;
+       }
+
+       return TRUE;
+}

diff --git a/src/libserver/cfg_rcl.h b/src/libserver/cfg_rcl.h
index 6ba197be88c8796ef16b9322cc9a45c9ca5f67f7..333a3e1eead3741003a4bbf059e49693ae12dfc4 100644 (file)
--- a/src/libserver/cfg_rcl.h
+++ b/src/libserver/cfg_rcl.h
@@ -25,8 +25,7 @@
 #define CFG_RCL_H_
 
 #include "config.h"
-#include "ucl/include/ucl.h"
-#include "uthash.h"
+#include "ucl.h"
 
 #define CFG_RCL_ERROR cfg_rcl_error_quark ()
 static inline GQuark
@@ -37,6 +36,7 @@ cfg_rcl_error_quark (void)
 
 struct rspamd_rcl_section;
 struct rspamd_config;
+struct rspamd_rcl_default_handler_data;
 
 struct rspamd_rcl_struct_parser {
        gpointer user_struct;
@@ -75,25 +75,7 @@ typedef gboolean (*rspamd_rcl_handler_t) (struct rspamd_config *cfg,
 typedef void (*rspamd_rcl_section_fin_t)(struct rspamd_config *cfg,
        gpointer ud);
 
-struct rspamd_rcl_default_handler_data {
-       struct rspamd_rcl_struct_parser pd;
-       const gchar *key;
-       rspamd_rcl_handler_t handler;
-       UT_hash_handle hh;
-};
 
-struct rspamd_rcl_section {
-       const gchar *name;                  /**< name of section */
-       rspamd_rcl_handler_t handler;       /**< handler of section attributes */
-       enum ucl_type type;         /**< type of attribute */
-       gboolean required;                  /**< whether this param is required */
-       gboolean strict_type;               /**< whether we need strict type */
-       UT_hash_handle hh;                  /** hash handle */
-       struct rspamd_rcl_section *subsections; /**< hash table of subsections */
-       struct rspamd_rcl_default_handler_data *default_parser; /**< generic parsing fields */
-       rspamd_rcl_section_fin_t fin; /** called at the end of section parsing */
-       gpointer fin_ud;
-};
 
 /**
  * Init common sections known to rspamd

diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c
index 2eae0b0c3c5421f990c0b9b406823bafeef7c8b0..d5fbe70ac4a14d649316bb605c9dda227646f316 100644 (file)
--- a/src/libserver/cfg_utils.c
+++ b/src/libserver/cfg_utils.c
@@ -27,13 +27,14 @@
 
 #include "cfg_file.h"
 #include "main.h"
+#include "uthash_strcase.h"
 #include "filter.h"
 #include "classifiers/classifiers.h"
 #include "lua/lua_common.h"
 #include "kvstorage_config.h"
 #include "map.h"
 #include "dynamic_cfg.h"
-#include "xxhash.h"
+#include "utlist.h"
 
 #define DEFAULT_SCORE 10.0
 
@@ -799,7 +800,7 @@ rspamd_include_map_handler (const guchar *data, gsize len, void * ud)
 #define RSPAMD_PREFIX_MACRO "PREFIX"
 #define RSPAMD_VERSION_MACRO "VERSION"
 
-static void
+void
 rspamd_ucl_add_conf_variables (struct ucl_parser *parser)
 {
        ucl_parser_register_variable (parser,
@@ -821,7 +822,7 @@ rspamd_ucl_add_conf_variables (struct ucl_parser *parser)
        ucl_parser_register_variable (parser, RSPAMD_VERSION_MACRO, RVERSION);
 }
 
-static void
+void
 rspamd_ucl_add_conf_macros (struct ucl_parser *parser,
        struct rspamd_config *cfg)
 {
@@ -831,72 +832,6 @@ rspamd_ucl_add_conf_macros (struct ucl_parser *parser,
                cfg);
 }
 
-gboolean
-rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
-       const gchar *convert_to, rspamd_rcl_section_fin_t logger_fin,
-       gpointer logger_ud)
-{
-       struct stat st;
-       gint fd;
-       gchar *data;
-       GError *err = NULL;
-       struct rspamd_rcl_section *top, *logger;
-       gboolean res;
-       struct ucl_parser *parser;
-
-       if (stat (filename, &st) == -1) {
-               msg_err ("cannot stat %s: %s", filename, strerror (errno));
-               return FALSE;
-       }
-       if ((fd = open (filename, O_RDONLY)) == -1) {
-               msg_err ("cannot open %s: %s", filename, strerror (errno));
-               return FALSE;
-
-       }
-       /* Now mmap this file to simplify reading process */
-       if ((data =
-               mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) {
-               msg_err ("cannot mmap %s: %s", filename, strerror (errno));
-               close (fd);
-               return FALSE;
-       }
-       close (fd);
-
-       parser = ucl_parser_new (0);
-       rspamd_ucl_add_conf_variables (parser);
-       rspamd_ucl_add_conf_macros (parser, cfg);
-       if (!ucl_parser_add_chunk (parser, data, st.st_size)) {
-               msg_err ("ucl parser error: %s", ucl_parser_get_error (parser));
-               ucl_parser_free (parser);
-               munmap (data, st.st_size);
-               return FALSE;
-       }
-       munmap (data, st.st_size);
-       cfg->rcl_obj = ucl_parser_get_object (parser);
-       ucl_parser_free (parser);
-       res = TRUE;
-
-       if (!res) {
-               return FALSE;
-       }
-
-       top = rspamd_rcl_config_init ();
-       err = NULL;
-
-       HASH_FIND_STR (top, "logging", logger);
-       if (logger != NULL) {
-               logger->fin = logger_fin;
-               logger->fin_ud = logger_ud;
-       }
-
-       if (!rspamd_read_rcl_config (top, cfg, cfg->rcl_obj, &err)) {
-               msg_err ("rcl parse error: %s", err->message);
-               return FALSE;
-       }
-
-       return TRUE;
-}
-
 static void
 symbols_classifiers_callback (gpointer key, gpointer value, gpointer ud)
 {

diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c
index c074153ab2273eb3bb7d9180f8118decc47206cd..6bf9f90f6a1a2c472ee02b33aa3502ee9c3bb171 100644 (file)
--- a/src/libserver/dkim.c
+++ b/src/libserver/dkim.c
@@ -26,6 +26,7 @@
 #include "message.h"
 #include "dkim.h"
 #include "dns.h"
+#include "utlist.h"
 
 /* Parser of dkim params */
 typedef gboolean (*dkim_parse_param_f) (rspamd_dkim_context_t * ctx,

diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c
index 2e9e9c9e75af715180ec7355f3e04ebf412d2b6d..489a58bb431b370a67e069ecb25fc19f0e651876 100644 (file)
--- a/src/libserver/protocol.c
+++ b/src/libserver/protocol.c
@@ -27,6 +27,7 @@
 #include "util.h"
 #include "cfg_file.h"
 #include "message.h"
+#include "utlist.h"
 
 /* Max line size */
 #define OUTBUFSIZ BUFSIZ

diff --git a/src/libserver/spf.c b/src/libserver/spf.c
index 0ef952a31c0a95b4bd7417dd01ce35d30eea23e7..98a95bdf3049b04113854df7f39475e3f4596ebe 100644 (file)
--- a/src/libserver/spf.c
+++ b/src/libserver/spf.c
@@ -28,6 +28,7 @@
 #include "main.h"
 #include "message.h"
 #include "filter.h"
+#include "utlist.h"
 
 #define SPF_VER1_STR "v=spf1"
 #define SPF_VER2_STR "spf2."

diff --git a/src/libutil/hash.c b/src/libutil/hash.c
index 7e4e7f27e621726a0346d9804fbb25797807cf26..ec952e87bb37ad0dcd68643c9daa008e0e6b5a14 100644 (file)
--- a/src/libutil/hash.c
+++ b/src/libutil/hash.c
@@ -24,6 +24,7 @@
 
 #include "config.h"
 #include "hash.h"
+#define HASH_CASELESS
 #include "uthash_strcase.h"
 #include "utlist.h"
 
@@ -194,7 +195,7 @@ rspamd_lru_hash_insert (rspamd_lru_hash_t *hash, gpointer key, gpointer value,
        }
 
        res = rspamd_lru_create_node (hash, key, value, now, ttl);
-       HASH_ADD (hh, hash->elements, key, strlen (key), res);
+       HASH_ADD_KEYPTR (hh, hash->elements, key, strlen (key), res);
 }
 
 void

diff --git a/src/libutil/uthash_strcase.h b/src/libutil/uthash_strcase.h
index 1b4024cc6978167ca25ce0709a9fbc713c182fdd..4e28f31105882afce5b4aa277db185a7ac68f36c 100644 (file)
--- a/src/libutil/uthash_strcase.h
+++ b/src/libutil/uthash_strcase.h
@@ -25,13 +25,28 @@
 
 #include "xxhash.h"
 
+
 /* Utils for uthash tuning */
+#ifndef HASH_CASELESS
 #define HASH_FUNCTION(key,keylen,num_bkts,hashv,bkt) do {\
        hashv = XXH32(key, keylen, 0); \
        bkt = (hashv) & (num_bkts-1); \
 } while (0)
 
+#define HASH_KEYCMP(a,b,len) memcmp(a,b,len)
+#else
+#define HASH_FUNCTION(key,keylen,num_bkts,hashv,bkt) do {\
+       void *xxh = XXH32_init(0xdead); \
+       unsigned char *p = (unsigned char *)key, t;     \
+       for (unsigned int i = 0; i < keylen; i ++) {    \
+               t = g_ascii_tolower(p[i]);      \
+               XXH32_update(xxh, &t, 1);       \
+       }       \
+       hashv = XXH32_digest(xxh);      \
+       bkt = (hashv) & (num_bkts-1);   \
+} while (0)
 #define HASH_KEYCMP(a,b,len) strncasecmp(a,b,len)
+#endif
 
 #include "uthash.h"
 

diff --git a/src/lua/lua_dns.c b/src/lua/lua_dns.c
index fd3037b70b1af4c0a41aa66deb400a1e26f41ee5..bd130694713f8942c9d00676bd6e46103141de06 100644 (file)
--- a/src/lua/lua_dns.c
+++ b/src/lua/lua_dns.c
@@ -23,6 +23,7 @@
 
 #include "lua_common.h"
 #include "dns.h"
+#include "utlist.h"
 
 /* Public prototypes */
 struct rspamd_dns_resolver * lua_check_dns_resolver (lua_State * L);

diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c
index 7f44d8615eff83483ede11e564ef3b90b304840c..d1a4434c2ae13a7498830d8ee6fb39a7a28a102b 100644 (file)
--- a/src/lua/lua_http.c
+++ b/src/lua/lua_http.c
@@ -25,6 +25,7 @@
 #include "buffer.h"
 #include "dns.h"
 #include "http.h"
+#include "utlist.h"
 
 #define MAX_HEADERS_SIZE 8192
 

diff --git a/src/main.c b/src/main.c
index 85553dfc1babd4711f6654e5a0a0b865fc364c9c..d912eb6a36d073f5869d96c05d2f56e9043cfecd 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -35,6 +35,7 @@
 #include "lua/lua_common.h"
 #include "ottery.h"
 #include "xxhash.h"
+#include "utlist.h"
 #ifdef HAVE_OPENSSL
 #include <openssl/rand.h>
 #include <openssl/err.h>

diff --git a/src/main.h b/src/main.h
index 577cd5c425b90166d1a629c1a8a76cf113c8cacf..537cf575c85cf90f31227fdb63b7db343435d75a 100644 (file)
--- a/src/main.h
+++ b/src/main.h
@@ -14,7 +14,6 @@
 #include "protocol.h"
 #include "filter.h"
 #include "buffer.h"
-#include "hash.h"
 #include "events.h"
 #include "util.h"
 #include "logger.h"
@@ -101,7 +100,6 @@ struct rspamd_main {
        rspamd_mempool_t *server_pool;                                  /**< server's memory pool                                                       */
        statfile_pool_t *statfile_pool;                             /**< shared statfiles pool                                                  */
        GHashTable *workers;                                        /**< workers pool indexed by pid                    */
-       rspamd_hash_t *counters;                                    /**< symbol cache counters                                                  */
        rspamd_logger_t *logger;
        uid_t workers_uid;                                          /**< worker's uid running to                        */
        gid_t workers_gid;                                          /**< worker's gid running to                                                */
@@ -109,11 +107,6 @@ struct rspamd_main {
        struct roll_history *history;                               /**< rolling history                                                                */
 };
 
-struct counter_data {
-       guint64 value;
-       gint number;
-};
-
 /**
  * Structure to point exception in text from processing
  */

diff --git a/src/plugins/dkim_check.c b/src/plugins/dkim_check.c
index 6a162af40a8b78ae43ae55a9ffbccd5757ec76bb..3227902c391fceaf3541fac8f918a5cc1700151e 100644 (file)
--- a/src/plugins/dkim_check.c
+++ b/src/plugins/dkim_check.c
@@ -216,8 +216,7 @@ dkim_module_config (struct rspamd_config *cfg)
                        dkim_module_ctx->symbol_allow,
                        1);
 
-               dkim_module_ctx->dkim_hash = rspamd_lru_hash_new (rspamd_strcase_hash,
-                               rspamd_strcase_equal,
+               dkim_module_ctx->dkim_hash = rspamd_lru_hash_new (
                                cache_size,
                                cache_expire,
                                g_free,

diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c
index 97b253d9c800203d4d6ff2693eb2b2ebaa08a6ac..4604abbcc166bdd2da62c86408cc183e9bd20935 100644 (file)
--- a/src/plugins/fuzzy_check.c
+++ b/src/plugins/fuzzy_check.c
@@ -48,6 +48,7 @@
 #include "map.h"
 #include "images.h"
 #include "fuzzy_storage.h"
+#include "utlist.h"
 
 #define DEFAULT_SYMBOL "R_FUZZY_HASH"
 #define DEFAULT_UPSTREAM_ERROR_TIME 10

diff --git a/src/plugins/spf.c b/src/plugins/spf.c
index bf0956e78b6531115526a5e311bcb24aa3582cbd..1e40ed3d978ac89f89d65a908834370724ab7ed4 100644 (file)
--- a/src/plugins/spf.c
+++ b/src/plugins/spf.c
@@ -154,8 +154,7 @@ spf_module_config (struct rspamd_config *cfg)
        register_virtual_symbol (&cfg->cache, spf_module_ctx->symbol_softfail, 1);
        register_virtual_symbol (&cfg->cache, spf_module_ctx->symbol_allow,        1);
 
-       spf_module_ctx->spf_hash = rspamd_lru_hash_new (rspamd_strcase_hash,
-                       rspamd_strcase_equal,
+       spf_module_ctx->spf_hash = rspamd_lru_hash_new (
                        cache_size,
                        cache_expire,
                        g_free,

diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index be5a6027688b37aaf14c5798a5b9f7cfea6c239c..bf143b0f433357b84c314f2ab4143efeb3be9bff 100644 (file)
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -50,6 +50,7 @@
 #include "map.h"
 #include "dns.h"
 #include "hash.h"
+#include "utlist.h"
 
 #include "surbl.h"
 

diff --git a/src/smtp_proxy.c b/src/smtp_proxy.c
index 0b7660a81b4f91a43ec22276b4879a759d668933..e992a4b37555041d7f5277e64d97b62e218ab067 100644 (file)
--- a/src/smtp_proxy.c
+++ b/src/smtp_proxy.c
@@ -31,6 +31,7 @@
 #include "dns.h"
 #include "upstream.h"
 #include "proxy.h"
+#include "utlist.h"
 
 /*
  * SMTP proxy is a simple smtp proxy worker for dns resolving and

diff --git a/src/ucl b/src/ucl
index 3e1ff238ab86fdb951b2664c25563414a84ce0c9..006dd46060d6aa44191a3e404b975ba93bfb9b99 160000 (submodule)
--- a/src/ucl
+++ b/src/ucl
@@ -1 +1 @@
-Subproject commit 3e1ff238ab86fdb951b2664c25563414a84ce0c9
+Subproject commit 006dd46060d6aa44191a3e404b975ba93bfb9b99