@@ -936,7 +936,8 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session) | |||
hexbuf, sizeof (hexbuf) - 1); | |||
hexbuf[sizeof (hexbuf) - 1] = '\0'; | |||
if (rspamd_match_hash_map (session->ctx->skip_hashes, hexbuf)) { | |||
if (rspamd_match_hash_map (session->ctx->skip_hashes, | |||
hexbuf, sizeof (hexbuf) - 1)) { | |||
result.v1.value = 401; | |||
result.v1.prob = 0.0f; | |||
@@ -2010,7 +2011,7 @@ start_fuzzy (struct rspamd_worker *worker) | |||
rspamd_kv_list_fin, | |||
rspamd_kv_list_dtor, | |||
(void **)&ctx->skip_hashes, | |||
worker)) == NULL) { | |||
worker, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
msg_warn_config ("cannot load hashes list from %s", | |||
ucl_object_tostring (ctx->skip_map)); | |||
} |
@@ -1180,7 +1180,7 @@ rspamd_include_map_handler (const guchar *data, gsize len, | |||
rspamd_ucl_fin_cb, | |||
rspamd_ucl_dtor_cb, | |||
(void **)pcbdata, | |||
NULL) != NULL; | |||
NULL, RSPAMD_MAP_DEFAULT) != NULL; | |||
} | |||
/* | |||
@@ -2240,7 +2240,7 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg, | |||
rspamd_radix_fin, | |||
rspamd_radix_dtor, | |||
(void **)target, | |||
worker) == NULL) { | |||
worker, RSPAMD_MAP_DEFAULT) == NULL) { | |||
g_set_error (err, | |||
g_quark_from_static_string ("rspamd-config"), | |||
EINVAL, "bad map definition %s for %s", str, | |||
@@ -2267,7 +2267,7 @@ rspamd_config_radix_from_ucl (struct rspamd_config *cfg, | |||
rspamd_radix_fin, | |||
rspamd_radix_dtor, | |||
(void **)target, | |||
worker) == NULL) { | |||
worker, RSPAMD_MAP_DEFAULT) == NULL) { | |||
g_set_error (err, | |||
g_quark_from_static_string ("rspamd-config"), | |||
EINVAL, "bad map object for %s", ucl_object_key (obj)); |
@@ -283,7 +283,7 @@ init_dynamic_config (struct rspamd_config *cfg) | |||
json_config_read_cb, | |||
json_config_fin_cb, | |||
json_config_dtor_cb, | |||
(void **)pjb, NULL)) { | |||
(void **)pjb, NULL, RSPAMD_MAP_DEFAULT)) { | |||
msg_err ("cannot add map for configuration %s", cfg->dynamic_conf); | |||
} | |||
} |
@@ -788,72 +788,79 @@ read_map_file (struct rspamd_map *map, struct file_map_data *data, | |||
} | |||
if (len > 0) { | |||
if (bk->is_compressed) { | |||
bytes = rspamd_file_xmap (data->filename, PROT_READ, &len, TRUE); | |||
if (bytes == NULL) { | |||
msg_err_map ("can't open map %s: %s", data->filename, strerror (errno)); | |||
return FALSE; | |||
} | |||
ZSTD_DStream *zstream; | |||
ZSTD_inBuffer zin; | |||
ZSTD_outBuffer zout; | |||
guchar *out; | |||
gsize outlen, r; | |||
if (map->no_file_read) { | |||
/* We just call read callback with backend name */ | |||
map->read_callback (data->filename, strlen (data->filename), | |||
&periodic->cbdata, TRUE); | |||
} | |||
else { | |||
if (bk->is_compressed) { | |||
bytes = rspamd_file_xmap (data->filename, PROT_READ, &len, TRUE); | |||
zstream = ZSTD_createDStream (); | |||
ZSTD_initDStream (zstream); | |||
if (bytes == NULL) { | |||
msg_err_map ("can't open map %s: %s", data->filename, strerror (errno)); | |||
return FALSE; | |||
} | |||
zin.pos = 0; | |||
zin.src = bytes; | |||
zin.size = len; | |||
ZSTD_DStream *zstream; | |||
ZSTD_inBuffer zin; | |||
ZSTD_outBuffer zout; | |||
guchar *out; | |||
gsize outlen, r; | |||
if ((outlen = ZSTD_getDecompressedSize (zin.src, zin.size)) == 0) { | |||
outlen = ZSTD_DStreamOutSize (); | |||
} | |||
zstream = ZSTD_createDStream (); | |||
ZSTD_initDStream (zstream); | |||
out = g_malloc (outlen); | |||
zin.pos = 0; | |||
zin.src = bytes; | |||
zin.size = len; | |||
zout.dst = out; | |||
zout.pos = 0; | |||
zout.size = outlen; | |||
if ((outlen = ZSTD_getDecompressedSize (zin.src, zin.size)) == 0) { | |||
outlen = ZSTD_DStreamOutSize (); | |||
} | |||
while (zin.pos < zin.size) { | |||
r = ZSTD_decompressStream (zstream, &zout, &zin); | |||
out = g_malloc (outlen); | |||
if (ZSTD_isError (r)) { | |||
msg_err_map ("%s: cannot decompress data: %s", | |||
data->filename, | |||
ZSTD_getErrorName (r)); | |||
ZSTD_freeDStream (zstream); | |||
g_free (out); | |||
munmap (bytes, len); | |||
return FALSE; | |||
} | |||
zout.dst = out; | |||
zout.pos = 0; | |||
zout.size = outlen; | |||
while (zin.pos < zin.size) { | |||
r = ZSTD_decompressStream (zstream, &zout, &zin); | |||
if (ZSTD_isError (r)) { | |||
msg_err_map ("%s: cannot decompress data: %s", | |||
data->filename, | |||
ZSTD_getErrorName (r)); | |||
ZSTD_freeDStream (zstream); | |||
g_free (out); | |||
munmap (bytes, len); | |||
return FALSE; | |||
} | |||
if (zout.pos == zout.size) { | |||
/* We need to extend output buffer */ | |||
zout.size = zout.size * 2 + 1; | |||
out = g_realloc (zout.dst, zout.size); | |||
zout.dst = out; | |||
if (zout.pos == zout.size) { | |||
/* We need to extend output buffer */ | |||
zout.size = zout.size * 2 + 1; | |||
out = g_realloc (zout.dst, zout.size); | |||
zout.dst = out; | |||
} | |||
} | |||
} | |||
ZSTD_freeDStream (zstream); | |||
msg_info_map ("%s: read map data, %z bytes compressed, " | |||
"%z uncompressed)", data->filename, | |||
len, zout.pos); | |||
map->read_callback (out, zout.pos, &periodic->cbdata, TRUE); | |||
g_free (out); | |||
ZSTD_freeDStream (zstream); | |||
msg_info_map ("%s: read map data, %z bytes compressed, " | |||
"%z uncompressed)", data->filename, | |||
len, zout.pos); | |||
map->read_callback (out, zout.pos, &periodic->cbdata, TRUE); | |||
g_free (out); | |||
munmap (bytes, len); | |||
} | |||
else { | |||
/* Perform buffered read: fail-safe */ | |||
if (!read_map_file_chunks (map, &periodic->cbdata, data->filename, | |||
len, 0)) { | |||
return FALSE; | |||
munmap (bytes, len); | |||
} | |||
else { | |||
/* Perform buffered read: fail-safe */ | |||
if (!read_map_file_chunks (map, &periodic->cbdata, data->filename, | |||
len, 0)) { | |||
return FALSE; | |||
} | |||
} | |||
} | |||
} | |||
@@ -2610,7 +2617,8 @@ rspamd_map_add (struct rspamd_config *cfg, | |||
map_fin_cb_t fin_callback, | |||
map_dtor_t dtor, | |||
void **user_data, | |||
struct rspamd_worker *worker) | |||
struct rspamd_worker *worker, | |||
int flags) | |||
{ | |||
struct rspamd_map *map; | |||
struct rspamd_map_backend *bk; | |||
@@ -2642,6 +2650,7 @@ rspamd_map_add (struct rspamd_config *cfg, | |||
map->backends); | |||
g_ptr_array_add (map->backends, bk); | |||
map->name = rspamd_mempool_strdup (cfg->cfg_pool, map_line); | |||
map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ); | |||
if (bk->protocol == MAP_PROTO_FILE) { | |||
map->poll_timeout = (cfg->map_timeout * cfg->map_file_watch_multiplier); | |||
@@ -2685,7 +2694,8 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg, | |||
map_fin_cb_t fin_callback, | |||
map_dtor_t dtor, | |||
void **user_data, | |||
struct rspamd_worker *worker) | |||
struct rspamd_worker *worker, | |||
gint flags) | |||
{ | |||
ucl_object_iter_t it = NULL; | |||
const ucl_object_t *cur, *elt; | |||
@@ -2698,7 +2708,7 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg, | |||
if (ucl_object_type (obj) == UCL_STRING) { | |||
/* Just a plain string */ | |||
return rspamd_map_add (cfg, ucl_object_tostring (obj), description, | |||
read_callback, fin_callback, dtor, user_data, worker); | |||
read_callback, fin_callback, dtor, user_data, worker, flags); | |||
} | |||
map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_map)); | |||
@@ -2712,6 +2722,7 @@ rspamd_map_add_from_ucl (struct rspamd_config *cfg, | |||
rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); | |||
map->backends = g_ptr_array_new (); | |||
map->wrk = worker; | |||
map->no_file_read = (flags & RSPAMD_MAP_FILE_NO_READ); | |||
rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_ptr_array_free_hard, | |||
map->backends); | |||
map->poll_timeout = cfg->map_timeout; |
@@ -61,6 +61,12 @@ struct map_cb_data { | |||
*/ | |||
gboolean rspamd_map_is_map (const gchar *map_line); | |||
enum rspamd_map_flags { | |||
RSPAMD_MAP_DEFAULT = 0, | |||
RSPAMD_MAP_FILE_ONLY = 1u << 0u, | |||
RSPAMD_MAP_FILE_NO_READ = 1u << 1u, | |||
}; | |||
/** | |||
* Add map from line | |||
*/ | |||
@@ -71,7 +77,8 @@ struct rspamd_map *rspamd_map_add (struct rspamd_config *cfg, | |||
map_fin_cb_t fin_callback, | |||
map_dtor_t dtor, | |||
void **user_data, | |||
struct rspamd_worker *worker); | |||
struct rspamd_worker *worker, | |||
int flags); | |||
/** | |||
* Add map from ucl | |||
@@ -83,7 +90,8 @@ struct rspamd_map *rspamd_map_add_from_ucl (struct rspamd_config *cfg, | |||
map_fin_cb_t fin_callback, | |||
map_dtor_t dtor, | |||
void **user_data, | |||
struct rspamd_worker *worker); | |||
struct rspamd_worker *worker, | |||
int flags); | |||
enum rspamd_map_watch_type { | |||
RSPAMD_MAP_WATCH_MIN = 9, |
@@ -21,6 +21,7 @@ | |||
#include "rspamd.h" | |||
#include "cryptobox.h" | |||
#include "contrib/fastutf8/fastutf8.h" | |||
#include "contrib/cdb/cdb.h" | |||
#ifdef WITH_HYPERSCAN | |||
#include "hs.h" | |||
@@ -58,6 +59,12 @@ struct rspamd_hash_map_helper { | |||
rspamd_cryptobox_fast_hash_state_t hst; | |||
}; | |||
struct rspamd_cdb_map_helper { | |||
GQueue cdbs; | |||
rspamd_cryptobox_fast_hash_state_t hst; | |||
gsize total_size; | |||
}; | |||
struct rspamd_regexp_map_helper { | |||
rspamd_mempool_t *pool; | |||
struct rspamd_map *map; | |||
@@ -1332,7 +1339,8 @@ rspamd_match_regexp_map_all (struct rspamd_regexp_map_helper *map, | |||
} | |||
gconstpointer | |||
rspamd_match_hash_map (struct rspamd_hash_map_helper *map, const gchar *in) | |||
rspamd_match_hash_map (struct rspamd_hash_map_helper *map, const gchar *in, | |||
gsize len) | |||
{ | |||
khiter_t k; | |||
struct rspamd_map_helper_value *val; | |||
@@ -1393,5 +1401,175 @@ rspamd_match_radix_map_addr (struct rspamd_radix_map_helper *map, | |||
return val->value; | |||
} | |||
return NULL; | |||
} | |||
/* | |||
* CBD stuff | |||
*/ | |||
struct rspamd_cdb_map_helper * | |||
rspamd_map_helper_new_cdb (struct rspamd_map *map) | |||
{ | |||
struct rspamd_cdb_map_helper *n; | |||
n = g_malloc0 (sizeof (*n)); | |||
n->cdbs = (GQueue)G_QUEUE_INIT; | |||
rspamd_cryptobox_fast_hash_init (&n->hst, map_hash_seed); | |||
return n; | |||
} | |||
void | |||
rspamd_map_helper_destroy_cdb (struct rspamd_cdb_map_helper *c) | |||
{ | |||
if (c == NULL) { | |||
return; | |||
} | |||
GList *cur = c->cdbs.head; | |||
while (cur) { | |||
struct cdb *cdb = (struct cdb *)cur->data; | |||
cdb_free (cdb); | |||
g_free (cdb->filename); | |||
close (cdb->cdb_fd); | |||
g_free (cdb); | |||
cur = g_list_next (cur); | |||
} | |||
g_queue_clear (&c->cdbs); | |||
g_free (c); | |||
} | |||
gchar * | |||
rspamd_cdb_list_read (gchar *chunk, | |||
gint len, | |||
struct map_cb_data *data, | |||
gboolean final) | |||
{ | |||
struct rspamd_cdb_map_helper *cdb_data; | |||
struct cdb *found = NULL; | |||
struct rspamd_map *map = data->map; | |||
g_assert (map->no_file_read); | |||
if (data->cur_data == NULL) { | |||
cdb_data = rspamd_map_helper_new_cdb (data->map); | |||
data->cur_data = cdb_data; | |||
} | |||
else { | |||
cdb_data = (struct rspamd_cdb_map_helper *)data->cur_data; | |||
} | |||
GList *cur = cdb_data->cdbs.head; | |||
while (cur) { | |||
struct cdb *elt = (struct cdb *)cur->data; | |||
if (strcmp (elt->filename, chunk) == 0) { | |||
found = elt; | |||
break; | |||
} | |||
cur = g_list_next (cur); | |||
} | |||
if (found == NULL) { | |||
/* New cdb */ | |||
gint fd; | |||
struct cdb *cdb; | |||
fd = rspamd_file_xopen (chunk, O_RDONLY, 0, TRUE); | |||
if (fd == -1) { | |||
msg_err_map ("cannot open cdb map from %s: %s", chunk, strerror (errno)); | |||
return NULL; | |||
} | |||
cdb = g_malloc0 (sizeof (struct cdb)); | |||
if (cdb_init (cdb, fd) == -1) { | |||
msg_err_map ("cannot init cdb map from %s: %s", chunk, strerror (errno)); | |||
return NULL; | |||
} | |||
cdb->filename = g_strdup (chunk); | |||
g_queue_push_tail (&cdb_data->cdbs, cdb); | |||
cdb_data->total_size += cdb->cdb_fsize; | |||
rspamd_cryptobox_fast_hash_update (&cdb_data->hst, chunk, len); | |||
} | |||
return chunk + len; | |||
} | |||
void | |||
rspamd_cdb_list_fin (struct map_cb_data *data, void **target) | |||
{ | |||
struct rspamd_map *map = data->map; | |||
struct rspamd_cdb_map_helper *cdb_data; | |||
if (data->cur_data) { | |||
cdb_data = (struct rspamd_cdb_map_helper *)data->cur_data; | |||
msg_info_map ("read cdb of %Hz size", cdb_data->total_size); | |||
data->map->traverse_function = NULL; | |||
data->map->nelts = 0; | |||
data->map->digest = rspamd_cryptobox_fast_hash_final (&cdb_data->hst); | |||
} | |||
if (target) { | |||
*target = data->cur_data; | |||
} | |||
if (data->prev_data) { | |||
cdb_data = (struct rspamd_cdb_map_helper *)data->prev_data; | |||
rspamd_map_helper_destroy_cdb (cdb_data); | |||
} | |||
} | |||
void | |||
rspamd_cdb_list_dtor (struct map_cb_data *data) | |||
{ | |||
if (data->cur_data) { | |||
rspamd_map_helper_destroy_cdb (data->cur_data); | |||
} | |||
} | |||
gconstpointer | |||
rspamd_match_cdb_map (struct rspamd_cdb_map_helper *map, | |||
const gchar *in, gsize inlen) | |||
{ | |||
if (map == NULL || map->cdbs.head == NULL) { | |||
return NULL; | |||
} | |||
GList *cur = map->cdbs.head; | |||
static rspamd_ftok_t found; | |||
while (cur) { | |||
struct cdb *cdb = (struct cdb *)cur->data; | |||
if (cdb_find (cdb, in, inlen) > 0) { | |||
/* Extract and push value to lua as string */ | |||
unsigned vlen; | |||
gconstpointer vpos; | |||
vpos = cdb->cdb_mem + cdb_datapos (cdb); | |||
vlen = cdb_datalen (cdb); | |||
found.len = vlen; | |||
found.begin = vpos; | |||
return &found; /* Do not reuse! */ | |||
} | |||
cur = g_list_next (cur); | |||
} | |||
return NULL; | |||
} |
@@ -38,6 +38,7 @@ extern "C" { | |||
struct rspamd_radix_map_helper; | |||
struct rspamd_hash_map_helper; | |||
struct rspamd_regexp_map_helper; | |||
struct rspamd_cdb_map_helper; | |||
struct rspamd_map_helper_value; | |||
enum rspamd_regexp_map_flags { | |||
@@ -75,6 +76,18 @@ void rspamd_kv_list_fin (struct map_cb_data *data, void **target); | |||
void rspamd_kv_list_dtor (struct map_cb_data *data); | |||
/** | |||
* Cdb is a cdb mapped file with shared data | |||
* chunk must be filename! | |||
*/ | |||
gchar *rspamd_cdb_list_read ( | |||
gchar *chunk, | |||
gint len, | |||
struct map_cb_data *data, | |||
gboolean final); | |||
void rspamd_cdb_list_fin (struct map_cb_data *data, void **target); | |||
void rspamd_cdb_list_dtor (struct map_cb_data *data); | |||
/** | |||
* Regexp list is a list of regular expressions | |||
*/ | |||
@@ -149,7 +162,17 @@ GPtrArray *rspamd_match_regexp_map_all (struct rspamd_regexp_map_helper *map, | |||
* @return | |||
*/ | |||
gconstpointer rspamd_match_hash_map (struct rspamd_hash_map_helper *map, | |||
const gchar *in); | |||
const gchar *in, gsize len); | |||
/** | |||
* Find value matching specific key in a cdb map | |||
* @param map | |||
* @param in | |||
* @param len | |||
* @return rspamd_ftok_t pointer (allocated in a static buffer!) | |||
*/ | |||
gconstpointer rspamd_match_cdb_map (struct rspamd_cdb_map_helper *map, | |||
const gchar *in, gsize len); | |||
/** | |||
* Find value matching specific key in a hash map |
@@ -155,10 +155,11 @@ struct rspamd_map { | |||
ev_tstamp timeout; | |||
gdouble poll_timeout; | |||
time_t next_check; | |||
gboolean active_http; | |||
gboolean non_trivial; /* E.g. has http backends in active mode */ | |||
gboolean file_only; /* No HTTP backends found */ | |||
gboolean static_only; /* No need to check */ | |||
bool active_http; | |||
bool non_trivial; /* E.g. has http backends in active mode */ | |||
bool file_only; /* No HTTP backends found */ | |||
bool static_only; /* No need to check */ | |||
bool no_file_read; /* Do not read files */ | |||
/* Shared lock for temporary disabling of map reading (e.g. when this map is written by UI) */ | |||
gint *locked; | |||
gchar tag[MEMPOOL_UID_LEN]; |
@@ -66,6 +66,8 @@ lua_cdb_create (lua_State *L) | |||
cdb = g_malloc (sizeof (struct cdb)); | |||
cdb->filename = g_strdup (filename); | |||
if (cdb_init (cdb, fd) == -1) { | |||
g_free (cdb->filename); | |||
g_free (cdb); | |||
msg_warn ("cannot open cdb: %s, %s", filename, strerror (errno)); | |||
lua_pushnil (L); | |||
} |
@@ -124,6 +124,7 @@ enum rspamd_lua_map_type { | |||
RSPAMD_LUA_MAP_REGEXP, | |||
RSPAMD_LUA_MAP_REGEXP_MULTIPLE, | |||
RSPAMD_LUA_MAP_CALLBACK, | |||
RSPAMD_LUA_MAP_CDB, | |||
RSPAMD_LUA_MAP_UNKNOWN, | |||
}; | |||
@@ -136,6 +137,7 @@ struct rspamd_lua_map { | |||
struct rspamd_radix_map_helper *radix; | |||
struct rspamd_hash_map_helper *hash; | |||
struct rspamd_regexp_map_helper *re_map; | |||
struct rspamd_cdb_map_helper *cdb_map; | |||
struct lua_map_callback_data *cbdata; | |||
} data; | |||
}; |
@@ -162,7 +162,7 @@ lua_config_add_radix_map (lua_State *L) | |||
rspamd_radix_fin, | |||
rspamd_radix_dtor, | |||
(void **)&map->data.radix, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
msg_warn_config ("invalid radix map %s", map_line); | |||
lua_pushnil (L); | |||
@@ -220,7 +220,7 @@ lua_config_radix_from_config (lua_State *L) | |||
rspamd_radix_fin, | |||
rspamd_radix_dtor, | |||
(void **)&map->data.radix, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
msg_err_config ("invalid radix map static"); | |||
lua_pushnil (L); | |||
ucl_object_unref (fake_obj); | |||
@@ -282,7 +282,7 @@ lua_config_radix_from_ucl (lua_State *L) | |||
rspamd_radix_fin, | |||
rspamd_radix_dtor, | |||
(void **)&map->data.radix, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
msg_err_config ("invalid radix map static"); | |||
lua_pushnil (L); | |||
ucl_object_unref (fake_obj); | |||
@@ -328,7 +328,7 @@ lua_config_add_hash_map (lua_State *L) | |||
rspamd_kv_list_fin, | |||
rspamd_kv_list_dtor, | |||
(void **)&map->data.hash, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
msg_warn_config ("invalid set map %s", map_line); | |||
lua_pushnil (L); | |||
return 1; | |||
@@ -369,7 +369,7 @@ lua_config_add_kv_map (lua_State *L) | |||
rspamd_kv_list_fin, | |||
rspamd_kv_list_dtor, | |||
(void **)&map->data.hash, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
msg_warn_config ("invalid hash map %s", map_line); | |||
lua_pushnil (L); | |||
@@ -535,7 +535,7 @@ lua_config_add_map (lua_State *L) | |||
lua_map_fin, | |||
lua_map_dtor, | |||
(void **)&map->data.cbdata, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
if (cbidx != -1) { | |||
luaL_unref (L, LUA_REGISTRYINDEX, cbidx); | |||
@@ -561,7 +561,7 @@ lua_config_add_map (lua_State *L) | |||
rspamd_kv_list_fin, | |||
rspamd_kv_list_dtor, | |||
(void **)&map->data.hash, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
lua_pushnil (L); | |||
ucl_object_unref (map_obj); | |||
@@ -579,7 +579,7 @@ lua_config_add_map (lua_State *L) | |||
rspamd_kv_list_fin, | |||
rspamd_kv_list_dtor, | |||
(void **)&map->data.hash, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
lua_pushnil (L); | |||
ucl_object_unref (map_obj); | |||
@@ -597,7 +597,7 @@ lua_config_add_map (lua_State *L) | |||
rspamd_radix_fin, | |||
rspamd_radix_dtor, | |||
(void **)&map->data.radix, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
lua_pushnil (L); | |||
ucl_object_unref (map_obj); | |||
@@ -615,7 +615,7 @@ lua_config_add_map (lua_State *L) | |||
rspamd_regexp_list_fin, | |||
rspamd_regexp_list_dtor, | |||
(void **) &map->data.re_map, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
lua_pushnil (L); | |||
ucl_object_unref (map_obj); | |||
@@ -633,7 +633,7 @@ lua_config_add_map (lua_State *L) | |||
rspamd_regexp_list_fin, | |||
rspamd_regexp_list_dtor, | |||
(void **) &map->data.re_map, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
lua_pushnil (L); | |||
ucl_object_unref (map_obj); | |||
@@ -651,7 +651,7 @@ lua_config_add_map (lua_State *L) | |||
rspamd_regexp_list_fin, | |||
rspamd_regexp_list_dtor, | |||
(void **) &map->data.re_map, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
lua_pushnil (L); | |||
ucl_object_unref (map_obj); | |||
@@ -669,7 +669,25 @@ lua_config_add_map (lua_State *L) | |||
rspamd_regexp_list_fin, | |||
rspamd_regexp_list_dtor, | |||
(void **) &map->data.re_map, | |||
NULL)) == NULL) { | |||
NULL, RSPAMD_MAP_DEFAULT)) == NULL) { | |||
lua_pushnil (L); | |||
ucl_object_unref (map_obj); | |||
return 1; | |||
} | |||
m->lua_map = map; | |||
} | |||
else if (strcmp (type, "cdb") == 0) { | |||
map = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (*map)); | |||
map->data.cdb_map = NULL; | |||
map->type = RSPAMD_LUA_MAP_CDB; | |||
if ((m = rspamd_map_add_from_ucl (cfg, map_obj, description, | |||
rspamd_cdb_list_read, | |||
rspamd_cdb_list_fin, | |||
rspamd_cdb_list_dtor, | |||
(void **) &map->data.cdb_map, | |||
NULL, RSPAMD_MAP_FILE_ONLY|RSPAMD_MAP_FILE_NO_READ)) == NULL) { | |||
lua_pushnil (L); | |||
ucl_object_unref (map_obj); | |||
@@ -857,7 +875,7 @@ lua_map_get_key (lua_State * L) | |||
key = lua_map_process_string_key (L, 2, &len); | |||
if (key && map->data.hash) { | |||
ret = rspamd_match_hash_map (map->data.hash, key) != NULL; | |||
ret = rspamd_match_hash_map (map->data.hash, key, len) != NULL; | |||
} | |||
} | |||
else if (map->type == RSPAMD_LUA_MAP_REGEXP) { | |||
@@ -903,7 +921,7 @@ lua_map_get_key (lua_State * L) | |||
key = lua_map_process_string_key (L, 2, &len); | |||
if (key && map->data.hash) { | |||
value = rspamd_match_hash_map (map->data.hash, key); | |||
value = rspamd_match_hash_map (map->data.hash, key, len); | |||
} | |||
if (value) { | |||
@@ -911,6 +929,21 @@ lua_map_get_key (lua_State * L) | |||
return 1; | |||
} | |||
} | |||
else if (map->type == RSPAMD_LUA_MAP_CDB) { | |||
/* cdb map */ | |||
const rspamd_ftok_t *tok = NULL; | |||
key = lua_map_process_string_key (L, 2, &len); | |||
if (key && map->data.cdb_map) { | |||
tok = rspamd_match_cdb_map (map->data.cdb_map, key, len); | |||
} | |||
if (tok) { | |||
lua_pushlstring (L, tok->begin, tok->len); | |||
return 1; | |||
} | |||
} | |||
else { | |||
/* callback map or unknown type map */ | |||
lua_pushnil (L); |
@@ -6382,7 +6382,7 @@ lua_lookup_words_array (lua_State *L, | |||
case RSPAMD_LUA_MAP_SET: | |||
case RSPAMD_LUA_MAP_HASH: | |||
/* We know that tok->normalized is zero terminated in fact */ | |||
if (rspamd_match_hash_map (map->data.hash, key)) { | |||
if (rspamd_match_hash_map (map->data.hash, key, keylen)) { | |||
matched = TRUE; | |||
} | |||
break; |
@@ -446,7 +446,7 @@ dkim_module_config (struct rspamd_config *cfg) | |||
rspamd_kv_list_fin, | |||
rspamd_kv_list_dtor, | |||
(void **)&dkim_module_ctx->dkim_domains, | |||
NULL)) { | |||
NULL, RSPAMD_MAP_DEFAULT)) { | |||
msg_warn_config ("cannot load dkim domains list from %s", | |||
ucl_object_tostring (value)); | |||
} | |||
@@ -463,7 +463,7 @@ dkim_module_config (struct rspamd_config *cfg) | |||
rspamd_kv_list_fin, | |||
rspamd_kv_list_dtor, | |||
(void **)&dkim_module_ctx->dkim_domains, | |||
NULL)) { | |||
NULL, RSPAMD_MAP_DEFAULT)) { | |||
msg_warn_config ("cannot load dkim domains list from %s", | |||
ucl_object_tostring (value)); | |||
} | |||
@@ -950,9 +950,12 @@ dkim_module_check (struct dkim_check_result *res) | |||
if (dkim_module_ctx->dkim_domains != NULL) { | |||
/* Perform strict check */ | |||
const gchar *domain = rspamd_dkim_get_domain (cur->ctx); | |||
if ((strict_value = | |||
rspamd_match_hash_map (dkim_module_ctx->dkim_domains, | |||
rspamd_dkim_get_domain (cur->ctx))) != NULL) { | |||
domain, | |||
strlen (domain))) != NULL) { | |||
if (!dkim_module_parse_strict (strict_value, &cur->mult_allow, | |||
&cur->mult_deny)) { | |||
cur->mult_allow = dkim_module_ctx->strict_multiplier; | |||
@@ -1212,11 +1215,12 @@ dkim_symbol_callback (struct rspamd_task *task, | |||
else { | |||
/* Get key */ | |||
cur->ctx = ctx; | |||
const gchar *domain = rspamd_dkim_get_domain (cur->ctx); | |||
if (dkim_module_ctx->trusted_only && | |||
(dkim_module_ctx->dkim_domains == NULL || | |||
rspamd_match_hash_map (dkim_module_ctx->dkim_domains, | |||
rspamd_dkim_get_domain (ctx)) == NULL)) { | |||
domain, strlen (domain)) == NULL)) { | |||
msg_debug_task ("skip dkim check for %s domain", | |||
rspamd_dkim_get_domain (ctx)); | |||
@@ -361,7 +361,7 @@ fuzzy_parse_rule (struct rspamd_config *cfg, const ucl_object_t *obj, | |||
rspamd_kv_list_fin, | |||
rspamd_kv_list_dtor, | |||
(void **)&rule->skip_map, | |||
NULL); | |||
NULL, RSPAMD_MAP_DEFAULT); | |||
} | |||
if ((value = ucl_object_lookup (obj, "headers")) != NULL) { | |||
@@ -1987,7 +1987,8 @@ fuzzy_insert_result (struct fuzzy_client_session *session, | |||
rspamd_encode_hex_buf (cmd->digest, sizeof (cmd->digest), | |||
hexbuf, sizeof (hexbuf) - 1); | |||
hexbuf[sizeof (hexbuf) - 1] = '\0'; | |||
if (rspamd_match_hash_map (session->rule->skip_map, hexbuf)) { | |||
if (rspamd_match_hash_map (session->rule->skip_map, hexbuf, | |||
sizeof (hexbuf) - 1)) { | |||
return; | |||
} | |||
} |