${CMAKE_CURRENT_SOURCE_DIR}/http.c
${CMAKE_CURRENT_SOURCE_DIR}/logger.c
${CMAKE_CURRENT_SOURCE_DIR}/map.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/map_helpers.c
${CMAKE_CURRENT_SOURCE_DIR}/mem_pool.c
${CMAKE_CURRENT_SOURCE_DIR}/printf.c
${CMAKE_CURRENT_SOURCE_DIR}/radix.c
#include "rspamd.h"
#include "contrib/zstd/zstd.h"
-#ifdef WITH_HYPERSCAN
-#include "hs.h"
-#endif
-#ifndef WITH_PCRE2
-#include <pcre.h>
-#else
-#include <pcre2.h>
-#endif
-
#undef MAP_DEBUG_REFS
#ifdef MAP_DEBUG_REFS
#define MAP_RETAIN(x, t) do { \
#define MAP_RELEASE(x, t) REF_RELEASE(x)
#endif
-static const gchar *hash_fill = "1";
static void free_http_cbdata_common (struct http_callback_data *cbd, gboolean plan_new);
static void free_http_cbdata_dtor (gpointer p);
static void free_http_cbdata (struct http_callback_data *cbd);
return NULL;
}
-
-/**
- * FSM for parsing lists
- */
-
-#define MAP_STORE_KEY do { \
- while (g_ascii_isspace (*c) && p > c) { c ++; } \
- key = g_malloc (p - c + 1); \
- rspamd_strlcpy (key, c, p - c + 1); \
- key = g_strchomp (key); \
-} while (0)
-
-#define MAP_STORE_VALUE do { \
- while (g_ascii_isspace (*c) && p > c) { c ++; } \
- value = g_malloc (p - c + 1); \
- rspamd_strlcpy (value, c, p - c + 1); \
- value = g_strchomp (value); \
-} while (0)
-
-gchar *
-rspamd_parse_kv_list (
- gchar * chunk,
- gint len,
- struct map_cb_data *data,
- insert_func func,
- const gchar *default_value,
- gboolean final)
-{
- enum {
- map_skip_spaces_before_key = 0,
- map_read_key,
- map_read_key_quoted,
- map_read_key_slashed,
- map_skip_spaces_after_key,
- map_backslash_quoted,
- map_backslash_slashed,
- map_read_key_after_slash,
- map_read_value,
- map_read_comment_start,
- map_skip_comment,
- map_read_eol,
- };
-
- gchar *c, *p, *key = NULL, *value = NULL, *end;
- struct rspamd_map *map = data->map;
-
- p = chunk;
- c = p;
- end = p + len;
-
- while (p < end) {
- switch (data->state) {
- case map_skip_spaces_before_key:
- if (g_ascii_isspace (*p)) {
- p ++;
- }
- else {
- if (*p == '"') {
- p++;
- c = p;
- data->state = map_read_key_quoted;
- }
- else if (*p == '/') {
- /* Note that c is on '/' here as '/' is a part of key */
- c = p;
- p++;
- data->state = map_read_key_slashed;
- }
- else {
- c = p;
- data->state = map_read_key;
- }
- }
- break;
- case map_read_key:
- /* read key */
- /* Check here comments, eol and end of buffer */
- if (*p == '#') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func (data->cur_data, key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s",
- key, default_value);
- g_free (key);
- }
-
- key = NULL;
- data->state = map_read_comment_start;
- }
- else if (*p == '\r' || *p == '\n') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func (data->cur_data, key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s",
- key, default_value);
- g_free (key);
- }
-
- data->state = map_read_eol;
- key = NULL;
- }
- else if (g_ascii_isspace (*p)) {
- if (p - c > 0) {
- MAP_STORE_KEY;
- data->state = map_skip_spaces_after_key;
- }
- else {
- /* Should not happen */
- g_assert_not_reached ();
- }
- }
- else {
- p++;
- }
- break;
- case map_read_key_quoted:
- if (*p == '\\') {
- data->state = map_backslash_quoted;
- p ++;
- }
- else if (*p == '"') {
- /* Allow empty keys in this case */
- if (p - c >= 0) {
- MAP_STORE_KEY;
- data->state = map_skip_spaces_after_key;
- }
- else {
- g_assert_not_reached ();
- }
- p ++;
- }
- else {
- p ++;
- }
- break;
- case map_read_key_slashed:
- if (*p == '\\') {
- data->state = map_backslash_slashed;
- p ++;
- }
- else if (*p == '/') {
- /* Allow empty keys in this case */
- if (p - c >= 0) {
- data->state = map_read_key_after_slash;
- }
- else {
- g_assert_not_reached ();
- }
- }
- else {
- p ++;
- }
- break;
- case map_read_key_after_slash:
- /*
- * This state is equal to reading of key but '/' is not
- * treated specially
- */
- if (*p == '#') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func (data->cur_data, key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s",
- key, default_value);
- g_free (key);
- key = NULL;
- }
-
- data->state = map_read_comment_start;
- }
- else if (*p == '\r' || *p == '\n') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func (data->cur_data, key, default_value);
-
- msg_debug_map ("insert key only pair: %s -> %s",
- key, default_value);
- g_free (key);
- key = NULL;
- }
-
- data->state = map_read_eol;
- key = NULL;
- }
- else if (g_ascii_isspace (*p)) {
- if (p - c > 0) {
- MAP_STORE_KEY;
- data->state = map_skip_spaces_after_key;
- }
- else {
- /* Should not happen */
- g_assert_not_reached ();
- }
- }
- else {
- p ++;
- }
- break;
- case map_backslash_quoted:
- p ++;
- data->state = map_read_key_quoted;
- break;
- case map_backslash_slashed:
- p ++;
- data->state = map_read_key_slashed;
- break;
- case map_skip_spaces_after_key:
- if (*p == ' ' || *p == '\t') {
- p ++;
- }
- else {
- c = p;
- data->state = map_read_value;
- }
- break;
- case map_read_value:
- g_assert (key != NULL);
- if (*p == '#') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_VALUE;
- func (data->cur_data, key, value);
- msg_debug_map ("insert key value pair: %s -> %s",
- key, value);
- g_free (key);
- g_free (value);
- key = NULL;
- value = NULL;
- }
- else {
- func (data->cur_data, key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s",
- key, default_value);
- g_free (key);
- key = NULL;
- }
-
- data->state = map_read_comment_start;
- }
- else if (*p == '\r' || *p == '\n') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_VALUE;
- func (data->cur_data, key, value);
- msg_debug_map ("insert key value pair: %s -> %s",
- key, value);
- g_free (key);
- g_free (value);
- key = NULL;
- value = NULL;
- }
- else {
- func (data->cur_data, key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s",
- key, default_value);
- g_free (key);
- key = NULL;
- }
-
- data->state = map_read_eol;
- key = NULL;
- }
- else {
- p ++;
- }
- break;
- case map_read_comment_start:
- if (*p == '#') {
- data->state = map_skip_comment;
- p ++;
- key = NULL;
- value = NULL;
- }
- else {
- g_assert_not_reached ();
- }
- break;
- case map_skip_comment:
- if (*p == '\r' || *p == '\n') {
- data->state = map_read_eol;
- }
- else {
- p ++;
- }
- break;
- case map_read_eol:
- /* Skip \r\n and whitespaces */
- if (*p == '\r' || *p == '\n') {
- p++;
- }
- else {
- data->state = map_skip_spaces_before_key;
- }
- break;
- default:
- g_assert_not_reached ();
- break;
- }
- }
-
- if (final) {
- /* Examine the state */
- switch (data->state) {
- case map_read_key:
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func (data->cur_data, key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s",
- key, default_value);
- g_free (key);
- key = NULL;
- }
- break;
- case map_read_value:
- g_assert (key != NULL);
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_VALUE;
- func (data->cur_data, key, value);
- msg_debug_map ("insert key value pair: %s -> %s",
- key, value);
- g_free (key);
- g_free (value);
- key = NULL;
- value = NULL;
- }
- else {
- func (data->cur_data, key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s",
- key, default_value);
- g_free (key);
- key = NULL;
- }
- break;
- }
- }
-
- return c;
-}
-
-/**
- * Radix tree helper function
- */
-static void
-radix_tree_insert_helper (gpointer st, gconstpointer key, gconstpointer value)
-{
- radix_compressed_t *tree = (radix_compressed_t *)st;
- rspamd_mempool_t *pool;
- gpointer nvalue;
-
- pool = radix_get_pool (tree);
- nvalue = rspamd_mempool_strdup (pool, value);
- rspamd_radix_add_iplist (key, ",", tree, nvalue, FALSE);
-}
-
-static void
-hash_insert_helper (gpointer st, gconstpointer key, gconstpointer value)
-{
- GHashTable *ht = st;
- gpointer k, v;
-
- k = g_strdup (key);
- v = g_strdup (value);
- g_hash_table_replace (ht, k, v);
-}
-
-/* Helpers */
-gchar *
-rspamd_hosts_read (
- gchar * chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
-{
- if (data->cur_data == NULL) {
- data->cur_data = g_hash_table_new_full (rspamd_strcase_hash,
- rspamd_strcase_equal, g_free, g_free);
- }
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- hash_insert_helper,
- hash_fill,
- final);
-}
-
-void
-rspamd_hosts_fin (struct map_cb_data *data)
-{
- struct rspamd_map *map = data->map;
-
- if (data->prev_data) {
- g_hash_table_unref (data->prev_data);
- }
- if (data->cur_data) {
- msg_info_map ("read hash of %d elements", g_hash_table_size
- (data->cur_data));
- }
-}
-
-gchar *
-rspamd_kv_list_read (
- gchar * chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
-{
- if (data->cur_data == NULL) {
- data->cur_data = g_hash_table_new_full (rspamd_strcase_hash,
- rspamd_strcase_equal, g_free, g_free);
- }
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- hash_insert_helper,
- "",
- final);
-}
-
-void
-rspamd_kv_list_fin (struct map_cb_data *data)
-{
- struct rspamd_map *map = data->map;
-
- if (data->prev_data) {
- g_hash_table_unref (data->prev_data);
- }
- if (data->cur_data) {
- msg_info_map ("read hash of %d elements", g_hash_table_size
- (data->cur_data));
- }
-}
-
-gchar *
-rspamd_radix_read (
- gchar * chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
-{
- radix_compressed_t *tree;
- rspamd_mempool_t *rpool;
- struct rspamd_map *map = data->map;
-
- if (data->cur_data == NULL) {
- tree = radix_create_compressed ();
- rpool = radix_get_pool (tree);
- memcpy (rpool->tag.uid, map->tag, sizeof (rpool->tag.uid));
- data->cur_data = tree;
- }
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- radix_tree_insert_helper,
- hash_fill,
- final);
-}
-
-void
-rspamd_radix_fin (struct map_cb_data *data)
-{
- struct rspamd_map *map = data->map;
-
- if (data->prev_data) {
- radix_destroy_compressed (data->prev_data);
- }
- if (data->cur_data) {
- msg_info_map ("read radix trie of %z elements: %s",
- radix_get_size (data->cur_data), radix_get_info (data->cur_data));
- }
-}
-
-enum rspamd_regexp_map_flags {
- RSPAMD_REGEXP_FLAG_UTF = (1 << 0),
- RSPAMD_REGEXP_FLAG_MULTIPLE = (1 << 1)
-};
-
-struct rspamd_regexp_map {
- struct rspamd_map *map;
- GPtrArray *regexps;
- GPtrArray *values;
- enum rspamd_regexp_map_flags map_flags;
-#ifdef WITH_HYPERSCAN
- hs_database_t *hs_db;
- hs_scratch_t *hs_scratch;
- const gchar **patterns;
- gint *flags;
- gint *ids;
-#endif
-};
-
-static struct rspamd_regexp_map *
-rspamd_regexp_map_create (struct rspamd_map *map,
- enum rspamd_regexp_map_flags flags)
-{
- struct rspamd_regexp_map *re_map;
-
- re_map = g_malloc0 (sizeof (*re_map));
- re_map->values = g_ptr_array_new ();
- re_map->regexps = g_ptr_array_new ();
- re_map->map = map;
- re_map->map_flags = flags;
-
- return re_map;
-}
-
-
-static void
-rspamd_regexp_map_destroy (struct rspamd_regexp_map *re_map)
-{
- rspamd_regexp_t *re;
- guint i;
-
- for (i = 0; i < re_map->regexps->len; i ++) {
- re = g_ptr_array_index (re_map->regexps, i);
- rspamd_regexp_unref (re);
- }
-
- for (i = 0; i < re_map->values->len; i ++) {
- g_free (g_ptr_array_index (re_map->values, i));
- }
-
- g_ptr_array_free (re_map->regexps, TRUE);
- g_ptr_array_free (re_map->values, TRUE);
-
-#ifdef WITH_HYPERSCAN
- if (re_map->hs_scratch) {
- hs_free_scratch (re_map->hs_scratch);
- }
- if (re_map->hs_db) {
- hs_free_database (re_map->hs_db);
- }
- if (re_map->patterns) {
- g_free (re_map->patterns);
- }
- if (re_map->flags) {
- g_free (re_map->flags);
- }
- if (re_map->ids) {
- g_free (re_map->ids);
- }
-#endif
-
- g_free (re_map);
-}
-
-static void
-rspamd_re_map_insert_helper (gpointer st, gconstpointer key, gconstpointer value)
-{
- struct rspamd_regexp_map *re_map = st;
- struct rspamd_map *map;
- rspamd_regexp_t *re;
- GError *err = NULL;
- gint pcre_flags;
-
- map = re_map->map;
- re = rspamd_regexp_new (key, NULL, &err);
-
- if (re == NULL) {
- msg_err_map ("cannot parse regexp %s: %e", key, err);
-
- if (err) {
- g_error_free (err);
- }
-
- return;
- }
-
- pcre_flags = rspamd_regexp_get_pcre_flags (re);
-
-#ifndef WITH_PCRE2
- if (pcre_flags & PCRE_FLAG(UTF8)) {
- re_map->map_flags |= RSPAMD_REGEXP_FLAG_UTF;
- }
-#else
- if (pcre_flags & PCRE_FLAG(UTF)) {
- re_map->map_flags |= RSPAMD_REGEXP_FLAG_UTF;
- }
-#endif
-
- g_ptr_array_add (re_map->regexps, re);
- g_ptr_array_add (re_map->values, g_strdup (value));
-}
-
-static void
-rspamd_glob_map_insert_helper (gpointer st, gconstpointer key, gconstpointer value)
-{
- struct rspamd_regexp_map *re_map = st;
- struct rspamd_map *map;
- rspamd_regexp_t *re;
- gchar *escaped;
- GError *err = NULL;
- gint pcre_flags;
- gsize escaped_len;
-
- map = re_map->map;
- escaped = rspamd_str_regexp_escape (key, strlen (key), &escaped_len, TRUE);
- re = rspamd_regexp_new (escaped, NULL, &err);
- g_free (escaped);
-
- if (re == NULL) {
- msg_err_map ("cannot parse regexp %s: %e", key, err);
-
- if (err) {
- g_error_free (err);
- }
-
- return;
- }
-
- pcre_flags = rspamd_regexp_get_pcre_flags (re);
-
-#ifndef WITH_PCRE2
- if (pcre_flags & PCRE_FLAG(UTF8)) {
- re_map->map_flags |= RSPAMD_REGEXP_FLAG_UTF;
- }
-#else
- if (pcre_flags & PCRE_FLAG(UTF)) {
- re_map->map_flags |= RSPAMD_REGEXP_FLAG_UTF;
- }
-#endif
-
- g_ptr_array_add (re_map->regexps, re);
- g_ptr_array_add (re_map->values, g_strdup (value));
-}
-
-static void
-rspamd_re_map_finalize (struct rspamd_regexp_map *re_map)
-{
-#ifdef WITH_HYPERSCAN
- guint i;
- hs_platform_info_t plt;
- hs_compile_error_t *err;
- struct rspamd_map *map;
- rspamd_regexp_t *re;
- gint pcre_flags;
-
- map = re_map->map;
-
- if (!(map->cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) {
- msg_info_map ("disable hyperscan for map %s, ssse3 instructons are not supported by CPU",
- map->name);
- return;
- }
-
- if (hs_populate_platform (&plt) != HS_SUCCESS) {
- msg_err_map ("cannot populate hyperscan platform");
- return;
- }
-
- re_map->patterns = g_new (const gchar *, re_map->regexps->len);
- re_map->flags = g_new (gint, re_map->regexps->len);
- re_map->ids = g_new (gint, re_map->regexps->len);
-
- for (i = 0; i < re_map->regexps->len; i ++) {
- re = g_ptr_array_index (re_map->regexps, i);
- re_map->patterns[i] = rspamd_regexp_get_pattern (re);
- re_map->flags[i] = HS_FLAG_SINGLEMATCH;
- pcre_flags = rspamd_regexp_get_pcre_flags (re);
-
-#ifndef WITH_PCRE2
- if (pcre_flags & PCRE_FLAG(UTF8)) {
- re_map->flags[i] |= HS_FLAG_UTF8;
- }
-#else
- if (pcre_flags & PCRE_FLAG(UTF)) {
- re_map->flags[i] |= HS_FLAG_UTF8;
- }
-#endif
- if (pcre_flags & PCRE_FLAG(CASELESS)) {
- re_map->flags[i] |= HS_FLAG_CASELESS;
- }
- if (pcre_flags & PCRE_FLAG(MULTILINE)) {
- re_map->flags[i] |= HS_FLAG_MULTILINE;
- }
- if (pcre_flags & PCRE_FLAG(DOTALL)) {
- re_map->flags[i] |= HS_FLAG_DOTALL;
- }
- if (rspamd_regexp_get_maxhits (re) == 1) {
- re_map->flags[i] |= HS_FLAG_SINGLEMATCH;
- }
-
- re_map->ids[i] = i;
- }
-
- if (re_map->regexps->len > 0 && re_map->patterns) {
- if (hs_compile_multi (re_map->patterns,
- re_map->flags,
- re_map->ids,
- re_map->regexps->len,
- HS_MODE_BLOCK,
- &plt,
- &re_map->hs_db,
- &err) != HS_SUCCESS) {
-
- msg_err_map ("cannot create tree of regexp when processing '%s': %s",
- err->expression >= 0 ?
- re_map->patterns[err->expression] :
- "unknown regexp", err->message);
- re_map->hs_db = NULL;
- hs_free_compile_error (err);
-
- return;
- }
-
- if (hs_alloc_scratch (re_map->hs_db, &re_map->hs_scratch) != HS_SUCCESS) {
- msg_err_map ("cannot allocate scratch space for hyperscan");
- hs_free_database (re_map->hs_db);
- re_map->hs_db = NULL;
- }
- }
- else {
- msg_err_map ("regexp map is empty");
- }
-#endif
-}
-
-gchar *
-rspamd_regexp_list_read_single (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
-{
- struct rspamd_regexp_map *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_regexp_map_create (data->map, 0);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- rspamd_re_map_insert_helper,
- hash_fill,
- final);
-}
-
-gchar *
-rspamd_glob_list_read_single (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
-{
- struct rspamd_regexp_map *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_regexp_map_create (data->map, 0);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- rspamd_glob_map_insert_helper,
- hash_fill,
- final);
-}
-
-gchar *
-rspamd_regexp_list_read_multiple (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
-{
- struct rspamd_regexp_map *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_regexp_map_create (data->map, RSPAMD_REGEXP_FLAG_MULTIPLE);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- rspamd_re_map_insert_helper,
- hash_fill,
- final);
-}
-
-void
-rspamd_regexp_list_fin (struct map_cb_data *data)
-{
- struct rspamd_regexp_map *re_map;
- struct rspamd_map *map = data->map;
-
- if (data->prev_data) {
- rspamd_regexp_map_destroy (data->prev_data);
- }
- if (data->cur_data) {
- re_map = data->cur_data;
- rspamd_re_map_finalize (re_map);
- msg_info_map ("read regexp list of %ud elements",
- re_map->regexps->len);
- }
-}
-
-#ifdef WITH_HYPERSCAN
-static int
-rspamd_match_hs_single_handler (unsigned int id, unsigned long long from,
- unsigned long long to,
- unsigned int flags, void *context)
-{
- guint *i = context;
- /* Always return non-zero as we need a single match here */
-
- *i = id;
-
- return 1;
-}
-#endif
-
-gpointer
-rspamd_match_regexp_map_single (struct rspamd_regexp_map *map,
- const gchar *in, gsize len)
-{
- guint i;
- rspamd_regexp_t *re;
- gint res = 0;
- gpointer ret = NULL;
- gboolean validated = FALSE;
-
- g_assert (in != NULL);
-
- if (map == NULL || len == 0) {
- return NULL;
- }
-
- if (map->map_flags & RSPAMD_REGEXP_FLAG_UTF) {
- if (g_utf8_validate (in, len, NULL)) {
- validated = TRUE;
- }
- }
- else {
- validated = TRUE;
- }
-
-#ifdef WITH_HYPERSCAN
- if (map->hs_db && map->hs_scratch) {
-
- if (validated) {
-
- res = hs_scan (map->hs_db, in, len, 0, map->hs_scratch,
- rspamd_match_hs_single_handler, (void *)&i);
-
- if (res == HS_SCAN_TERMINATED) {
- res = 1;
- ret = g_ptr_array_index (map->values, i);
- }
-
- return ret;
- }
- }
-#endif
-
- if (!res) {
- /* PCRE version */
- for (i = 0; i < map->regexps->len; i ++) {
- re = g_ptr_array_index (map->regexps, i);
-
- if (rspamd_regexp_search (re, in, len, NULL, NULL, !validated, NULL)) {
- ret = g_ptr_array_index (map->values, i);
- break;
- }
- }
- }
-
- return ret;
-}
-
-#ifdef WITH_HYPERSCAN
-struct rspamd_multiple_cbdata {
- GPtrArray *ar;
- struct rspamd_regexp_map *map;
-};
-
-static int
-rspamd_match_hs_multiple_handler (unsigned int id, unsigned long long from,
- unsigned long long to,
- unsigned int flags, void *context)
-{
- struct rspamd_multiple_cbdata *cbd = context;
-
- if (id < cbd->map->values->len) {
- g_ptr_array_add (cbd->ar, g_ptr_array_index (cbd->map->values, id));
- }
-
- /* Always return zero as we need all matches here */
- return 0;
-}
-#endif
-
-gpointer
-rspamd_match_regexp_map_all (struct rspamd_regexp_map *map,
- const gchar *in, gsize len)
-{
- guint i;
- rspamd_regexp_t *re;
- GPtrArray *ret;
- gint res = 0;
- gboolean validated = FALSE;
-
- g_assert (in != NULL);
-
- if (map == NULL || len == 0) {
- return NULL;
- }
-
- if (map->map_flags & RSPAMD_REGEXP_FLAG_UTF) {
- if (g_utf8_validate (in, len, NULL)) {
- validated = TRUE;
- }
- }
- else {
- validated = TRUE;
- }
-
- ret = g_ptr_array_new ();
-
-#ifdef WITH_HYPERSCAN
- if (map->hs_db && map->hs_scratch) {
-
- if (validated) {
- struct rspamd_multiple_cbdata cbd;
-
- cbd.ar = ret;
- cbd.map = map;
-
- if (hs_scan (map->hs_db, in, len, 0, map->hs_scratch,
- rspamd_match_hs_multiple_handler, &cbd) == HS_SUCCESS) {
- res = 1;
- }
- }
- }
-#endif
-
- if (!res) {
- /* PCRE version */
- for (i = 0; i < map->regexps->len; i ++) {
- re = g_ptr_array_index (map->regexps, i);
-
- if (rspamd_regexp_search (re, in, len, NULL, NULL,
- !validated, NULL)) {
- g_ptr_array_add (ret, g_ptr_array_index (map->values, i));
- }
- }
- }
-
- if (ret->len > 0) {
- return ret;
- }
-
- g_ptr_array_free (ret, TRUE);
-
- return NULL;
-}
*/
void rspamd_map_remove_all (struct rspamd_config *cfg);
-typedef void (*insert_func) (gpointer st, gconstpointer key,
- gconstpointer value);
-
-/**
- * Common callbacks for frequent types of lists
- */
-
-/**
- * Radix list is a list like ip/mask
- */
-gchar * rspamd_radix_read (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final);
-void rspamd_radix_fin (struct map_cb_data *data);
-
-/**
- * Host list is an ordinal list of hosts or domains
- */
-gchar * rspamd_hosts_read (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final);
-void rspamd_hosts_fin (struct map_cb_data *data);
-
-/**
- * Kv list is an ordinal list of keys and values separated by whitespace
- */
-gchar * rspamd_kv_list_read (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final);
-void rspamd_kv_list_fin (struct map_cb_data *data);
-
-/**
- * Regexp list is a list of regular expressions
- */
-struct rspamd_regexp_map;
-
-gchar * rspamd_regexp_list_read_single (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final);
-gchar * rspamd_regexp_list_read_multiple (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final);
-gchar * rspamd_glob_list_read_single (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final);
-void rspamd_regexp_list_fin (struct map_cb_data *data);
-
-/**
- * FSM for lists parsing (support comments, blank lines and partial replies)
- */
-gchar *
-rspamd_parse_kv_list (
- gchar * chunk,
- gint len,
- struct map_cb_data *data,
- insert_func func,
- const gchar *default_value,
- gboolean final);
-
-/**
- * Find a single (any) matching regexp for the specified text or NULL if
- * no matches found
- * @param map
- * @param in
- * @param len
- * @return
- */
-gpointer rspamd_match_regexp_map_single (struct rspamd_regexp_map *map,
- const gchar *in, gsize len);
-
-/**
- * Find a multiple (all) matching regexp for the specified text or NULL if
- * no matches found. Returns GPtrArray that *must* be freed by a caller if not NULL
- * @param map
- * @param in
- * @param len
- * @return
- */
-gpointer rspamd_match_regexp_map_all (struct rspamd_regexp_map *map,
- const gchar *in, gsize len);
-
#endif
--- /dev/null
+/*-
+ * Copyright 2018 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "map_helpers.h"
+#include "map_private.h"
+#include "khash.h"
+#include "radix.h"
+
+#ifdef WITH_HYPERSCAN
+#include "hs.h"
+#endif
+#ifndef WITH_PCRE2
+#include <pcre.h>
+#else
+#include <pcre2.h>
+#endif
+
+static const gchar *hash_fill = "1";
+
+struct rspamd_map_helper_value {
+ gsize hits;
+ gchar value[]; /* Null terminated */
+};
+
+struct rspamd_radix_map_helper {
+ rspamd_mempool_t *pool;
+ radix_compressed_t *trie;
+};
+
+KHASH_INIT (rspamd_map_hash, const gchar *,
+ struct rspamd_map_helper_value *, true,
+ rspamd_strcase_hash, rspamd_strcase_equal);
+
+struct rspamd_hash_map_helper {
+ rspamd_mempool_t *pool;
+ khash_t(rspamd_map_hash) *htb;
+};
+
+enum rspamd_regexp_map_flags {
+ RSPAMD_REGEXP_FLAG_UTF = (1u << 0),
+ RSPAMD_REGEXP_FLAG_MULTIPLE = (1u << 1)
+};
+
+struct rspamd_regexp_map_helper {
+ rspamd_mempool_t *pool;
+ struct rspamd_map *map;
+ GPtrArray *regexps;
+ GPtrArray *values;
+ enum rspamd_regexp_map_flags map_flags;
+#ifdef WITH_HYPERSCAN
+ hs_database_t *hs_db;
+ hs_scratch_t *hs_scratch;
+ const gchar **patterns;
+ gint *flags;
+ gint *ids;
+#endif
+};
+
+/**
+ * FSM for parsing lists
+ */
+
+#define MAP_STORE_KEY do { \
+ while (g_ascii_isspace (*c) && p > c) { c ++; } \
+ key = g_malloc (p - c + 1); \
+ rspamd_strlcpy (key, c, p - c + 1); \
+ key = g_strchomp (key); \
+} while (0)
+
+#define MAP_STORE_VALUE do { \
+ while (g_ascii_isspace (*c) && p > c) { c ++; } \
+ value = g_malloc (p - c + 1); \
+ rspamd_strlcpy (value, c, p - c + 1); \
+ value = g_strchomp (value); \
+} while (0)
+
+gchar *
+rspamd_parse_kv_list (
+ gchar * chunk,
+ gint len,
+ struct map_cb_data *data,
+ insert_func func,
+ const gchar *default_value,
+ gboolean final)
+{
+ enum {
+ map_skip_spaces_before_key = 0,
+ map_read_key,
+ map_read_key_quoted,
+ map_read_key_slashed,
+ map_skip_spaces_after_key,
+ map_backslash_quoted,
+ map_backslash_slashed,
+ map_read_key_after_slash,
+ map_read_value,
+ map_read_comment_start,
+ map_skip_comment,
+ map_read_eol,
+ };
+
+ gchar *c, *p, *key = NULL, *value = NULL, *end;
+ struct rspamd_map *map = data->map;
+
+ p = chunk;
+ c = p;
+ end = p + len;
+
+ while (p < end) {
+ switch (data->state) {
+ case map_skip_spaces_before_key:
+ if (g_ascii_isspace (*p)) {
+ p ++;
+ }
+ else {
+ if (*p == '"') {
+ p++;
+ c = p;
+ data->state = map_read_key_quoted;
+ }
+ else if (*p == '/') {
+ /* Note that c is on '/' here as '/' is a part of key */
+ c = p;
+ p++;
+ data->state = map_read_key_slashed;
+ }
+ else {
+ c = p;
+ data->state = map_read_key;
+ }
+ }
+ break;
+ case map_read_key:
+ /* read key */
+ /* Check here comments, eol and end of buffer */
+ if (*p == '#') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_KEY;
+ func (data->cur_data, key, default_value);
+ msg_debug_map ("insert key only pair: %s -> %s",
+ key, default_value);
+ g_free (key);
+ }
+
+ key = NULL;
+ data->state = map_read_comment_start;
+ }
+ else if (*p == '\r' || *p == '\n') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_KEY;
+ func (data->cur_data, key, default_value);
+ msg_debug_map ("insert key only pair: %s -> %s",
+ key, default_value);
+ g_free (key);
+ }
+
+ data->state = map_read_eol;
+ key = NULL;
+ }
+ else if (g_ascii_isspace (*p)) {
+ if (p - c > 0) {
+ MAP_STORE_KEY;
+ data->state = map_skip_spaces_after_key;
+ }
+ else {
+ /* Should not happen */
+ g_assert_not_reached ();
+ }
+ }
+ else {
+ p++;
+ }
+ break;
+ case map_read_key_quoted:
+ if (*p == '\\') {
+ data->state = map_backslash_quoted;
+ p ++;
+ }
+ else if (*p == '"') {
+ /* Allow empty keys in this case */
+ if (p - c >= 0) {
+ MAP_STORE_KEY;
+ data->state = map_skip_spaces_after_key;
+ }
+ else {
+ g_assert_not_reached ();
+ }
+ p ++;
+ }
+ else {
+ p ++;
+ }
+ break;
+ case map_read_key_slashed:
+ if (*p == '\\') {
+ data->state = map_backslash_slashed;
+ p ++;
+ }
+ else if (*p == '/') {
+ /* Allow empty keys in this case */
+ if (p - c >= 0) {
+ data->state = map_read_key_after_slash;
+ }
+ else {
+ g_assert_not_reached ();
+ }
+ }
+ else {
+ p ++;
+ }
+ break;
+ case map_read_key_after_slash:
+ /*
+ * This state is equal to reading of key but '/' is not
+ * treated specially
+ */
+ if (*p == '#') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_KEY;
+ func (data->cur_data, key, default_value);
+ msg_debug_map ("insert key only pair: %s -> %s",
+ key, default_value);
+ g_free (key);
+ key = NULL;
+ }
+
+ data->state = map_read_comment_start;
+ }
+ else if (*p == '\r' || *p == '\n') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_KEY;
+ func (data->cur_data, key, default_value);
+
+ msg_debug_map ("insert key only pair: %s -> %s",
+ key, default_value);
+ g_free (key);
+ key = NULL;
+ }
+
+ data->state = map_read_eol;
+ key = NULL;
+ }
+ else if (g_ascii_isspace (*p)) {
+ if (p - c > 0) {
+ MAP_STORE_KEY;
+ data->state = map_skip_spaces_after_key;
+ }
+ else {
+ /* Should not happen */
+ g_assert_not_reached ();
+ }
+ }
+ else {
+ p ++;
+ }
+ break;
+ case map_backslash_quoted:
+ p ++;
+ data->state = map_read_key_quoted;
+ break;
+ case map_backslash_slashed:
+ p ++;
+ data->state = map_read_key_slashed;
+ break;
+ case map_skip_spaces_after_key:
+ if (*p == ' ' || *p == '\t') {
+ p ++;
+ }
+ else {
+ c = p;
+ data->state = map_read_value;
+ }
+ break;
+ case map_read_value:
+ g_assert (key != NULL);
+ if (*p == '#') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_VALUE;
+ func (data->cur_data, key, value);
+ msg_debug_map ("insert key value pair: %s -> %s",
+ key, value);
+ g_free (key);
+ g_free (value);
+ key = NULL;
+ value = NULL;
+ }
+ else {
+ func (data->cur_data, key, default_value);
+ msg_debug_map ("insert key only pair: %s -> %s",
+ key, default_value);
+ g_free (key);
+ key = NULL;
+ }
+
+ data->state = map_read_comment_start;
+ }
+ else if (*p == '\r' || *p == '\n') {
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_VALUE;
+ func (data->cur_data, key, value);
+ msg_debug_map ("insert key value pair: %s -> %s",
+ key, value);
+ g_free (key);
+ g_free (value);
+ key = NULL;
+ value = NULL;
+ }
+ else {
+ func (data->cur_data, key, default_value);
+ msg_debug_map ("insert key only pair: %s -> %s",
+ key, default_value);
+ g_free (key);
+ key = NULL;
+ }
+
+ data->state = map_read_eol;
+ key = NULL;
+ }
+ else {
+ p ++;
+ }
+ break;
+ case map_read_comment_start:
+ if (*p == '#') {
+ data->state = map_skip_comment;
+ p ++;
+ key = NULL;
+ value = NULL;
+ }
+ else {
+ g_assert_not_reached ();
+ }
+ break;
+ case map_skip_comment:
+ if (*p == '\r' || *p == '\n') {
+ data->state = map_read_eol;
+ }
+ else {
+ p ++;
+ }
+ break;
+ case map_read_eol:
+ /* Skip \r\n and whitespaces */
+ if (*p == '\r' || *p == '\n') {
+ p++;
+ }
+ else {
+ data->state = map_skip_spaces_before_key;
+ }
+ break;
+ default:
+ g_assert_not_reached ();
+ break;
+ }
+ }
+
+ if (final) {
+ /* Examine the state */
+ switch (data->state) {
+ case map_read_key:
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_KEY;
+ func (data->cur_data, key, default_value);
+ msg_debug_map ("insert key only pair: %s -> %s",
+ key, default_value);
+ g_free (key);
+ key = NULL;
+ }
+ break;
+ case map_read_value:
+ g_assert (key != NULL);
+ if (p - c > 0) {
+ /* Store a single key */
+ MAP_STORE_VALUE;
+ func (data->cur_data, key, value);
+ msg_debug_map ("insert key value pair: %s -> %s",
+ key, value);
+ g_free (key);
+ g_free (value);
+ key = NULL;
+ value = NULL;
+ }
+ else {
+ func (data->cur_data, key, default_value);
+ msg_debug_map ("insert key only pair: %s -> %s",
+ key, default_value);
+ g_free (key);
+ key = NULL;
+ }
+ break;
+ }
+ }
+
+ return c;
+}
+
+/**
+ * Radix tree helper function
+ */
+static void
+radix_tree_insert_helper (gpointer st, gconstpointer key, gconstpointer value)
+{
+ radix_compressed_t *tree = (radix_compressed_t *)st;
+ rspamd_mempool_t *pool;
+ gpointer nvalue;
+
+ pool = radix_get_pool (tree);
+ nvalue = rspamd_mempool_strdup (pool, value);
+ rspamd_radix_add_iplist (key, ",", tree, nvalue, FALSE);
+}
+
+static void
+hash_insert_helper (gpointer st, gconstpointer key, gconstpointer value)
+{
+ GHashTable *ht = st;
+ gpointer k, v;
+
+ k = g_strdup (key);
+ v = g_strdup (value);
+ g_hash_table_replace (ht, k, v);
+}
+
+/* Helpers */
+gchar *
+rspamd_hosts_read (
+ gchar * chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ if (data->cur_data == NULL) {
+ data->cur_data = g_hash_table_new_full (rspamd_strcase_hash,
+ rspamd_strcase_equal, g_free, g_free);
+ }
+ return rspamd_parse_kv_list (
+ chunk,
+ len,
+ data,
+ hash_insert_helper,
+ hash_fill,
+ final);
+}
+
+void
+rspamd_hosts_fin (struct map_cb_data *data)
+{
+ struct rspamd_map *map = data->map;
+
+ if (data->prev_data) {
+ g_hash_table_unref (data->prev_data);
+ }
+ if (data->cur_data) {
+ msg_info_map ("read hash of %d elements", g_hash_table_size
+ (data->cur_data));
+ }
+}
+
+gchar *
+rspamd_kv_list_read (
+ gchar * chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ if (data->cur_data == NULL) {
+ data->cur_data = g_hash_table_new_full (rspamd_strcase_hash,
+ rspamd_strcase_equal, g_free, g_free);
+ }
+ return rspamd_parse_kv_list (
+ chunk,
+ len,
+ data,
+ hash_insert_helper,
+ "",
+ final);
+}
+
+void
+rspamd_kv_list_fin (struct map_cb_data *data)
+{
+ struct rspamd_map *map = data->map;
+
+ if (data->prev_data) {
+ g_hash_table_unref (data->prev_data);
+ }
+ if (data->cur_data) {
+ msg_info_map ("read hash of %d elements", g_hash_table_size
+ (data->cur_data));
+ }
+}
+
+gchar *
+rspamd_radix_read (
+ gchar * chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ radix_compressed_t *tree;
+ rspamd_mempool_t *rpool;
+ struct rspamd_map *map = data->map;
+
+ if (data->cur_data == NULL) {
+ tree = radix_create_compressed ();
+ rpool = radix_get_pool (tree);
+ memcpy (rpool->tag.uid, map->tag, sizeof (rpool->tag.uid));
+ data->cur_data = tree;
+ }
+ return rspamd_parse_kv_list (
+ chunk,
+ len,
+ data,
+ radix_tree_insert_helper,
+ hash_fill,
+ final);
+}
+
+void
+rspamd_radix_fin (struct map_cb_data *data)
+{
+ struct rspamd_map *map = data->map;
+
+ if (data->prev_data) {
+ radix_destroy_compressed (data->prev_data);
+ }
+ if (data->cur_data) {
+ msg_info_map ("read radix trie of %z elements: %s",
+ radix_get_size (data->cur_data), radix_get_info (data->cur_data));
+ }
+}
+
+static struct rspamd_regexp_map_helper *
+rspamd_regexp_map_create (struct rspamd_map *map,
+ enum rspamd_regexp_map_flags flags)
+{
+ struct rspamd_regexp_map_helper *re_map;
+
+ re_map = g_malloc0 (sizeof (*re_map));
+ re_map->values = g_ptr_array_new ();
+ re_map->regexps = g_ptr_array_new ();
+ re_map->map = map;
+ re_map->map_flags = flags;
+
+ return re_map;
+}
+
+
+static void
+rspamd_regexp_map_destroy (struct rspamd_regexp_map_helper *re_map)
+{
+ rspamd_regexp_t *re;
+ guint i;
+
+ for (i = 0; i < re_map->regexps->len; i ++) {
+ re = g_ptr_array_index (re_map->regexps, i);
+ rspamd_regexp_unref (re);
+ }
+
+ for (i = 0; i < re_map->values->len; i ++) {
+ g_free (g_ptr_array_index (re_map->values, i));
+ }
+
+ g_ptr_array_free (re_map->regexps, TRUE);
+ g_ptr_array_free (re_map->values, TRUE);
+
+#ifdef WITH_HYPERSCAN
+ if (re_map->hs_scratch) {
+ hs_free_scratch (re_map->hs_scratch);
+ }
+ if (re_map->hs_db) {
+ hs_free_database (re_map->hs_db);
+ }
+ if (re_map->patterns) {
+ g_free (re_map->patterns);
+ }
+ if (re_map->flags) {
+ g_free (re_map->flags);
+ }
+ if (re_map->ids) {
+ g_free (re_map->ids);
+ }
+#endif
+
+ g_free (re_map);
+}
+
+static void
+rspamd_re_map_insert_helper (gpointer st, gconstpointer key, gconstpointer value)
+{
+ struct rspamd_regexp_map_helper *re_map = st;
+ struct rspamd_map *map;
+ rspamd_regexp_t *re;
+ GError *err = NULL;
+ gint pcre_flags;
+
+ map = re_map->map;
+ re = rspamd_regexp_new (key, NULL, &err);
+
+ if (re == NULL) {
+ msg_err_map ("cannot parse regexp %s: %e", key, err);
+
+ if (err) {
+ g_error_free (err);
+ }
+
+ return;
+ }
+
+ pcre_flags = rspamd_regexp_get_pcre_flags (re);
+
+#ifndef WITH_PCRE2
+ if (pcre_flags & PCRE_FLAG(UTF8)) {
+ re_map->map_flags |= RSPAMD_REGEXP_FLAG_UTF;
+ }
+#else
+ if (pcre_flags & PCRE_FLAG(UTF)) {
+ re_map->map_flags |= RSPAMD_REGEXP_FLAG_UTF;
+ }
+#endif
+
+ g_ptr_array_add (re_map->regexps, re);
+ g_ptr_array_add (re_map->values, g_strdup (value));
+}
+
+static void
+rspamd_glob_map_insert_helper (gpointer st, gconstpointer key, gconstpointer value)
+{
+ struct rspamd_regexp_map_helper *re_map = st;
+ struct rspamd_map *map;
+ rspamd_regexp_t *re;
+ gchar *escaped;
+ GError *err = NULL;
+ gint pcre_flags;
+ gsize escaped_len;
+
+ map = re_map->map;
+ escaped = rspamd_str_regexp_escape (key, strlen (key), &escaped_len, TRUE);
+ re = rspamd_regexp_new (escaped, NULL, &err);
+ g_free (escaped);
+
+ if (re == NULL) {
+ msg_err_map ("cannot parse regexp %s: %e", key, err);
+
+ if (err) {
+ g_error_free (err);
+ }
+
+ return;
+ }
+
+ pcre_flags = rspamd_regexp_get_pcre_flags (re);
+
+#ifndef WITH_PCRE2
+ if (pcre_flags & PCRE_FLAG(UTF8)) {
+ re_map->map_flags |= RSPAMD_REGEXP_FLAG_UTF;
+ }
+#else
+ if (pcre_flags & PCRE_FLAG(UTF)) {
+ re_map->map_flags |= RSPAMD_REGEXP_FLAG_UTF;
+ }
+#endif
+
+ g_ptr_array_add (re_map->regexps, re);
+ g_ptr_array_add (re_map->values, g_strdup (value));
+}
+
+static void
+rspamd_re_map_finalize (struct rspamd_regexp_map_helper *re_map)
+{
+#ifdef WITH_HYPERSCAN
+ guint i;
+ hs_platform_info_t plt;
+ hs_compile_error_t *err;
+ struct rspamd_map *map;
+ rspamd_regexp_t *re;
+ gint pcre_flags;
+
+ map = re_map->map;
+
+ if (!(map->cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) {
+ msg_info_map ("disable hyperscan for map %s, ssse3 instructons are not supported by CPU",
+ map->name);
+ return;
+ }
+
+ if (hs_populate_platform (&plt) != HS_SUCCESS) {
+ msg_err_map ("cannot populate hyperscan platform");
+ return;
+ }
+
+ re_map->patterns = g_new (const gchar *, re_map->regexps->len);
+ re_map->flags = g_new (gint, re_map->regexps->len);
+ re_map->ids = g_new (gint, re_map->regexps->len);
+
+ for (i = 0; i < re_map->regexps->len; i ++) {
+ re = g_ptr_array_index (re_map->regexps, i);
+ re_map->patterns[i] = rspamd_regexp_get_pattern (re);
+ re_map->flags[i] = HS_FLAG_SINGLEMATCH;
+ pcre_flags = rspamd_regexp_get_pcre_flags (re);
+
+#ifndef WITH_PCRE2
+ if (pcre_flags & PCRE_FLAG(UTF8)) {
+ re_map->flags[i] |= HS_FLAG_UTF8;
+ }
+#else
+ if (pcre_flags & PCRE_FLAG(UTF)) {
+ re_map->flags[i] |= HS_FLAG_UTF8;
+ }
+#endif
+ if (pcre_flags & PCRE_FLAG(CASELESS)) {
+ re_map->flags[i] |= HS_FLAG_CASELESS;
+ }
+ if (pcre_flags & PCRE_FLAG(MULTILINE)) {
+ re_map->flags[i] |= HS_FLAG_MULTILINE;
+ }
+ if (pcre_flags & PCRE_FLAG(DOTALL)) {
+ re_map->flags[i] |= HS_FLAG_DOTALL;
+ }
+ if (rspamd_regexp_get_maxhits (re) == 1) {
+ re_map->flags[i] |= HS_FLAG_SINGLEMATCH;
+ }
+
+ re_map->ids[i] = i;
+ }
+
+ if (re_map->regexps->len > 0 && re_map->patterns) {
+ if (hs_compile_multi (re_map->patterns,
+ re_map->flags,
+ re_map->ids,
+ re_map->regexps->len,
+ HS_MODE_BLOCK,
+ &plt,
+ &re_map->hs_db,
+ &err) != HS_SUCCESS) {
+
+ msg_err_map ("cannot create tree of regexp when processing '%s': %s",
+ err->expression >= 0 ?
+ re_map->patterns[err->expression] :
+ "unknown regexp", err->message);
+ re_map->hs_db = NULL;
+ hs_free_compile_error (err);
+
+ return;
+ }
+
+ if (hs_alloc_scratch (re_map->hs_db, &re_map->hs_scratch) != HS_SUCCESS) {
+ msg_err_map ("cannot allocate scratch space for hyperscan");
+ hs_free_database (re_map->hs_db);
+ re_map->hs_db = NULL;
+ }
+ }
+ else {
+ msg_err_map ("regexp map is empty");
+ }
+#endif
+}
+
+gchar *
+rspamd_regexp_list_read_single (
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct rspamd_regexp_map_helper *re_map;
+
+ if (data->cur_data == NULL) {
+ re_map = rspamd_regexp_map_create (data->map, 0);
+ data->cur_data = re_map;
+ }
+
+ return rspamd_parse_kv_list (
+ chunk,
+ len,
+ data,
+ rspamd_re_map_insert_helper,
+ hash_fill,
+ final);
+}
+
+gchar *
+rspamd_glob_list_read_single (
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct rspamd_regexp_map_helper *re_map;
+
+ if (data->cur_data == NULL) {
+ re_map = rspamd_regexp_map_create (data->map, 0);
+ data->cur_data = re_map;
+ }
+
+ return rspamd_parse_kv_list (
+ chunk,
+ len,
+ data,
+ rspamd_glob_map_insert_helper,
+ hash_fill,
+ final);
+}
+
+gchar *
+rspamd_regexp_list_read_multiple (
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final)
+{
+ struct rspamd_regexp_map_helper *re_map;
+
+ if (data->cur_data == NULL) {
+ re_map = rspamd_regexp_map_create (data->map, RSPAMD_REGEXP_FLAG_MULTIPLE);
+ data->cur_data = re_map;
+ }
+
+ return rspamd_parse_kv_list (
+ chunk,
+ len,
+ data,
+ rspamd_re_map_insert_helper,
+ hash_fill,
+ final);
+}
+
+void
+rspamd_regexp_list_fin (struct map_cb_data *data)
+{
+ struct rspamd_regexp_map_helper *re_map;
+ struct rspamd_map *map = data->map;
+
+ if (data->prev_data) {
+ rspamd_regexp_map_destroy (data->prev_data);
+ }
+ if (data->cur_data) {
+ re_map = data->cur_data;
+ rspamd_re_map_finalize (re_map);
+ msg_info_map ("read regexp list of %ud elements",
+ re_map->regexps->len);
+ }
+}
+
+#ifdef WITH_HYPERSCAN
+static int
+rspamd_match_hs_single_handler (unsigned int id, unsigned long long from,
+ unsigned long long to,
+ unsigned int flags, void *context)
+{
+ guint *i = context;
+ /* Always return non-zero as we need a single match here */
+
+ *i = id;
+
+ return 1;
+}
+#endif
+
+gpointer
+rspamd_match_regexp_map_single (struct rspamd_regexp_map_helper *map,
+ const gchar *in, gsize len)
+{
+ guint i;
+ rspamd_regexp_t *re;
+ gint res = 0;
+ gpointer ret = NULL;
+ gboolean validated = FALSE;
+
+ g_assert (in != NULL);
+
+ if (map == NULL || len == 0) {
+ return NULL;
+ }
+
+ if (map->map_flags & RSPAMD_REGEXP_FLAG_UTF) {
+ if (g_utf8_validate (in, len, NULL)) {
+ validated = TRUE;
+ }
+ }
+ else {
+ validated = TRUE;
+ }
+
+#ifdef WITH_HYPERSCAN
+ if (map->hs_db && map->hs_scratch) {
+
+ if (validated) {
+
+ res = hs_scan (map->hs_db, in, len, 0, map->hs_scratch,
+ rspamd_match_hs_single_handler, (void *)&i);
+
+ if (res == HS_SCAN_TERMINATED) {
+ res = 1;
+ ret = g_ptr_array_index (map->values, i);
+ }
+
+ return ret;
+ }
+ }
+#endif
+
+ if (!res) {
+ /* PCRE version */
+ for (i = 0; i < map->regexps->len; i ++) {
+ re = g_ptr_array_index (map->regexps, i);
+
+ if (rspamd_regexp_search (re, in, len, NULL, NULL, !validated, NULL)) {
+ ret = g_ptr_array_index (map->values, i);
+ break;
+ }
+ }
+ }
+
+ return ret;
+}
+
+#ifdef WITH_HYPERSCAN
+struct rspamd_multiple_cbdata {
+ GPtrArray *ar;
+ struct rspamd_regexp_map_helper *map;
+};
+
+static int
+rspamd_match_hs_multiple_handler (unsigned int id, unsigned long long from,
+ unsigned long long to,
+ unsigned int flags, void *context)
+{
+ struct rspamd_multiple_cbdata *cbd = context;
+
+ if (id < cbd->map->values->len) {
+ g_ptr_array_add (cbd->ar, g_ptr_array_index (cbd->map->values, id));
+ }
+
+ /* Always return zero as we need all matches here */
+ return 0;
+}
+#endif
+
+gpointer
+rspamd_match_regexp_map_all (struct rspamd_regexp_map_helper *map,
+ const gchar *in, gsize len)
+{
+ guint i;
+ rspamd_regexp_t *re;
+ GPtrArray *ret;
+ gint res = 0;
+ gboolean validated = FALSE;
+
+ g_assert (in != NULL);
+
+ if (map == NULL || len == 0) {
+ return NULL;
+ }
+
+ if (map->map_flags & RSPAMD_REGEXP_FLAG_UTF) {
+ if (g_utf8_validate (in, len, NULL)) {
+ validated = TRUE;
+ }
+ }
+ else {
+ validated = TRUE;
+ }
+
+ ret = g_ptr_array_new ();
+
+#ifdef WITH_HYPERSCAN
+ if (map->hs_db && map->hs_scratch) {
+
+ if (validated) {
+ struct rspamd_multiple_cbdata cbd;
+
+ cbd.ar = ret;
+ cbd.map = map;
+
+ if (hs_scan (map->hs_db, in, len, 0, map->hs_scratch,
+ rspamd_match_hs_multiple_handler, &cbd) == HS_SUCCESS) {
+ res = 1;
+ }
+ }
+ }
+#endif
+
+ if (!res) {
+ /* PCRE version */
+ for (i = 0; i < map->regexps->len; i ++) {
+ re = g_ptr_array_index (map->regexps, i);
+
+ if (rspamd_regexp_search (re, in, len, NULL, NULL,
+ !validated, NULL)) {
+ g_ptr_array_add (ret, g_ptr_array_index (map->values, i));
+ }
+ }
+ }
+
+ if (ret->len > 0) {
+ return ret;
+ }
+
+ g_ptr_array_free (ret, TRUE);
+
+ return NULL;
+}
--- /dev/null
+/*-
+ * Copyright 2018 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_MAP_HELPERS_H
+#define RSPAMD_MAP_HELPERS_H
+
+#include "config.h"
+#include "map.h"
+
+/**
+ * @file map_helpers.h
+ *
+ * Defines helper structures to deal with different map types
+ */
+
+/**
+ * Common structures, abstract for simplicity
+ */
+struct rspamd_radix_map_helper;
+struct rspamd_hash_map_helper;
+struct rspamd_regexp_map_helper;
+
+typedef void (*insert_func) (gpointer st, gconstpointer key,
+ gconstpointer value);
+
+/**
+ * Radix list is a list like ip/mask
+ */
+gchar * rspamd_radix_read (
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+void rspamd_radix_fin (struct map_cb_data *data);
+
+/**
+ * Host list is an ordinal list of hosts or domains
+ */
+gchar * rspamd_hosts_read (
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+void rspamd_hosts_fin (struct map_cb_data *data);
+
+/**
+ * Kv list is an ordinal list of keys and values separated by whitespace
+ */
+gchar * rspamd_kv_list_read (
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+void rspamd_kv_list_fin (struct map_cb_data *data);
+
+/**
+ * Regexp list is a list of regular expressions
+ */
+
+gchar * rspamd_regexp_list_read_single (
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+gchar * rspamd_regexp_list_read_multiple (
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+gchar * rspamd_glob_list_read_single (
+ gchar *chunk,
+ gint len,
+ struct map_cb_data *data,
+ gboolean final);
+void rspamd_regexp_list_fin (struct map_cb_data *data);
+
+/**
+ * FSM for lists parsing (support comments, blank lines and partial replies)
+ */
+gchar *
+rspamd_parse_kv_list (
+ gchar * chunk,
+ gint len,
+ struct map_cb_data *data,
+ insert_func func,
+ const gchar *default_value,
+ gboolean final);
+
+/**
+ * Find a single (any) matching regexp for the specified text or NULL if
+ * no matches found
+ * @param map
+ * @param in
+ * @param len
+ * @return
+ */
+gpointer rspamd_match_regexp_map_single (struct rspamd_regexp_map *map,
+ const gchar *in, gsize len);
+
+/**
+ * Find a multiple (all) matching regexp for the specified text or NULL if
+ * no matches found. Returns GPtrArray that *must* be freed by a caller if not NULL
+ * @param map
+ * @param in
+ * @param len
+ * @return
+ */
+gpointer rspamd_match_regexp_map_all (struct rspamd_regexp_map *map,
+ const gchar *in, gsize len);
+
+#endif
projects / rspamd.git / commitdiff