#include "../message.h"
#include "../modules.h"
#include "../cfg_file.h"
+#include "../map.h"
#include "../util.h"
#include "../expressions.h"
#include "../view.h"
#include "../lua/lua_common.h"
+#include "../json/jansson.h"
#define DEFAULT_STATFILE_PREFIX "./"
char *statfile_prefix;
memory_pool_t *regexp_pool;
+ memory_pool_t *dynamic_pool;
+};
+
+struct regexp_json_buf {
+ u_char *buf;
+ u_char *pos;
+ size_t buflen;
+ struct config_file *cfg;
};
static struct regexp_ctx *regexp_module_ctx = NULL;
static void process_regexp_item (struct worker_task *task, void *user_data);
-int
-regexp_module_init (struct config_file *cfg, struct module_ctx **ctx)
+static void
+regexp_dynamic_insert_result (struct worker_task *task, void *user_data)
{
- regexp_module_ctx = g_malloc (sizeof (struct regexp_ctx));
+ char *symbol = user_data;
+
+ insert_result (task, regexp_module_ctx->metric, symbol, 1, NULL);
+}
- regexp_module_ctx->filter = regexp_common_filter;
- regexp_module_ctx->regexp_pool = memory_pool_new (memory_pool_get_size ());
- regexp_module_ctx->autolearn_symbols = g_hash_table_new (g_str_hash, g_str_equal);
+static gboolean
+parse_regexp_ipmask (const char *begin, struct dynamic_map_item *addr)
+{
+ const char *pos;
+ char ip_buf[sizeof ("255.255.255.255")], mask_buf[3], *p;
+ int state = 0, dots = 0;
+
+ bzero (ip_buf, sizeof (ip_buf));
+ bzero (mask_buf, sizeof (mask_buf));
+ pos = begin;
+ p = ip_buf;
+
+ while (*pos) {
+ switch (state) {
+ case 0:
+ state = 1;
+ p = ip_buf;
+ dots = 0;
+ break;
+ case 1:
+ /* Begin parse ip */
+ if (p - ip_buf >= sizeof (ip_buf) || dots > 3) {
+ return FALSE;
+ }
+ if (g_ascii_isdigit (*pos)) {
+ *p ++ = *pos ++;
+ }
+ else if (*pos == '.') {
+ *p ++ = *pos ++;
+ dots ++;
+ }
+ else if (*pos == '/') {
+ pos ++;
+ p = mask_buf;
+ state = 2;
+ }
+ else {
+ /* Invalid character */
+ return FALSE;
+ }
+ break;
+ case 2:
+ /* Parse mask */
+ if (p - mask_buf > 2) {
+ return FALSE;
+ }
+ if (g_ascii_isdigit (*pos)) {
+ *p ++ = *pos ++;
+ }
+ else {
+ return FALSE;
+ }
+ break;
+ }
+ }
- *ctx = (struct module_ctx *)regexp_module_ctx;
- register_expression_function ("regexp_match_number", rspamd_regexp_match_number, NULL);
- register_expression_function ("raw_header_exists", rspamd_raw_header_exists, NULL);
- register_expression_function ("check_smtp_data", rspamd_check_smtp_data, NULL);
+ if (!inet_aton (ip_buf, &addr->addr)) {
+ return FALSE;
+ }
+ if (state == 2) {
+ /* Also parse mask */
+ addr->mask = (mask_buf[0] - '0') * 10 + mask_buf[1] - '0';
+ if (addr->mask > 32) {
+ msg_info ("bad ipmask value: '%s'", begin);
+ return FALSE;
+ }
+ }
+ else {
+ addr->mask = 32;
+ }
+
+ return TRUE;
- return 0;
}
+/* Process regexp expression */
static gboolean
-read_regexp_expression (memory_pool_t * pool, struct regexp_module_item *chain, char *symbol, char *line, struct config_file *cfg)
+read_regexp_expression (memory_pool_t * pool, struct regexp_module_item *chain, char *symbol, char *line, gboolean raw_mode)
{
struct expression *e, *cur;
- e = parse_expression (regexp_module_ctx->regexp_pool, line);
+ e = parse_expression (pool, line);
if (e == NULL) {
msg_warn ("%s = \"%s\" is invalid regexp expression", symbol, line);
return FALSE;
cur = e;
while (cur) {
if (cur->type == EXPR_REGEXP) {
- cur->content.operand = parse_regexp (pool, cur->content.operand, cfg->raw_mode);
+ cur->content.operand = parse_regexp (pool, cur->content.operand, raw_mode);
if (cur->content.operand == NULL) {
msg_warn ("cannot parse regexp, skip expression %s = \"%s\"", symbol, line);
return FALSE;
return TRUE;
}
+
+/* Callbacks for reading json dynamic rules */
+u_char *
+json_regexp_read_cb (memory_pool_t * pool, u_char * chunk, size_t len, struct map_cb_data *data)
+{
+ struct regexp_json_buf *jb;
+ size_t free, off;
+
+ if (data->cur_data == NULL) {
+ jb = g_malloc (sizeof (struct regexp_json_buf));
+ jb->cfg = ((struct regexp_json_buf *)data->prev_data)->cfg;
+ jb->buf = NULL;
+ jb->pos = NULL;
+ data->cur_data = jb;
+ }
+ else {
+ jb = data->cur_data;
+ }
+
+ if (jb->buf == NULL) {
+ /* Allocate memory for buffer */
+ jb->buflen = len * 2;
+ jb->buf = g_malloc (jb->buflen);
+ jb->pos = jb->buf;
+ }
+
+ off = jb->pos - jb->buf;
+ free = jb->buflen - off;
+
+ if (free < len) {
+ jb->buflen = MAX (jb->buflen * 2, jb->buflen + len * 2);
+ jb->buf = g_realloc (jb->buf, jb->buflen);
+ jb->pos = jb->buf + off;
+ }
+
+ memcpy (jb->pos, chunk, len);
+ jb->pos += len;
+
+ /* Say not to copy any part of this buffer */
+ return NULL;
+}
+
+void
+json_regexp_fin_cb (memory_pool_t * pool, struct map_cb_data *data)
+{
+ struct regexp_json_buf *jb;
+ int nelts, i, j;
+ json_t *js, *cur_elt, *cur_nm, *it_val;
+ json_error_t je;
+ char *cur_rule, *cur_symbol;
+ double score;
+ struct regexp_module_item *cur_item;
+ GList *cur_networks = NULL;
+ struct dynamic_map_item *cur_nitem;
+ memory_pool_t *new_pool;
+ struct metric *metric;
+
+ if (data->prev_data) {
+ jb = data->prev_data;
+ /* Clean prev data */
+ if (jb->buf) {
+ g_free (jb->buf);
+ }
+ g_free (jb);
+ }
+
+ /* Now parse json */
+ if (data->cur_data) {
+ jb = data->cur_data;
+ }
+ else {
+ msg_err ("no data read");
+ return;
+ }
+ if (jb->buf == NULL) {
+ msg_err ("no data read");
+ return;
+ }
+ /* NULL terminate current buf */
+ *jb->pos = '\0';
+
+ js = json_loads (jb->buf, &je);
+ if (!js) {
+ msg_err ("cannot load json data: parse error %s, on line %d", je.text, je.line);
+ return;
+ }
+
+ if (!json_is_array (js)) {
+ json_decref (js);
+ msg_err ("loaded json is not an array");
+ return;
+ }
+
+ new_pool = memory_pool_new (memory_pool_get_size ());
+ metric = g_hash_table_lookup (jb->cfg->metrics, regexp_module_ctx->metric);
+ if (metric == NULL) {
+ msg_err ("cannot find metric definition %s", regexp_module_ctx->metric);
+ return;
+ }
+
+ remove_dynamic_rules (metric->cache);
+ if (regexp_module_ctx->dynamic_pool != NULL) {
+ memory_pool_delete (regexp_module_ctx->dynamic_pool);
+ }
+ regexp_module_ctx->dynamic_pool = new_pool;
+
+ nelts = json_array_size (js);
+ for (i = 0; i < nelts; i++) {
+ cur_networks = NULL;
+ cur_rule = NULL;
+
+ cur_elt = json_array_get (js, i);
+ if (!cur_elt || !json_is_object (cur_elt)) {
+ msg_err ("loaded json is not an object");
+ continue;
+ }
+ /* Factor param */
+ cur_nm = json_object_get (cur_elt, "factor");
+ if (cur_nm == NULL || !json_is_number (cur_nm)) {
+ msg_err ("factor is not a number or not exists, but is required");
+ continue;
+ }
+ score = json_number_value (cur_nm);
+ /* Symbol param */
+ cur_nm = json_object_get (cur_elt, "symbol");
+ if (cur_nm == NULL || !json_is_string (cur_nm)) {
+ msg_err ("symbol is not a string or not exists, but is required");
+ continue;
+ }
+ cur_symbol = memory_pool_strdup (new_pool, json_string_value (cur_nm));
+ /* Now check other settings */
+ /* Rule */
+ cur_nm = json_object_get (cur_elt, "rule");
+ if (cur_nm != NULL && json_is_string (cur_nm)) {
+ cur_rule = memory_pool_strdup (new_pool, json_string_value (cur_nm));
+ }
+ /* Networks array */
+ cur_nm = json_object_get (cur_elt, "networks");
+ if (cur_nm != NULL && json_is_array (cur_nm)) {
+ for (j = 0; j < json_array_size (cur_nm); j++) {
+ it_val = json_array_get (cur_nm, i);
+ if (it_val && json_is_string (it_val)) {
+ cur_nitem = memory_pool_alloc (new_pool, sizeof (struct dynamic_map_item));
+ if (parse_regexp_ipmask (json_string_value (it_val), cur_nitem)) {
+ cur_networks = g_list_prepend (cur_networks, cur_nitem);
+ }
+ }
+ }
+ }
+ if (cur_rule) {
+ /* Dynamic rule has rule option */
+ cur_item = memory_pool_alloc0 (new_pool, sizeof (struct regexp_module_item));
+ cur_item->symbol = cur_symbol;
+ if (read_regexp_expression (new_pool, cur_item, cur_symbol, cur_rule, jb->cfg->raw_mode)) {
+ register_dynamic_symbol (new_pool, &metric->cache, cur_symbol, score, process_regexp_item, cur_item, cur_networks);
+ }
+ else {
+ msg_warn ("cannot parse dynamic rule");
+ }
+ }
+ else {
+ /* Just rule that is allways true (for whitelisting for example) */
+ register_dynamic_symbol (new_pool, &metric->cache, cur_symbol, score, regexp_dynamic_insert_result, cur_symbol, cur_networks);
+ }
+ if (cur_networks) {
+ g_list_free (cur_networks);
+ }
+ }
+ json_decref (js);
+}
+
+/* Init function */
+int
+regexp_module_init (struct config_file *cfg, struct module_ctx **ctx)
+{
+ regexp_module_ctx = g_malloc (sizeof (struct regexp_ctx));
+
+ regexp_module_ctx->filter = regexp_common_filter;
+ regexp_module_ctx->regexp_pool = memory_pool_new (memory_pool_get_size ());
+ regexp_module_ctx->dynamic_pool = NULL;
+ regexp_module_ctx->autolearn_symbols = g_hash_table_new (g_str_hash, g_str_equal);
+
+ *ctx = (struct module_ctx *)regexp_module_ctx;
+ register_expression_function ("regexp_match_number", rspamd_regexp_match_number, NULL);
+ register_expression_function ("raw_header_exists", rspamd_raw_header_exists, NULL);
+ register_expression_function ("check_smtp_data", rspamd_check_smtp_data, NULL);
+
+ return 0;
+}
+
+
/*
* Parse string in format:
* SYMBOL:statfile:weight
char *value;
int res = TRUE;
double *w;
+ struct regexp_json_buf *jb, **pjb;
if ((value = get_module_opt (cfg, "regexp", "metric")) != NULL) {
regexp_module_ctx->metric = memory_pool_strdup (regexp_module_ctx->regexp_pool, value);
else {
regexp_module_ctx->statfile_prefix = DEFAULT_STATFILE_PREFIX;
}
+ if ((value = get_module_opt (cfg, "regexp", "dynamic_rules")) != NULL) {
+ jb = g_malloc (sizeof (struct regexp_json_buf));
+ pjb = g_malloc (sizeof (struct regexp_json_buf *));
+ jb->buf = NULL;
+ jb->cfg = cfg;
+ *pjb = jb;
+ if (!add_map (value, json_regexp_read_cb, json_regexp_fin_cb, (void **)pjb)) {
+ msg_err ("cannot add map %s", value);
+ }
+ }
metric = g_hash_table_lookup (cfg->metrics, regexp_module_ctx->metric);
if (metric == NULL) {
cur_item = memory_pool_alloc0 (regexp_module_ctx->regexp_pool, sizeof (struct regexp_module_item));
cur_item->symbol = cur->param;
if (cur->is_lua && cur->lua_type == LUA_VAR_STRING) {
- if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, cur->param, cur->actual_data, cfg)) {
+ if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, cur->param, cur->actual_data, cfg->raw_mode)) {
res = FALSE;
}
}
cur_item->lua_function = cur->actual_data;
}
else if (! cur->is_lua) {
- if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, cur->param, cur->value, cfg)) {
+ if (!read_regexp_expression (regexp_module_ctx->regexp_pool, cur_item, cur->param, cur->value, cfg->raw_mode)) {
res = FALSE;
}
}
}
void
-register_dynamic_symbol (struct symbols_cache **cache, const char *name, double weight, symbol_func_t func,
- gpointer user_data, struct dynamic_map_item *networks, gsize network_count)
+register_dynamic_symbol (memory_pool_t *dynamic_pool, struct symbols_cache **cache,
+ const char *name, double weight, symbol_func_t func,
+ gpointer user_data, GList *networks)
{
struct cache_item *item = NULL;
struct symbols_cache *pcache = *cache;
- GList **target, *t;
- gsize i;
+ GList **target, *t, *cur;
uintptr_t r;
uint32_t mask = 0xFFFFFFFF;
+ struct dynamic_map_item *it;
if (*cache == NULL) {
pcache = g_new0 (struct symbols_cache, 1);
pcache->static_pool = memory_pool_new (memory_pool_get_size ());
}
- if (pcache->dynamic_pool == NULL) {
- pcache->dynamic_pool = memory_pool_new (memory_pool_get_size ());
- }
- item = memory_pool_alloc0 (pcache->dynamic_pool, sizeof (struct cache_item));
- item->s = memory_pool_alloc (pcache->dynamic_pool, sizeof (struct saved_cache_item));
+ item = memory_pool_alloc0 (dynamic_pool, sizeof (struct cache_item));
+ item->s = memory_pool_alloc (dynamic_pool, sizeof (struct saved_cache_item));
g_strlcpy (item->s->symbol, name, sizeof (item->s->symbol));
item->func = func;
item->user_data = user_data;
msg_debug ("used items: %d, added symbol: %s", (*cache)->used_items, name);
set_counter (item->s->symbol, 0);
- if (network_count == 0 || networks == NULL) {
+ if (networks == NULL) {
target = &pcache->dynamic_items;
}
else {
if (pcache->dynamic_map == NULL) {
pcache->dynamic_map = radix_tree_create ();
}
- for (i = 0; i < network_count; i ++) {
- mask = mask << (32 - networks[i].mask);
- r = ntohl (networks[i].addr.s_addr & mask);
+ cur = networks;
+ while (cur) {
+ it = cur->data;
+ mask = mask << (32 - it->mask);
+ r = ntohl (it->addr.s_addr & mask);
if ((r = radix32tree_find (pcache->dynamic_map, r)) != RADIX_NO_VALUE) {
t = (GList *)((gpointer)r);
target = &t;
}
else {
t = g_list_prepend (NULL, item);
- memory_pool_add_destructor (pcache->dynamic_pool, (pool_destruct_func)g_list_free, t);
- r = radix32tree_insert (pcache->dynamic_map, ntohl (networks[i].addr.s_addr), mask, (uintptr_t)t);
+ memory_pool_add_destructor (dynamic_pool, (pool_destruct_func)g_list_free, t);
+ r = radix32tree_insert (pcache->dynamic_map, ntohl (it->addr.s_addr), mask, (uintptr_t)t);
if (r == -1) {
- msg_warn ("cannot insert ip to tree: %s, mask %X", inet_ntoa (networks[i].addr), mask);
+ msg_warn ("cannot insert ip to tree: %s, mask %X", inet_ntoa (it->addr), mask);
}
else if (r == 1) {
- msg_warn ("ip %s, mask %X, value already exists", inet_ntoa (networks[i].addr), mask);
+ msg_warn ("ip %s, mask %X, value already exists", inet_ntoa (it->addr), mask);
}
- return;
}
+ cur = g_list_next (cur);
}
}
*target = g_list_prepend (*target, item);
}
void
-remove_dynamic_items (struct symbols_cache *cache)
+remove_dynamic_rules (struct symbols_cache *cache)
{
if (cache->dynamic_items) {
g_list_free (cache->dynamic_items);
if (cache->dynamic_map) {
radix_tree_free (cache->dynamic_map);
}
-
- /* Do magic */
- memory_pool_delete (cache->dynamic_pool);
- cache->dynamic_pool = NULL;
}
static void
}
memory_pool_delete (cache->static_pool);
- if (cache->dynamic_pool) {
- memory_pool_delete (cache->dynamic_pool);
- }
g_free (cache);
}
projects / rspamd.git / commitdiff