/*- * Copyright 2016 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* * Implementation of map files handling */ #include "config.h" #include "map.h" #include "http.h" #include "rspamd.h" #include "cryptobox.h" #include "unix-std.h" static const gchar *hash_fill = "1"; /** * Data specific to file maps */ struct file_map_data { const gchar *filename; struct stat st; }; /** * Data specific to HTTP maps */ struct http_map_data { struct addrinfo *addr; guint16 port; gchar *path; gchar *host; time_t last_checked; gboolean request_sent; struct rspamd_http_connection *conn; }; struct http_callback_data { struct event_base *ev_base; struct timeval tv; struct rspamd_map *map; struct http_map_data *data; struct map_cb_data cbdata; GString *remain_buf; gint fd; }; /* Value in seconds after whitch we would try to do stat on list file */ /* HTTP timeouts */ #define HTTP_CONNECT_TIMEOUT 2 #define HTTP_READ_TIMEOUT 10 /** * Helper for HTTP connection establishment */ static gint connect_http (struct rspamd_map *map, struct http_map_data *data, gboolean is_async) { gint sock; rspamd_mempool_t *pool; pool = map->pool; if ((sock = rspamd_socket_tcp (data->addr, FALSE, is_async)) == -1) { msg_info_pool ("cannot connect to http server %s: %d, %s", data->host, errno, strerror (errno)); return -1; } return sock; } /** * Write HTTP request */ static void write_http_request (struct http_callback_data *cbd) { gchar datebuf[128]; struct tm *tm; struct rspamd_http_message *msg; msg = rspamd_http_new_message (HTTP_REQUEST); msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path)); if (cbd->data->last_checked != 0) { tm = gmtime (&cbd->data->last_checked); strftime (datebuf, sizeof (datebuf), "%a, %d %b %Y %H:%M:%S %Z", tm); rspamd_http_message_add_header (msg, "If-Modified-Since", datebuf); } rspamd_http_connection_write_message (cbd->data->conn, msg, cbd->data->host, NULL, cbd, cbd->fd, &cbd->tv, cbd->ev_base); } /** * Callback for destroying HTTP callback data */ static void free_http_cbdata (struct http_callback_data *cbd) { g_atomic_int_set (cbd->map->locked, 0); if (cbd->remain_buf) { g_string_free (cbd->remain_buf, TRUE); } rspamd_http_connection_reset (cbd->data->conn); close (cbd->fd); g_slice_free1 (sizeof (struct http_callback_data), cbd); } /* * HTTP callbacks */ static void http_map_error (struct rspamd_http_connection *conn, GError *err) { struct http_callback_data *cbd = conn->ud; rspamd_mempool_t *pool; pool = cbd->map->pool; msg_err_pool ("connection with http server terminated incorrectly: %s", err->message); free_http_cbdata (cbd); } static int http_map_finish (struct rspamd_http_connection *conn, struct rspamd_http_message *msg) { struct http_callback_data *cbd = conn->ud; struct rspamd_map *map; rspamd_mempool_t *pool; map = cbd->map; pool = cbd->map->pool; if (msg->code == 200) { if (cbd->remain_buf != NULL) { /* Append \n to avoid issues */ g_string_append_c (cbd->remain_buf, '\n'); map->read_callback (map->pool, cbd->remain_buf->str, cbd->remain_buf->len, &cbd->cbdata); } map->fin_callback (map->pool, &cbd->cbdata); *map->user_data = cbd->cbdata.cur_data; cbd->data->last_checked = msg->date; msg_info_pool ("read map data from %s", cbd->data->host); } else if (msg->code == 304) { msg_debug_pool ("data is not modified for server %s", cbd->data->host); cbd->data->last_checked = msg->date; } else { msg_info_pool ("cannot load map %s from %s: HTTP error %d", map->uri, cbd->data->host, msg->code); } free_http_cbdata (cbd); return 0; } static int http_map_read (struct rspamd_http_connection *conn, struct rspamd_http_message *msg, const gchar *chunk, gsize len) { struct http_callback_data *cbd = conn->ud; gchar *pos; struct rspamd_map *map; if (msg->code != 200 || len == 0) { /* Ignore not full replies */ return 0; } map = cbd->map; if (cbd->remain_buf != NULL) { /* We need to concatenate incoming buf with the remaining buf */ g_string_append_len (cbd->remain_buf, chunk, len); pos = map->read_callback (map->pool, cbd->remain_buf->str, cbd->remain_buf->len, &cbd->cbdata); /* All read */ if (pos == NULL) { g_string_free (cbd->remain_buf, TRUE); cbd->remain_buf = NULL; } else { /* Need to erase data processed */ g_string_erase (cbd->remain_buf, 0, pos - cbd->remain_buf->str); } } else { pos = map->read_callback (map->pool, (gchar *)chunk, len, &cbd->cbdata); if (pos != NULL) { /* Store data in remain buf */ cbd->remain_buf = g_string_new_len (pos, len - (pos - chunk)); } } return 0; } /** * Callback for reading data from file */ static void read_map_file (struct rspamd_map *map, struct file_map_data *data) { struct map_cb_data cbdata; gchar buf[BUFSIZ], *remain = NULL; ssize_t r; gint fd, rlen, tlen; rspamd_mempool_t *pool = map->pool; if (map->read_callback == NULL || map->fin_callback == NULL) { msg_err_pool ("bad callback for reading map file"); return; } if ((fd = open (data->filename, O_RDONLY)) == -1) { msg_warn_pool ("cannot open file '%s': %s", data->filename, strerror (errno)); return; } cbdata.state = 0; cbdata.prev_data = *map->user_data; cbdata.cur_data = NULL; cbdata.map = map; rlen = 0; tlen = 0; while ((r = read (fd, buf + rlen, sizeof (buf) - rlen - 2)) > 0) { r += rlen; tlen += r; buf[r] = '\0'; remain = map->read_callback (map->pool, buf, r, &cbdata); if (remain != NULL) { /* copy remaining buffer to start of buffer */ rlen = r - (remain - buf); memmove (buf, remain, rlen); } else { rlen = 0; } } if (remain != NULL && remain > buf) { g_assert (rlen <= (gint)sizeof (buf) - 2); buf[rlen++] = '\n'; buf[rlen] = '\0'; tlen += rlen; map->read_callback (map->pool, buf, rlen, &cbdata); } close (fd); if (tlen > 0) { map->fin_callback (map->pool, &cbdata); *map->user_data = cbdata.cur_data; } } static void jitter_timeout_event (struct rspamd_map *map, gboolean locked, gboolean initial) { gdouble jittered_sec; gdouble timeout = initial ? 1.0 : map->cfg->map_timeout; /* Plan event again with jitter */ evtimer_del (&map->ev); jittered_sec = rspamd_time_jitter (locked ? timeout * 4 : timeout, 0); double_to_tv (jittered_sec, &map->tv); evtimer_add (&map->ev, &map->tv); } /** * Common file callback */ static void file_callback (gint fd, short what, void *ud) { struct rspamd_map *map = ud; struct file_map_data *data = map->map_data; struct stat st; rspamd_mempool_t *pool; pool = map->pool; if (g_atomic_int_get (map->locked)) { msg_info_pool ( "don't try to reread map as it is locked by other process, will reread it later"); jitter_timeout_event (map, TRUE, FALSE); return; } g_atomic_int_inc (map->locked); jitter_timeout_event (map, FALSE, FALSE); if (stat (data->filename, &st) != -1 && (st.st_mtime > data->st.st_mtime || data->st.st_mtime == -1)) { /* File was modified since last check */ memcpy (&data->st, &st, sizeof (struct stat)); } else { g_atomic_int_set (map->locked, 0); return; } msg_info_pool ("rereading map file %s", data->filename); read_map_file (map, data); g_atomic_int_set (map->locked, 0); } /** * Async HTTP callback */ static void http_callback (gint fd, short what, void *ud) { struct rspamd_map *map = ud; struct http_map_data *data; gint sock; struct http_callback_data *cbd; rspamd_mempool_t *pool; data = map->map_data; pool = map->pool; if (g_atomic_int_get (map->locked)) { msg_info_pool ( "don't try to reread map as it is locked by other process, will reread it later"); if (data->conn->ud == NULL) { jitter_timeout_event (map, TRUE, TRUE); } else { jitter_timeout_event (map, TRUE, FALSE); } return; } g_atomic_int_inc (map->locked); jitter_timeout_event (map, FALSE, FALSE); /* Connect asynced */ if ((sock = connect_http (map, data, TRUE)) == -1) { g_atomic_int_set (map->locked, 0); return; } else { /* Plan event */ cbd = g_slice_alloc (sizeof (struct http_callback_data)); cbd->ev_base = map->ev_base; cbd->map = map; cbd->data = data; cbd->remain_buf = NULL; cbd->cbdata.state = 0; cbd->cbdata.prev_data = *cbd->map->user_data; cbd->cbdata.cur_data = NULL; cbd->cbdata.map = cbd->map; cbd->tv.tv_sec = HTTP_CONNECT_TIMEOUT; cbd->tv.tv_usec = 0; cbd->fd = sock; data->conn->ud = cbd; msg_debug_pool ("reading map data from %s", data->host); write_http_request (cbd); } } /* Start watching event for all maps */ void rspamd_map_watch (struct rspamd_config *cfg, struct event_base *ev_base) { GList *cur = cfg->maps; struct rspamd_map *map; struct file_map_data *fdata; /* First of all do synced read of data */ while (cur) { map = cur->data; map->ev_base = ev_base; event_base_set (map->ev_base, &map->ev); if (map->protocol == MAP_PROTO_FILE) { evtimer_set (&map->ev, file_callback, map); /* Read initial data */ fdata = map->map_data; if (fdata->st.st_mtime != -1) { /* Do not try to read non-existent file */ read_map_file (map, map->map_data); } /* Plan event with jitter */ jitter_timeout_event (map, FALSE, TRUE); } else if (map->protocol == MAP_PROTO_HTTP) { evtimer_set (&map->ev, http_callback, map); jitter_timeout_event (map, FALSE, TRUE); } cur = g_list_next (cur); } } void rspamd_map_remove_all (struct rspamd_config *cfg) { g_list_free (cfg->maps); cfg->maps = NULL; if (cfg->map_pool != NULL) { rspamd_mempool_delete (cfg->map_pool); cfg->map_pool = NULL; } } gboolean rspamd_map_check_proto (const gchar *map_line, gint *res, const gchar **pos) { g_assert (res != NULL); g_assert (pos != NULL); if (g_ascii_strncasecmp (map_line, "http://", sizeof ("http://") - 1) == 0) { *res = MAP_PROTO_HTTP; *pos = map_line + sizeof ("http://") - 1; } else if (g_ascii_strncasecmp (map_line, "file://", sizeof ("file://") - 1) == 0) { *res = MAP_PROTO_FILE; *pos = map_line + sizeof ("file://") - 1; } else if (*map_line == '/') { /* Trivial file case */ *res = MAP_PROTO_FILE; *pos = map_line; } else { msg_debug ("invalid map fetching protocol: %s", map_line); return FALSE; } return TRUE; } gboolean rspamd_map_add (struct rspamd_config *cfg, const gchar *map_line, const gchar *description, map_cb_t read_callback, map_fin_cb_t fin_callback, void **user_data) { struct rspamd_map *new_map; enum fetch_proto proto; const gchar *def, *p, *hostend; struct file_map_data *fdata; struct http_map_data *hdata; gchar portbuf[6], *cksum_encoded, cksum[rspamd_cryptobox_HASHBYTES]; gint i, s, r; struct addrinfo hints, *res; rspamd_mempool_t *pool; /* First of all detect protocol line */ if (!rspamd_map_check_proto (map_line, (int *)&proto, &def)) { return FALSE; } /* Constant pool */ if (cfg->map_pool == NULL) { cfg->map_pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map"); memcpy (cfg->map_pool->tag.uid, cfg->cfg_pool->tag.uid, sizeof (cfg->map_pool->tag.uid)); } new_map = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct rspamd_map)); new_map->read_callback = read_callback; new_map->fin_callback = fin_callback; new_map->user_data = user_data; new_map->protocol = proto; new_map->cfg = cfg; new_map->id = g_random_int (); new_map->locked = rspamd_mempool_alloc0_shared (cfg->cfg_pool, sizeof (gint)); if (proto == MAP_PROTO_FILE) { new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, def); def = new_map->uri; } else { new_map->uri = rspamd_mempool_strdup (cfg->cfg_pool, map_line); } if (description != NULL) { new_map->description = rspamd_mempool_strdup (cfg->cfg_pool, description); } /* Now check for each proto separately */ if (proto == MAP_PROTO_FILE) { fdata = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct file_map_data)); if (access (def, R_OK) == -1) { if (errno != ENOENT) { msg_err_config ("cannot open file '%s': %s", def, strerror (errno)); return FALSE; } msg_info_config ( "map '%s' is not found, but it can be loaded automatically later", def); /* We still can add this file */ fdata->st.st_mtime = -1; } else { stat (def, &fdata->st); } fdata->filename = rspamd_mempool_strdup (cfg->map_pool, def); new_map->map_data = fdata; } else if (proto == MAP_PROTO_HTTP) { hdata = rspamd_mempool_alloc0 (cfg->map_pool, sizeof (struct http_map_data)); /* Try to search port */ if ((p = strchr (def, ':')) != NULL) { hostend = p; i = 0; p++; while (g_ascii_isdigit (*p) && i < (gint)sizeof (portbuf) - 1) { portbuf[i++] = *p++; } if (*p != '/') { msg_info_config ("bad http map definition: %s", def); return FALSE; } portbuf[i] = '\0'; hdata->port = atoi (portbuf); } else { /* Default http port */ rspamd_snprintf (portbuf, sizeof (portbuf), "80"); hdata->port = 80; /* Now separate host from path */ if ((p = strchr (def, '/')) == NULL) { msg_info_config ("bad http map definition: %s", def); return FALSE; } hostend = p; } hdata->host = rspamd_mempool_alloc (cfg->map_pool, hostend - def + 1); rspamd_strlcpy (hdata->host, def, hostend - def + 1); hdata->path = rspamd_mempool_strdup (cfg->map_pool, p); /* Now try to resolve */ memset (&hints, 0, sizeof (hints)); hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */ hints.ai_socktype = SOCK_STREAM; /* Stream socket */ hints.ai_flags = 0; hints.ai_protocol = 0; /* Any protocol */ hints.ai_canonname = NULL; hints.ai_addr = NULL; hints.ai_next = NULL; if ((r = getaddrinfo (hdata->host, portbuf, &hints, &res)) == 0) { hdata->addr = res; rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)freeaddrinfo, hdata->addr); } else { msg_err_config ("address resolution for %s failed: %s", hdata->host, gai_strerror (r)); return FALSE; } /* Now try to connect */ if ((s = rspamd_socket_tcp (hdata->addr, FALSE, FALSE)) == -1) { msg_info_config ("cannot connect to http server %s: %d, %s", hdata->host, errno, strerror (errno)); return FALSE; } close (s); hdata->conn = rspamd_http_connection_new (http_map_read, http_map_error, http_map_finish, RSPAMD_HTTP_BODY_PARTIAL | RSPAMD_HTTP_CLIENT_SIMPLE, RSPAMD_HTTP_CLIENT, NULL); new_map->map_data = hdata; } /* Temp pool */ rspamd_cryptobox_hash (cksum, new_map->uri, strlen (new_map->uri), NULL, 0); cksum_encoded = rspamd_encode_base32 (cksum, sizeof (cksum)); new_map->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "map"); memcpy (new_map->pool->tag.uid, cksum_encoded, sizeof (new_map->pool->tag.uid)); g_free (cksum_encoded); pool = new_map->pool; msg_info_pool ("added map %s", new_map->uri); cfg->maps = g_list_prepend (cfg->maps, new_map); return TRUE; } static gchar* strip_map_elt (rspamd_mempool_t *pool, const gchar *start, size_t len) { gchar *res = NULL; const gchar *c = start, *p = start + len - 1; /* Strip starting spaces */ while (g_ascii_isspace (*c)) { c ++; } /* Strip ending spaces */ while (g_ascii_isspace (*p) && p >= c) { p --; } /* One symbol up */ p ++; if (p - c > 0) { res = rspamd_mempool_alloc (pool, p - c + 1); rspamd_strlcpy (res, c, p - c + 1); } return res; } /** * FSM for parsing lists */ gchar * abstract_parse_kv_list (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map_cb_data *data, insert_func func) { gchar *c, *p, *key = NULL, *value = NULL, *end; p = chunk; c = p; end = p + len; while (p < end) { switch (data->state) { case 0: /* read key */ /* Check here comments, eol and end of buffer */ if (*p == '#') { if (key != NULL && p - c >= 0) { value = rspamd_mempool_alloc (pool, p - c + 1); memcpy (value, c, p - c); value[p - c] = '\0'; value = g_strstrip (value); func (data->cur_data, key, value); msg_debug_pool ("insert kv pair: %s -> %s", key, value); } data->state = 99; } else if (*p == '\r' || *p == '\n') { if (key != NULL && p - c >= 0) { value = rspamd_mempool_alloc (pool, p - c + 1); memcpy (value, c, p - c); value[p - c] = '\0'; value = g_strstrip (value); func (data->cur_data, key, value); msg_debug_pool ("insert kv pair: %s -> %s", key, value); } else if (key == NULL && p - c > 0) { /* Key only line */ key = rspamd_mempool_alloc (pool, p - c + 1); memcpy (key, c, p - c); key[p - c] = '\0'; value = rspamd_mempool_alloc (pool, 1); *value = '\0'; func (data->cur_data, key, value); msg_debug_pool ("insert kv pair: %s -> %s", key, value); } data->state = 100; key = NULL; } else if (g_ascii_isspace (*p)) { if (p - c > 0) { key = rspamd_mempool_alloc (pool, p - c + 1); memcpy (key, c, p - c); key[p - c] = '\0'; data->state = 2; } else { key = NULL; } } else { p++; } break; case 2: /* Skip spaces before value */ if (!g_ascii_isspace (*p)) { c = p; data->state = 0; } else { p++; } break; case 99: /* SKIP_COMMENT */ /* Skip comment till end of line */ if (*p == '\r' || *p == '\n') { while ((*p == '\r' || *p == '\n') && p < end) { p++; } c = p; key = NULL; data->state = 0; } else { p++; } break; case 100: /* Skip \r\n and whitespaces */ if (*p == '\r' || *p == '\n' || g_ascii_isspace (*p)) { p++; } else { c = p; key = NULL; data->state = 0; } break; } } return c; } gchar * rspamd_parse_abstract_list (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map_cb_data *data, insert_func func) { gchar *p, *c, *end, *s; p = chunk; c = p; end = p + len; while (p < end) { switch (data->state) { /* READ_SYMBOL */ case 0: if (*p == '#') { /* Got comment */ if (p > c) { /* Save previous string in lines like: "127.0.0.1 #localhost" */ s = strip_map_elt (pool, c, p - c); if (s) { func (data->cur_data, s, hash_fill); msg_debug_pool ("insert element (before comment): %s", s); } } c = p; data->state = 1; } else if (*p == '\r' || *p == '\n') { /* Got EOL marker, save stored string */ s = strip_map_elt (pool, c, p - c); if (s) { func (data->cur_data, s, hash_fill); msg_debug_pool ("insert element (before EOL): %s", s); } /* Skip EOL symbols */ while ((*p == '\r' || *p == '\n') && p < end) { p++; } if (p == end) { p ++; c = NULL; } else { c = p; } } else { p++; } break; /* SKIP_COMMENT */ case 1: /* Skip comment till end of line */ if (*p == '\r' || *p == '\n') { while ((*p == '\r' || *p == '\n') && p < end) { p++; } if (p == end) { p ++; c = NULL; } else { c = p; } data->state = 0; } else { p++; } break; } } if (c >= end) { c = NULL; } return c; } /** * Radix tree helper function */ static void radix_tree_insert_helper (gpointer st, gconstpointer key, gpointer value) { radix_compressed_t *tree = (radix_compressed_t *)st; rspamd_radix_add_iplist ((gchar *)key, " ,;", tree); } /* Helpers */ gchar * rspamd_hosts_read (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map_cb_data *data) { if (data->cur_data == NULL) { data->cur_data = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); } return rspamd_parse_abstract_list (pool, chunk, len, data, (insert_func) g_hash_table_insert); } void rspamd_hosts_fin (rspamd_mempool_t * pool, struct map_cb_data *data) { if (data->prev_data) { g_hash_table_destroy (data->prev_data); } if (data->cur_data) { msg_info_pool ("read hash of %d elements", g_hash_table_size (data->cur_data)); } } gchar * rspamd_kv_list_read (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map_cb_data *data) { if (data->cur_data == NULL) { data->cur_data = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); } return abstract_parse_kv_list (pool, chunk, len, data, (insert_func) g_hash_table_insert); } void rspamd_kv_list_fin (rspamd_mempool_t * pool, struct map_cb_data *data) { if (data->prev_data) { g_hash_table_destroy (data->prev_data); } if (data->cur_data) { msg_info_pool ("read hash of %d elements", g_hash_table_size (data->cur_data)); } } gchar * rspamd_radix_read (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map_cb_data *data) { radix_compressed_t *tree; rspamd_mempool_t *rpool; if (data->cur_data == NULL) { tree = radix_create_compressed (); rpool = radix_get_pool (tree); memcpy (rpool->tag.uid, pool->tag.uid, sizeof (rpool->tag.uid)); data->cur_data = tree; } return rspamd_parse_abstract_list (pool, chunk, len, data, (insert_func) radix_tree_insert_helper); } void rspamd_radix_fin (rspamd_mempool_t * pool, struct map_cb_data *data) { if (data->prev_data) { radix_destroy_compressed (data->prev_data); } if (data->cur_data) { msg_info_pool ("read radix trie of %z elements", radix_get_size (data->cur_data)); } } /47905/stable29 Nextcloud server, a safe home for all your data: https://github.com/nextcloud/serverwww-data
summaryrefslogtreecommitdiffstats
path: root/.jshintrc
blob: 0b055afde3d78bc03ca67136ec21b3ef79bc3bef (plain) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34