From 7a1aac9058a1b5f06d3212f1abf1eae701775a92 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Tue, 16 Feb 2016 17:05:24 +0000 Subject: [PATCH] Rewrite HTTP maps code, add signed maps support --- src/libutil/map.c | 417 ++++++++++++++++++++++++++------------ src/libutil/map_private.h | 3 +- 2 files changed, 288 insertions(+), 132 deletions(-) diff --git a/src/libutil/map.c b/src/libutil/map.c index 7b5f7c027..d8b55eeee 100644 --- a/src/libutil/map.c +++ b/src/libutil/map.c @@ -39,30 +39,173 @@ write_http_request (struct http_callback_data *cbd) msg = rspamd_http_new_message (HTTP_REQUEST); - msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path)); - if (cbd->data->last_checked != 0) { - tm = gmtime (&cbd->data->last_checked); - strftime (datebuf, sizeof (datebuf), "%a, %d %b %Y %H:%M:%S %Z", tm); + if (cbd->stage == map_load_file) { + msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path)); - rspamd_http_message_add_header (msg, "If-Modified-Since", datebuf); + if (cbd->data->last_checked != 0 && cbd->stage == map_load_file) { + tm = gmtime (&cbd->data->last_checked); + strftime (datebuf, sizeof (datebuf), "%a, %d %b %Y %H:%M:%S %Z", tm); + + rspamd_http_message_add_header (msg, "If-Modified-Since", datebuf); + } + } + else if (cbd->stage == map_load_pubkey) { + msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path)); + msg->url = rspamd_fstring_append (msg->url, ".pub", 4); + } + else if (cbd->stage == map_load_signature) { + msg->url = rspamd_fstring_new_init (cbd->data->path, strlen (cbd->data->path)); + msg->url = rspamd_fstring_append (msg->url, ".sig", 4); } rspamd_http_connection_write_message (cbd->conn, msg, cbd->data->host, NULL, cbd, cbd->fd, &cbd->tv, cbd->ev_base); } +static gboolean +rspamd_map_check_sig_pk (const char *fname, + struct rspamd_map *map, + const guchar *input, + gsize inlen, + struct rspamd_cryptobox_pubkey *pk) +{ + gchar fpath[PATH_MAX]; + rspamd_mempool_t *pool = map->pool; + guchar *data; + GString *b32_key; + gsize len = 0; + + /* Now load signature */ + rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", fname); + data = rspamd_file_xmap (fpath, PROT_READ, &len); + + if (data == NULL) { + msg_err_pool ("can't open signature %s: %s", fpath, strerror (errno)); + rspamd_pubkey_unref (pk); + return FALSE; + } + + if (len != rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519)) { + msg_err_pool ("can't open signature %s: invalid signature", fpath); + rspamd_pubkey_unref (pk); + munmap (data, len); + + return FALSE; + } + + if (!rspamd_cryptobox_verify (data, input, inlen, + rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519)) { + msg_err_pool ("can't verify signature %s: incorrect signature", fpath); + rspamd_pubkey_unref (pk); + munmap (data, len); + + return FALSE; + } + + b32_key = rspamd_pubkey_print (pk, + RSPAMD_KEYPAIR_BASE32|RSPAMD_KEYPAIR_PUBKEY); + msg_info_pool ("verified signature in file %s using trusted key %v", + fpath, b32_key); + g_string_free (b32_key, TRUE); + + rspamd_pubkey_unref (pk); + munmap (data, len); + + return TRUE; +} + +static gboolean +rspamd_map_check_file_sig (const char *fname, + struct rspamd_map *map, const guchar *input, + gsize inlen) +{ + gchar fpath[PATH_MAX]; + rspamd_mempool_t *pool = map->pool; + guchar *data; + struct rspamd_cryptobox_pubkey *pk = NULL; + GString *b32_key; + gsize len = 0; + + if (map->trusted_pubkey == NULL) { + /* Try to load and check pubkey */ + rspamd_snprintf (fpath, sizeof (fpath), "%s.pub", fname); + + data = rspamd_file_xmap (fpath, PROT_READ, &len); + + if (data == NULL) { + msg_err_pool ("can't open pubkey %s: %s", fpath, strerror (errno)); + return FALSE; + } + + pk = rspamd_pubkey_from_base32 (data, len, RSPAMD_KEYPAIR_SIGN, + RSPAMD_CRYPTOBOX_MODE_25519); + munmap (data, len); + + if (pk == NULL) { + msg_err_pool ("can't load pubkey %s", fpath); + return FALSE; + } + + /* We just check pk against the trusted db of keys */ + b32_key = rspamd_pubkey_print (pk, + RSPAMD_KEYPAIR_BASE32|RSPAMD_KEYPAIR_PUBKEY); + g_assert (b32_key != NULL); + + if (g_hash_table_lookup (map->cfg->trusted_keys, b32_key->str) == NULL) { + msg_err_pool ("pubkey loaded from %s is untrusted: %v", fpath, + b32_key); + g_string_free (b32_key, TRUE); + rspamd_pubkey_unref (pk); + + return FALSE; + } + + g_string_free (b32_key, TRUE); + } + else { + pk = rspamd_pubkey_ref (map->trusted_pubkey); + } + + return rspamd_map_check_sig_pk (fname, map, input, inlen, pk); +} + /** * Callback for destroying HTTP callback data */ static void free_http_cbdata (struct http_callback_data *cbd) { - if (cbd->remain_buf) { - g_string_free (cbd->remain_buf, TRUE); + char fpath[PATH_MAX]; + struct stat st; + + if (cbd->out_fd != -1) { + close (cbd->out_fd); + } + + rspamd_snprintf (fpath, sizeof (fpath), "%s", cbd->tmpfile); + if (stat (fpath, &st) != -1 && S_ISREG (st.st_mode)) { + (void)unlink (fpath); + } + + rspamd_snprintf (fpath, sizeof (fpath), "%s.pub", cbd->tmpfile); + if (stat (fpath, &st) != -1 && S_ISREG (st.st_mode)) { + (void)unlink (fpath); + } + + rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", cbd->tmpfile); + if (stat (fpath, &st) != -1 && S_ISREG (st.st_mode)) { + (void)unlink (fpath); + } + + if (cbd->pk) { + rspamd_pubkey_unref (cbd->pk); } rspamd_http_connection_free (cbd->conn); - close (cbd->fd); + if (cbd->fd != -1) { + close (cbd->fd); + } + g_slice_free1 (sizeof (struct http_callback_data), cbd); } @@ -90,24 +233,146 @@ http_map_finish (struct rspamd_http_connection *conn, struct http_callback_data *cbd = conn->ud; struct rspamd_map *map; rspamd_mempool_t *pool; + char fpath[PATH_MAX]; + guchar *aux_data, *in = NULL; + gsize inlen = 0; + struct stat st; map = cbd->map; pool = cbd->map->pool; if (msg->code == 200) { - if (cbd->remain_buf != NULL) { - /* Append \n to avoid issues */ - g_string_append_c (cbd->remain_buf, '\n'); - map->read_callback (map->pool, cbd->remain_buf->str, - cbd->remain_buf->len, &cbd->cbdata); + + if (cbd->stage == map_load_file) { + /* Maybe we need to check signature ? */ + if (map->is_signed) { + close (cbd->out_fd); + + if (map->trusted_pubkey) { + /* No need to load key */ + cbd->stage = map_load_signature; + cbd->pk = rspamd_pubkey_ref (map->trusted_pubkey); + rspamd_snprintf (fpath, sizeof (fpath), "%s.sig"); + } + else { + rspamd_snprintf (fpath, sizeof (fpath), "%s.pub"); + cbd->stage = map_load_pubkey; + } + + cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644); + + if (cbd->out_fd == -1) { + msg_err_pool ("cannot open pubkey file %s for writing: %s", + fpath, strerror (errno)); + free_http_cbdata (cbd); + + return 0; + } + + rspamd_http_connection_reset (cbd->conn); + write_http_request (cbd); + + return 0; + } + else { + /* Unsinged version - just open file */ + in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen); + + if (in == NULL) { + msg_err_pool ("cannot read tempfile %s: %s", cbd->tmpfile, + strerror (errno)); + free_http_cbdata (cbd); + + return 0; + } + } } + else if (cbd->stage == map_load_pubkey) { + /* We now can load pubkey */ + (void)lseek (cbd->out_fd, 0, SEEK_SET); + + if (fstat (cbd->out_fd, &st) == -1) { + msg_err_pool ("cannot stat pubkey file %s: %s", + fpath, strerror (errno)); + free_http_cbdata (cbd); + + return 0; + } + + aux_data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, + cbd->out_fd, 0); + close (cbd->out_fd); + cbd->out_fd = -1; + + if (aux_data == MAP_FAILED) { + msg_err_pool ("cannot map pubkey file %s: %s", + fpath, strerror (errno)); + free_http_cbdata (cbd); + + return 0; + } + cbd->pk = rspamd_pubkey_from_base32 (aux_data, st.st_size, + RSPAMD_KEYPAIR_SIGN, RSPAMD_CRYPTOBOX_MODE_25519); + munmap (aux_data, st.st_size); + + if (cbd->pk == NULL) { + msg_err_pool ("cannot load pubkey file %s: bad pubkey", + fpath); + free_http_cbdata (cbd); + + return 0; + } + + rspamd_snprintf (fpath, sizeof (fpath), "%s.sig"); + cbd->out_fd = rspamd_file_xopen (fpath, O_RDWR|O_CREAT, 00644); + + if (cbd->out_fd == -1) { + msg_err_pool ("cannot open signature file %s for writing: %s", + fpath, strerror (errno)); + free_http_cbdata (cbd); + + return 0; + } + + cbd->stage = map_load_signature; + rspamd_http_connection_reset (cbd->conn); + write_http_request (cbd); + + return 0; + } + else if (cbd->stage == map_load_signature) { + /* We can now check signature */ + close (cbd->out_fd); + cbd->out_fd = -1; + + in = rspamd_file_xmap (cbd->tmpfile, PROT_READ, &inlen); + + if (in == NULL) { + msg_err_pool ("cannot read tempfile %s: %s", cbd->tmpfile, + strerror (errno)); + free_http_cbdata (cbd); + + return 0; + } + + if (!rspamd_map_check_sig_pk (cbd->tmpfile, map, in, inlen, cbd->pk)) { + free_http_cbdata (cbd); + + return 0; + } + } + + g_assert (in != NULL); + + map->read_callback (map->pool, in, inlen, &cbd->cbdata); map->fin_callback (map->pool, &cbd->cbdata); + *map->user_data = cbd->cbdata.cur_data; cbd->data->last_checked = msg->date; msg_info_pool ("read map data from %s", cbd->data->host); } - else if (msg->code == 304) { + else if (msg->code == 304 && cbd->stage == map_load_file) { msg_debug_pool ("data is not modified for server %s", cbd->data->host); cbd->data->last_checked = msg->date; @@ -129,135 +394,25 @@ http_map_read (struct rspamd_http_connection *conn, gsize len) { struct http_callback_data *cbd = conn->ud; - gchar *pos; - struct rspamd_map *map; + rspamd_mempool_t *pool; if (msg->code != 200 || len == 0) { /* Ignore not full replies */ return 0; } - map = cbd->map; - if (cbd->remain_buf != NULL) { - /* We need to concatenate incoming buf with the remaining buf */ - g_string_append_len (cbd->remain_buf, chunk, len); + pool = cbd->map->pool; - pos = map->read_callback (map->pool, cbd->remain_buf->str, - cbd->remain_buf->len, &cbd->cbdata); + if (write (cbd->out_fd, chunk, len) == -1) { + msg_err_pool ("cannot write to %s: %s", cbd->tmpfile, strerror (errno)); + free_http_cbdata (cbd); - /* All read */ - if (pos == NULL) { - g_string_free (cbd->remain_buf, TRUE); - cbd->remain_buf = NULL; - } - else { - /* Need to erase data processed */ - g_string_erase (cbd->remain_buf, 0, pos - cbd->remain_buf->str); - } - } - else { - pos = map->read_callback (map->pool, (gchar *)chunk, len, &cbd->cbdata); - - if (pos != NULL) { - /* Store data in remain buf */ - cbd->remain_buf = g_string_new_len (pos, len - (pos - chunk)); - } + return -1; } return 0; } -static gboolean -rspamd_map_check_file_sig (const char *fname, - struct rspamd_map *map, const guchar *input, - gsize inlen) -{ - gchar fpath[PATH_MAX]; - rspamd_mempool_t *pool = map->pool; - guchar *data; - struct rspamd_cryptobox_pubkey *pk = NULL; - GString *b32_key; - gsize len = 0; - - if (map->trusted_pubkey == NULL) { - /* Try to load and check pubkey */ - rspamd_snprintf (fpath, sizeof (fpath), "%s.pub", fname); - - data = rspamd_file_xmap (fpath, PROT_READ, &len); - - if (data == NULL) { - msg_err_pool ("can't open pubkey %s: %s", fpath, strerror (errno)); - return FALSE; - } - - pk = rspamd_pubkey_from_base32 (data, len, RSPAMD_KEYPAIR_SIGN, - RSPAMD_CRYPTOBOX_MODE_25519); - munmap (data, len); - - if (pk == NULL) { - msg_err_pool ("can't load pubkey %s", fpath); - return FALSE; - } - - /* We just check pk against the trusted db of keys */ - b32_key = rspamd_pubkey_print (pk, - RSPAMD_KEYPAIR_BASE32|RSPAMD_KEYPAIR_PUBKEY); - g_assert (b32_key != NULL); - - if (g_hash_table_lookup (map->cfg->trusted_keys, b32_key->str) == NULL) { - msg_err_pool ("pubkey loaded from %s is untrusted: %v", fpath, - b32_key); - g_string_free (b32_key, TRUE); - rspamd_pubkey_unref (pk); - - return FALSE; - } - - g_string_free (b32_key, TRUE); - } - else { - pk = rspamd_pubkey_ref (map->trusted_pubkey); - } - - /* Now load signature */ - rspamd_snprintf (fpath, sizeof (fpath), "%s.sig", fname); - data = rspamd_file_xmap (fpath, PROT_READ, &len); - - if (data == NULL) { - msg_err_pool ("can't open signature %s: %s", fpath, strerror (errno)); - rspamd_pubkey_unref (pk); - return FALSE; - } - - if (len != rspamd_cryptobox_signature_bytes (RSPAMD_CRYPTOBOX_MODE_25519)) { - msg_err_pool ("can't open signature %s: invalid signature", fpath); - rspamd_pubkey_unref (pk); - munmap (data, len); - - return FALSE; - } - - if (!rspamd_cryptobox_verify (data, input, inlen, - rspamd_pubkey_get_pk (pk, NULL), RSPAMD_CRYPTOBOX_MODE_25519)) { - msg_err_pool ("can't verify signature %s: incorrect signature", fpath); - rspamd_pubkey_unref (pk); - munmap (data, len); - - return FALSE; - } - - b32_key = rspamd_pubkey_print (pk, - RSPAMD_KEYPAIR_BASE32|RSPAMD_KEYPAIR_PUBKEY); - msg_info_pool ("verified signature in file %s using trusted key %v", - fpath, b32_key); - g_string_free (b32_key, TRUE); - - rspamd_pubkey_unref (pk); - munmap (data, len); - - return TRUE; -} - /** * Callback for reading data from file */ @@ -441,7 +596,7 @@ http_callback (gint fd, short what, void *ud) cbd->ev_base = map->ev_base; cbd->map = map; cbd->data = data; - cbd->remain_buf = NULL; + cbd->fd = -1; cbd->cbdata.state = 0; cbd->cbdata.prev_data = *cbd->map->user_data; cbd->cbdata.cur_data = NULL; diff --git a/src/libutil/map_private.h b/src/libutil/map_private.h index 16967c7d1..10f94c3ea 100644 --- a/src/libutil/map_private.h +++ b/src/libutil/map_private.h @@ -76,7 +76,8 @@ struct http_callback_data { struct rspamd_map *map; struct http_map_data *data; struct map_cb_data cbdata; - GString *remain_buf; + struct rspamd_cryptobox_pubkey *pk; + enum { map_resolve_host2 = 0, /* 2 requests sent */ map_resolve_host1, /* 1 requests sent */ -- 2.39.5