diff options
Diffstat (limited to 'src/libserver')
38 files changed, 5722 insertions, 3527 deletions
diff --git a/src/libserver/binlog.c b/src/libserver/binlog.c index ec191eac7..ee0bf86bc 100644 --- a/src/libserver/binlog.c +++ b/src/libserver/binlog.c @@ -22,9 +22,9 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "config.h" #include "binlog.h" #include "cfg_file.h" +#include "config.h" #include "tokenizers/tokenizers.h" #define BINLOG_SUFFIX ".binlog" @@ -35,7 +35,7 @@ static GHashTable *binlog_opened = NULL; static rspamd_mempool_t *binlog_pool = NULL; -static gboolean +static gboolean binlog_write_header (struct rspamd_binlog *log) { struct rspamd_binlog_header header = { @@ -43,27 +43,35 @@ binlog_write_header (struct rspamd_binlog *log) .version = VALID_VERSION, .padding = { '\0', '\0' }, }; - + header.create_time = time (NULL); lock_file (log->fd, FALSE); - + if (write (log->fd, &header, sizeof (struct rspamd_binlog_header)) == -1) { - msg_warn ("cannot write file %s, error %d, %s", log->filename, errno, strerror (errno)); + msg_warn ("cannot write file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); return FALSE; } memcpy (&log->header, &header, sizeof (struct rspamd_binlog_header)); - + /* Metaindex */ log->metaindex = g_malloc (sizeof (struct rspamd_binlog_metaindex)); bzero (log->metaindex, sizeof (struct rspamd_binlog_metaindex)); /* Offset to metaindex */ - log->metaindex->indexes[0] = sizeof (struct rspamd_binlog_metaindex) + sizeof (struct rspamd_binlog_header); + log->metaindex->indexes[0] = sizeof (struct rspamd_binlog_metaindex) + + sizeof (struct rspamd_binlog_header); - if (write (log->fd, log->metaindex, sizeof (struct rspamd_binlog_metaindex)) == -1) { + if (write (log->fd, log->metaindex, + sizeof (struct rspamd_binlog_metaindex)) == -1) { g_free (log->metaindex); - msg_warn ("cannot write file %s, error %d, %s", log->filename, errno, strerror (errno)); + msg_warn ("cannot write file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); unlock_file (log->fd, FALSE); return FALSE; } @@ -71,9 +79,13 @@ binlog_write_header (struct rspamd_binlog *log) /* Alloc, write, mmap */ log->cur_idx = g_malloc (sizeof (struct rspamd_index_block)); bzero (log->cur_idx, sizeof (struct rspamd_index_block)); - if (write (log->fd, log->cur_idx, sizeof (struct rspamd_index_block)) == -1) { + if (write (log->fd, log->cur_idx, + sizeof (struct rspamd_index_block)) == -1) { g_free (log->cur_idx); - msg_warn ("cannot write file %s, error %d, %s", log->filename, errno, strerror (errno)); + msg_warn ("cannot write file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); unlock_file (log->fd, FALSE); return FALSE; } @@ -88,14 +100,20 @@ binlog_check_file (struct rspamd_binlog *log) { static gchar valid_magic[] = VALID_MAGIC, valid_version[] = VALID_VERSION; - if (read (log->fd, &log->header, sizeof (struct rspamd_binlog_header)) != sizeof (struct rspamd_binlog_header)) { - msg_warn ("cannot read file %s, error %d, %s", log->filename, errno, strerror (errno)); + if (read (log->fd, &log->header, + sizeof (struct rspamd_binlog_header)) != + sizeof (struct rspamd_binlog_header)) { + msg_warn ("cannot read file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); return FALSE; } /* Now check all fields */ - if (memcmp (&log->header.magic, valid_magic, sizeof (valid_magic)) != 0 || - memcmp (&log->header.version, valid_version, sizeof (valid_version)) != 0) { + if (memcmp (&log->header.magic, valid_magic, sizeof (valid_magic)) != 0 || + memcmp (&log->header.version, valid_version, + sizeof (valid_version)) != 0) { msg_warn ("cannot validate file %s"); return FALSE; } @@ -103,24 +121,38 @@ binlog_check_file (struct rspamd_binlog *log) if (log->metaindex == NULL) { log->metaindex = g_malloc (sizeof (struct rspamd_binlog_metaindex)); } - if ((read (log->fd, log->metaindex, sizeof (struct rspamd_binlog_metaindex))) != sizeof (struct rspamd_binlog_metaindex)) { - msg_warn ("cannot read metaindex of file %s, error %d, %s", log->filename, errno, strerror (errno)); + if ((read (log->fd, log->metaindex, + sizeof (struct rspamd_binlog_metaindex))) != + sizeof (struct rspamd_binlog_metaindex)) { + msg_warn ("cannot read metaindex of file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); return FALSE; } /* Current index */ if (log->cur_idx == NULL) { log->cur_idx = g_malloc (sizeof (struct rspamd_index_block)); } - if (lseek (log->fd, log->metaindex->indexes[log->metaindex->last_index], SEEK_SET) == -1) { - msg_info ("cannot seek in file: %s, error: %s", log->filename, strerror (errno)); + if (lseek (log->fd, log->metaindex->indexes[log->metaindex->last_index], + SEEK_SET) == -1) { + msg_info ("cannot seek in file: %s, error: %s", + log->filename, + strerror (errno)); return FALSE; } - if ((read (log->fd, log->cur_idx, sizeof (struct rspamd_index_block))) != sizeof (struct rspamd_index_block)) { - msg_warn ("cannot read index in file %s, error %d, %s", log->filename, errno, strerror (errno)); + if ((read (log->fd, log->cur_idx, + sizeof (struct rspamd_index_block))) != + sizeof (struct rspamd_index_block)) { + msg_warn ("cannot read index in file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); return FALSE; } - log->cur_seq = log->metaindex->last_index * BINLOG_IDX_LEN + log->cur_idx->last_index; + log->cur_seq = log->metaindex->last_index * BINLOG_IDX_LEN + + log->cur_idx->last_index; log->cur_time = log->cur_idx->indexes[log->cur_idx->last_index].time; return TRUE; @@ -130,8 +162,13 @@ binlog_check_file (struct rspamd_binlog *log) static gboolean binlog_create (struct rspamd_binlog *log) { - if ((log->fd = open (log->filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { - msg_info ("cannot create file %s, error %d, %s", log->filename, errno, strerror (errno)); + if ((log->fd = + open (log->filename, O_RDWR | O_TRUNC | O_CREAT, + S_IWUSR | S_IRUSR)) == -1) { + msg_info ("cannot create file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); return FALSE; } @@ -142,19 +179,25 @@ static gboolean binlog_open_real (struct rspamd_binlog *log) { if ((log->fd = open (log->filename, O_RDWR)) == -1) { - msg_info ("cannot open file %s, error %d, %s", log->filename, errno, strerror (errno)); + msg_info ("cannot open file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); return FALSE; } - + return binlog_check_file (log); } -struct rspamd_binlog* -binlog_open (rspamd_mempool_t *pool, const gchar *path, time_t rotate_time, gint rotate_jitter) +struct rspamd_binlog * +binlog_open (rspamd_mempool_t *pool, + const gchar *path, + time_t rotate_time, + gint rotate_jitter) { struct rspamd_binlog *new; - gint len = strlen (path); + gint len = strlen (path); struct stat st; new = rspamd_mempool_alloc0 (pool, sizeof (struct rspamd_binlog)); @@ -165,15 +208,16 @@ binlog_open (rspamd_mempool_t *pool, const gchar *path, time_t rotate_time, gint if (rotate_time) { new->rotate_jitter = g_random_int_range (0, rotate_jitter); } - + new->filename = rspamd_mempool_alloc (pool, len + sizeof (BINLOG_SUFFIX)); - rspamd_strlcpy (new->filename, path, len + 1); + rspamd_strlcpy (new->filename, path, len + 1); rspamd_strlcpy (new->filename + len, BINLOG_SUFFIX, sizeof (BINLOG_SUFFIX)); if (stat (new->filename, &st) == -1) { /* Check errno to check whether we should create this file */ if (errno != ENOENT) { - msg_err ("cannot stat file: %s, error %s", new->filename, strerror (errno)); + msg_err ("cannot stat file: %s, error %s", new->filename, + strerror (errno)); return NULL; } else { @@ -193,7 +237,7 @@ binlog_open (rspamd_mempool_t *pool, const gchar *path, time_t rotate_time, gint return new; } -void +void binlog_close (struct rspamd_binlog *log) { if (log) { @@ -210,16 +254,18 @@ binlog_close (struct rspamd_binlog *log) static gboolean binlog_tree_callback (gpointer key, gpointer value, gpointer data) { - token_node_t *node = key; - struct rspamd_binlog *log = data; - struct rspamd_binlog_element elt; + token_node_t *node = key; + struct rspamd_binlog *log = data; + struct rspamd_binlog_element elt; elt.h1 = node->h1; elt.h2 = node->h2; elt.value = node->value; - + if (write (log->fd, &elt, sizeof (elt)) == -1) { - msg_info ("cannot write token to file: %s, error: %s", log->filename, strerror (errno)); + msg_info ("cannot write token to file: %s, error: %s", + log->filename, + strerror (errno)); return TRUE; } @@ -233,12 +279,14 @@ write_binlog_tree (struct rspamd_binlog *log, GTree *nodes) struct rspamd_binlog_index *idx; lock_file (log->fd, FALSE); - log->cur_seq ++; + log->cur_seq++; /* Seek to end of file */ if ((seek = lseek (log->fd, 0, SEEK_END)) == -1) { unlock_file (log->fd, FALSE); - msg_info ("cannot seek in file: %s, error: %s", log->filename, strerror (errno)); + msg_info ("cannot seek in file: %s, error: %s", + log->filename, + strerror (errno)); return FALSE; } @@ -251,21 +299,28 @@ write_binlog_tree (struct rspamd_binlog *log, GTree *nodes) idx->time = (guint64)time (NULL); log->cur_time = idx->time; idx->len = g_tree_nnodes (nodes) * sizeof (struct rspamd_binlog_element); - if (lseek (log->fd, log->metaindex->indexes[log->metaindex->last_index], SEEK_SET) == -1) { + if (lseek (log->fd, log->metaindex->indexes[log->metaindex->last_index], + SEEK_SET) == -1) { unlock_file (log->fd, FALSE); - msg_info ("cannot seek in file: %s, error: %s, seek: %L, op: insert index", log->filename, - strerror (errno), log->metaindex->indexes[log->metaindex->last_index]); + msg_info ( + "cannot seek in file: %s, error: %s, seek: %L, op: insert index", + log->filename, + strerror (errno), + log->metaindex->indexes[log->metaindex->last_index]); return FALSE; } - log->cur_idx->last_index ++; - if (write (log->fd, log->cur_idx, sizeof (struct rspamd_index_block)) == -1) { + log->cur_idx->last_index++; + if (write (log->fd, log->cur_idx, + sizeof (struct rspamd_index_block)) == -1) { unlock_file (log->fd, FALSE); - msg_info ("cannot write index to file: %s, error: %s", log->filename, strerror (errno)); + msg_info ("cannot write index to file: %s, error: %s", + log->filename, + strerror (errno)); return FALSE; } unlock_file (log->fd, FALSE); - + return TRUE; } @@ -275,18 +330,24 @@ create_new_metaindex_block (struct rspamd_binlog *log) off_t seek; lock_file (log->fd, FALSE); - - log->metaindex->last_index ++; + + log->metaindex->last_index++; /* Seek to end of file */ if ((seek = lseek (log->fd, 0, SEEK_END)) == -1) { unlock_file (log->fd, FALSE); - msg_info ("cannot seek in file: %s, error: %s", log->filename, strerror (errno)); + msg_info ("cannot seek in file: %s, error: %s", + log->filename, + strerror (errno)); return FALSE; } - if (write (log->fd, log->cur_idx, sizeof (struct rspamd_index_block)) == -1) { + if (write (log->fd, log->cur_idx, + sizeof (struct rspamd_index_block)) == -1) { unlock_file (log->fd, FALSE); g_free (log->cur_idx); - msg_warn ("cannot write file %s, error %d, %s", log->filename, errno, strerror (errno)); + msg_warn ("cannot write file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); return FALSE; } /* Offset to metaindex */ @@ -294,12 +355,17 @@ create_new_metaindex_block (struct rspamd_binlog *log) /* Overwrite all metaindexes */ if (lseek (log->fd, sizeof (struct rspamd_binlog_header), SEEK_SET) == -1) { unlock_file (log->fd, FALSE); - msg_info ("cannot seek in file: %s, error: %s", log->filename, strerror (errno)); + msg_info ("cannot seek in file: %s, error: %s", + log->filename, + strerror (errno)); return FALSE; } - if (write (log->fd, log->metaindex, sizeof (struct rspamd_binlog_metaindex)) == -1) { + if (write (log->fd, log->metaindex, + sizeof (struct rspamd_binlog_metaindex)) == -1) { unlock_file (log->fd, FALSE); - msg_info ("cannot write metaindex in file: %s, error: %s", log->filename, strerror (errno)); + msg_info ("cannot write metaindex in file: %s, error: %s", + log->filename, + strerror (errno)); return FALSE; } bzero (log->cur_idx, sizeof (struct rspamd_index_block)); @@ -311,9 +377,11 @@ create_new_metaindex_block (struct rspamd_binlog *log) static gboolean maybe_rotate_binlog (struct rspamd_binlog *log) { - guint64 now = time (NULL); + guint64 now = time (NULL); - if (log->rotate_time && ((now - log->header.create_time) > (guint)(log->rotate_time + log->rotate_jitter))) { + if (log->rotate_time && + ((now - log->header.create_time) > + (guint)(log->rotate_time + log->rotate_jitter))) { return TRUE; } return FALSE; @@ -322,7 +390,7 @@ maybe_rotate_binlog (struct rspamd_binlog *log) static gboolean rotate_binlog (struct rspamd_binlog *log) { - gchar *backup_name; + gchar *backup_name; struct stat st; lock_file (log->fd, FALSE); @@ -355,13 +423,14 @@ rotate_binlog (struct rspamd_binlog *log) } -gboolean +gboolean binlog_insert (struct rspamd_binlog *log, GTree *nodes) { off_t seek; if (!log || !log->metaindex || !log->cur_idx || !nodes) { - msg_info ("improperly opened binlog: %s", log != NULL ? log->filename : "unknown"); + msg_info ("improperly opened binlog: %s", + log != NULL ? log->filename : "unknown"); return FALSE; } @@ -381,7 +450,9 @@ binlog_insert (struct rspamd_binlog *log, GTree *nodes) if (log->metaindex->last_index < METAINDEX_LEN) { /* Create new index block */ if ((seek = lseek (log->fd, 0, SEEK_END)) == (off_t)-1) { - msg_info ("cannot seek in file: %s, error: %s", log->filename, strerror (errno)); + msg_info ("cannot seek in file: %s, error: %s", + log->filename, + strerror (errno)); return FALSE; } if (!create_new_metaindex_block (log)) { @@ -389,7 +460,7 @@ binlog_insert (struct rspamd_binlog *log, GTree *nodes) } return write_binlog_tree (log, nodes); } - + /* All binlog is filled, we need to rotate it forcefully */ if (!rotate_binlog (log)) { return FALSE; @@ -398,16 +469,20 @@ binlog_insert (struct rspamd_binlog *log, GTree *nodes) return write_binlog_tree (log, nodes); } -gboolean -binlog_sync (struct rspamd_binlog *log, guint64 from_rev, guint64 *from_time, GByteArray **rep) +gboolean +binlog_sync (struct rspamd_binlog *log, + guint64 from_rev, + guint64 *from_time, + GByteArray **rep) { - guint32 metaindex_num; + guint32 metaindex_num; struct rspamd_index_block *idxb; struct rspamd_binlog_index *idx; gboolean idx_mapped = FALSE, res = TRUE, is_first = FALSE; if (!log || !log->metaindex || !log->cur_idx) { - msg_info ("improperly opened binlog: %s", log != NULL ? log->filename : "unknown"); + msg_info ("improperly opened binlog: %s", + log != NULL ? log->filename : "unknown"); return FALSE; } @@ -419,7 +494,7 @@ binlog_sync (struct rspamd_binlog *log, guint64 from_rev, guint64 *from_time, GB /* Unmap old fragment */ g_free ((*rep)->data); } - + if (from_rev == log->cur_seq) { /* Last record */ *rep = NULL; @@ -427,7 +502,11 @@ binlog_sync (struct rspamd_binlog *log, guint64 from_rev, guint64 *from_time, GB } else if (from_rev > log->cur_seq) { /* Slave has more actual copy, write this to log and abort sync */ - msg_warn ("slave has more recent revision of statfile %s: %uL and our is: %uL", log->filename, from_rev, log->cur_seq); + msg_warn ( + "slave has more recent revision of statfile %s: %uL and our is: %uL", + log->filename, + from_rev, + log->cur_seq); *rep = NULL; *from_time = 0; return FALSE; @@ -443,15 +522,24 @@ binlog_sync (struct rspamd_binlog *log, guint64 from_rev, guint64 *from_time, GB lock_file (log->fd, FALSE); idxb = g_malloc (sizeof (struct rspamd_index_block)); idx_mapped = TRUE; - if (lseek (log->fd, log->metaindex->indexes[metaindex_num], SEEK_SET) == -1) { + if (lseek (log->fd, log->metaindex->indexes[metaindex_num], + SEEK_SET) == -1) { unlock_file (log->fd, FALSE); - msg_warn ("cannot seek file %s, error %d, %s", log->filename, errno, strerror (errno)); + msg_warn ("cannot seek file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); res = FALSE; goto end; } - if ((read (log->fd, idxb, sizeof (struct rspamd_index_block))) != sizeof (struct rspamd_index_block)) { + if ((read (log->fd, idxb, + sizeof (struct rspamd_index_block))) != + sizeof (struct rspamd_index_block)) { unlock_file (log->fd, FALSE); - msg_warn ("cannot read index from file %s, error %d, %s", log->filename, errno, strerror (errno)); + msg_warn ("cannot read index from file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); res = FALSE; goto end; } @@ -474,17 +562,27 @@ binlog_sync (struct rspamd_binlog *log, guint64 from_rev, guint64 *from_time, GB /* Now fill reply structure */ (*rep)->len = idx->len; /* Read result */ - msg_info ("update from binlog '%s' from revision: %uL to revision %uL size is %uL", - log->filename, from_rev, log->cur_seq, idx->len); + msg_info ( + "update from binlog '%s' from revision: %uL to revision %uL size is %uL", + log->filename, + from_rev, + log->cur_seq, + idx->len); if (lseek (log->fd, idx->seek, SEEK_SET) == -1) { - msg_warn ("cannot seek file %s, error %d, %s", log->filename, errno, strerror (errno)); + msg_warn ("cannot seek file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); res = FALSE; goto end; } - + (*rep)->data = g_malloc (idx->len); if ((read (log->fd, (*rep)->data, idx->len)) != (ssize_t)idx->len) { - msg_warn ("cannot read file %s, error %d, %s", log->filename, errno, strerror (errno)); + msg_warn ("cannot read file %s, error %d, %s", + log->filename, + errno, + strerror (errno)); res = FALSE; goto end; } @@ -517,8 +615,11 @@ maybe_init_static (void) return TRUE; } -gboolean -maybe_write_binlog (struct rspamd_classifier_config *ccf, struct rspamd_statfile_config *st, stat_file_t *file, GTree *nodes) +gboolean +maybe_write_binlog (struct rspamd_classifier_config *ccf, + struct rspamd_statfile_config *st, + stat_file_t *file, + GTree *nodes) { struct rspamd_binlog *log; @@ -526,8 +627,9 @@ maybe_write_binlog (struct rspamd_classifier_config *ccf, struct rspamd_statfile return FALSE; } - - if (st == NULL || nodes == NULL || st->binlog == NULL || st->binlog->affinity != AFFINITY_MASTER) { + + if (st == NULL || nodes == NULL || st->binlog == NULL || + st->binlog->affinity != AFFINITY_MASTER) { return FALSE; } @@ -536,7 +638,9 @@ maybe_write_binlog (struct rspamd_classifier_config *ccf, struct rspamd_statfile } if ((log = g_hash_table_lookup (binlog_opened, st)) == NULL) { - if ((log = binlog_open (binlog_pool, st->path, st->binlog->rotate_time, st->binlog->rotate_time / 2)) != NULL) { + if ((log = + binlog_open (binlog_pool, st->path, st->binlog->rotate_time, + st->binlog->rotate_time / 2)) != NULL) { g_hash_table_insert (binlog_opened, st, log); } else { @@ -545,7 +649,9 @@ maybe_write_binlog (struct rspamd_classifier_config *ccf, struct rspamd_statfile } if (binlog_insert (log, nodes)) { - msg_info ("set new revision of statfile %s: %uL", st->symbol, log->cur_seq); + msg_info ("set new revision of statfile %s: %uL", + st->symbol, + log->cur_seq); (void)statfile_set_revision (file, log->cur_seq, log->cur_time); return TRUE; } @@ -553,12 +659,13 @@ maybe_write_binlog (struct rspamd_classifier_config *ccf, struct rspamd_statfile return FALSE; } -struct rspamd_binlog* +struct rspamd_binlog * get_binlog_by_statfile (struct rspamd_statfile_config *st) { struct rspamd_binlog *log; - if (st == NULL || st->binlog == NULL || st->binlog->affinity != AFFINITY_MASTER) { + if (st == NULL || st->binlog == NULL || st->binlog->affinity != + AFFINITY_MASTER) { return NULL; } @@ -567,13 +674,15 @@ get_binlog_by_statfile (struct rspamd_statfile_config *st) } if ((log = g_hash_table_lookup (binlog_opened, st)) == NULL) { - if ((log = binlog_open (binlog_pool, st->path, st->binlog->rotate_time, st->binlog->rotate_time / 2)) != NULL) { + if ((log = + binlog_open (binlog_pool, st->path, st->binlog->rotate_time, + st->binlog->rotate_time / 2)) != NULL) { g_hash_table_insert (binlog_opened, st, log); } else { return NULL; } } - + return log; } diff --git a/src/libserver/binlog.h b/src/libserver/binlog.h index 6b77a7613..c18a201f2 100644 --- a/src/libserver/binlog.h +++ b/src/libserver/binlog.h @@ -58,12 +58,15 @@ struct rspamd_classifier_config; /* * Open binlog at specified path with specified rotate params */ -struct rspamd_binlog* binlog_open (rspamd_mempool_t *pool, const gchar *path, time_t rotate_time, gint rotate_jitter); +struct rspamd_binlog * binlog_open (rspamd_mempool_t *pool, + const gchar *path, + time_t rotate_time, + gint rotate_jitter); /* * Get and open binlog for specified statfile */ -struct rspamd_binlog* get_binlog_by_statfile (struct rspamd_statfile_config *st); +struct rspamd_binlog * get_binlog_by_statfile (struct rspamd_statfile_config *st); /* * Close binlog @@ -83,11 +86,17 @@ gboolean binlog_insert (struct rspamd_binlog *log, GTree *nodes); * @param rep a portion of changes for revision is stored here * @return TRUE if there are more revisions to get and FALSE if synchronization is complete */ -gboolean binlog_sync (struct rspamd_binlog *log, guint64 from_rev, guint64 *from_time, GByteArray **rep); +gboolean binlog_sync (struct rspamd_binlog *log, + guint64 from_rev, + guint64 *from_time, + GByteArray **rep); /* * Conditional write to a binlog for specified statfile */ -gboolean maybe_write_binlog (struct rspamd_classifier_config *ccf, struct rspamd_statfile_config *st, stat_file_t *file, GTree *nodes); +gboolean maybe_write_binlog (struct rspamd_classifier_config *ccf, + struct rspamd_statfile_config *st, + stat_file_t *file, + GTree *nodes); #endif diff --git a/src/libserver/buffer.c b/src/libserver/buffer.c index 864f2fad6..4f684caed 100644 --- a/src/libserver/buffer.c +++ b/src/libserver/buffer.c @@ -22,19 +22,22 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "config.h" #include "buffer.h" +#include "config.h" #include "main.h" #ifdef HAVE_SYS_SENDFILE_H #include <sys/sendfile.h> #endif -#define G_DISPATCHER_ERROR dispatcher_error_quark() -#define debug_ip(...) rspamd_conditional_debug(rspamd_main->logger, NULL, __FUNCTION__, __VA_ARGS__) +#define G_DISPATCHER_ERROR dispatcher_error_quark () +#define debug_ip(...) rspamd_conditional_debug (rspamd_main->logger, \ + NULL, \ + __FUNCTION__, \ + __VA_ARGS__) -static void dispatcher_cb (gint fd, short what, void *arg); +static void dispatcher_cb (gint fd, short what, void *arg); -static inline GQuark +static inline GQuark dispatcher_error_quark (void) { return g_quark_from_static_string ("g-dispatcher-error-quark"); @@ -44,11 +47,11 @@ static gboolean sendfile_callback (rspamd_io_dispatcher_t *d) { - GError *err; + GError *err; #ifdef HAVE_SENDFILE # if defined(FREEBSD) || defined(DARWIN) - off_t off = 0; + off_t off = 0; #if defined(FREEBSD) /* FreeBSD version */ if (sendfile (d->sendfile_fd, d->fd, d->offset, 0, NULL, &off, 0) != 0) { @@ -58,13 +61,15 @@ sendfile_callback (rspamd_io_dispatcher_t *d) #endif if (errno != EAGAIN) { if (d->err_callback) { - err = g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror (errno)); + err = + g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror ( + errno)); d->err_callback (err, d->user_data); return FALSE; } } else { - debug_ip("partially write data, retry"); + debug_ip ("partially write data, retry"); /* Wait for other event */ d->offset += off; event_del (d->ev); @@ -76,30 +81,33 @@ sendfile_callback (rspamd_io_dispatcher_t *d) else { if (d->write_callback) { if (!d->write_callback (d->user_data)) { - debug_ip("callback set wanna_die flag, terminating"); + debug_ip ("callback set wanna_die flag, terminating"); return FALSE; } } event_del (d->ev); - event_set (d->ev, d->fd, EV_READ | EV_PERSIST, dispatcher_cb, (void *)d); + event_set (d->ev, d->fd, EV_READ | EV_PERSIST, dispatcher_cb, + (void *)d); event_base_set (d->ev_base, d->ev); event_add (d->ev, d->tv); d->in_sendfile = FALSE; } # else - ssize_t r; + ssize_t r; /* Linux version */ r = sendfile (d->fd, d->sendfile_fd, &d->offset, d->file_size); if (r == -1) { if (errno != EAGAIN) { if (d->err_callback) { - err = g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror (errno)); + err = + g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror ( + errno)); d->err_callback (err, d->user_data); return FALSE; } } else { - debug_ip("partially write data, retry"); + debug_ip ("partially write data, retry"); /* Wait for other event */ event_del (d->ev); event_set (d->ev, d->fd, EV_WRITE, dispatcher_cb, (void *)d); @@ -108,7 +116,7 @@ sendfile_callback (rspamd_io_dispatcher_t *d) } } else if (r + d->offset < (ssize_t)d->file_size) { - debug_ip("partially write data, retry"); + debug_ip ("partially write data, retry"); /* Wait for other event */ event_del (d->ev); event_set (d->ev, d->fd, EV_WRITE, dispatcher_cb, (void *)d); @@ -118,30 +126,33 @@ sendfile_callback (rspamd_io_dispatcher_t *d) else { if (d->write_callback) { if (!d->write_callback (d->user_data)) { - debug_ip("callback set wanna_die flag, terminating"); + debug_ip ("callback set wanna_die flag, terminating"); return FALSE; } } event_del (d->ev); - event_set (d->ev, d->fd, EV_READ | EV_PERSIST, dispatcher_cb, (void *)d); + event_set (d->ev, d->fd, EV_READ | EV_PERSIST, dispatcher_cb, + (void *)d); event_base_set (d->ev_base, d->ev); event_add (d->ev, d->tv); d->in_sendfile = FALSE; } # endif #else - ssize_t r; + ssize_t r; r = write (d->fd, d->map, d->file_size - d->offset); if (r == -1) { if (errno != EAGAIN) { if (d->err_callback) { - err = g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror (errno)); + err = + g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror ( + errno)); d->err_callback (err, d->user_data); return FALSE; } } else { - debug_ip("partially write data, retry"); + debug_ip ("partially write data, retry"); /* Wait for other event */ event_del (d->ev); event_set (d->ev, d->fd, EV_WRITE, dispatcher_cb, (void *)d); @@ -151,7 +162,7 @@ sendfile_callback (rspamd_io_dispatcher_t *d) } else if (r + d->offset < d->file_size) { d->offset += r; - debug_ip("partially write data, retry"); + debug_ip ("partially write data, retry"); /* Wait for other event */ event_del (d->ev); event_set (d->ev, d->fd, EV_WRITE, dispatcher_cb, (void *)d); @@ -161,12 +172,13 @@ sendfile_callback (rspamd_io_dispatcher_t *d) else { if (d->write_callback) { if (!d->write_callback (d->user_data)) { - debug_ip("callback set wanna_die flag, terminating"); + debug_ip ("callback set wanna_die flag, terminating"); return FALSE; } } event_del (d->ev); - event_set (d->ev, d->fd, EV_READ | EV_PERSIST, dispatcher_cb, (void *)d); + event_set (d->ev, d->fd, EV_READ | EV_PERSIST, dispatcher_cb, + (void *)d); event_base_set (d->ev_base, d->ev); event_add (d->ev, d->tv); d->in_sendfile = FALSE; @@ -177,25 +189,25 @@ sendfile_callback (rspamd_io_dispatcher_t *d) #define BUFREMAIN(x) (x)->data->size - ((x)->pos - (x)->data->begin) -#define APPEND_OUT_BUFFER(d, buf) do { \ - DL_APPEND((d)->out_buffers.buffers, buf); \ - (d)->out_buffers.pending ++; \ - } while (0) -#define DELETE_OUT_BUFFER(d, buf) do { \ - DL_DELETE((d)->out_buffers.buffers, (buf)); \ - g_string_free((buf->data), (buf)->allocated); \ - g_slice_free1(sizeof (struct rspamd_out_buffer_s), (buf)); \ - (d)->out_buffers.pending --; \ - } while (0) - -static gboolean +#define APPEND_OUT_BUFFER(d, buf) do { \ + DL_APPEND ((d)->out_buffers.buffers, buf); \ + (d)->out_buffers.pending++; \ +} while (0) +#define DELETE_OUT_BUFFER(d, buf) do { \ + DL_DELETE ((d)->out_buffers.buffers, (buf)); \ + g_string_free ((buf->data), (buf)->allocated); \ + g_slice_free1 (sizeof (struct rspamd_out_buffer_s), (buf)); \ + (d)->out_buffers.pending--; \ +} while (0) + +static gboolean write_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean is_delayed) { - GError *err = NULL; - struct rspamd_out_buffer_s *cur = NULL, *tmp; - ssize_t r; - struct iovec *iov; - guint i, len; + GError *err = NULL; + struct rspamd_out_buffer_s *cur = NULL, *tmp; + ssize_t r; + struct iovec *iov; + guint i, len; len = d->out_buffers.pending; while (len > 0) { @@ -203,24 +215,28 @@ write_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean is_delayed) is_delayed = TRUE; iov = g_slice_alloc (len * sizeof (struct iovec)); i = 0; - DL_FOREACH_SAFE (d->out_buffers.buffers, cur, tmp) { + DL_FOREACH_SAFE (d->out_buffers.buffers, cur, tmp) + { iov[i].iov_base = cur->data->str; iov[i].iov_len = cur->data->len; - i ++; + i++; } /* Now try to write the whole vector */ r = writev (fd, iov, len); if (r == -1 && errno != EAGAIN) { g_slice_free1 (len * sizeof (struct iovec), iov); if (d->err_callback) { - err = g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror (errno)); + err = + g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror ( + errno)); d->err_callback (err, d->user_data); return FALSE; } } else if (r > 0) { /* Find pos inside buffers */ - DL_FOREACH_SAFE (d->out_buffers.buffers, cur, tmp) { + DL_FOREACH_SAFE (d->out_buffers.buffers, cur, tmp) + { if (r >= (ssize_t)cur->data->len) { /* Mark this buffer as read */ r -= cur->data->len; @@ -253,7 +269,7 @@ write_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean is_delayed) } else if (r == -1 && errno == EAGAIN) { g_slice_free1 (len * sizeof (struct iovec), iov); - debug_ip("partially write data, retry"); + debug_ip ("partially write data, retry"); /* Wait for other event */ event_del (d->ev); event_set (d->ev, fd, EV_WRITE, dispatcher_cb, (void *)d); @@ -271,7 +287,7 @@ write_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean is_delayed) if (is_delayed && d->write_callback) { if (!d->write_callback (d->user_data)) { - debug_ip("callback set wanna_die flag, terminating"); + debug_ip ("callback set wanna_die flag, terminating"); return FALSE; } } @@ -295,13 +311,13 @@ write_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean is_delayed) static void read_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean skip_read) { - ssize_t r; - GError *err = NULL; - f_str_t res; - gchar *c, *b; - gchar *end; - size_t len; - enum io_policy saved_policy; + ssize_t r; + GError *err = NULL; + f_str_t res; + gchar *c, *b; + gchar *end; + size_t len; + enum io_policy saved_policy; if (d->wanna_die) { rspamd_remove_dispatcher (d); @@ -309,7 +325,8 @@ read_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean skip_read) } if (d->in_buf == NULL) { - d->in_buf = rspamd_mempool_alloc_tmp (d->pool, sizeof (rspamd_buffer_t)); + d->in_buf = + rspamd_mempool_alloc_tmp (d->pool, sizeof (rspamd_buffer_t)); if (d->policy == BUFFER_LINE || d->policy == BUFFER_ANY) { d->in_buf->data = fstralloc_tmp (d->pool, d->default_buf_size); } @@ -335,7 +352,9 @@ read_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean skip_read) r = read (fd, end, BUFREMAIN (d->in_buf)); if (r == -1 && errno != EAGAIN) { if (d->err_callback) { - err = g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror (errno)); + err = + g_error_new (G_DISPATCHER_ERROR, errno, "%s", strerror ( + errno)); d->err_callback (err, d->user_data); return; } @@ -364,7 +383,7 @@ read_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean skip_read) } } else if (r == -1 && errno == EAGAIN) { - debug_ip("partially read data, retry"); + debug_ip ("partially read data, retry"); return; } else { @@ -372,8 +391,12 @@ read_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean skip_read) d->in_buf->pos += r; d->in_buf->data->len += r; } - debug_ip("read %z characters, policy is %s, watermark is: %z, buffer has %z bytes", r, - d->policy == BUFFER_LINE ? "LINE" : "CHARACTER", d->nchars, d->in_buf->data->len); + debug_ip ( + "read %z characters, policy is %s, watermark is: %z, buffer has %z bytes", + r, + d->policy == BUFFER_LINE ? "LINE" : "CHARACTER", + d->nchars, + d->in_buf->data->len); } saved_policy = d->policy; @@ -382,16 +405,16 @@ read_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean skip_read) len = d->in_buf->data->len; b = c; r = 0; - + switch (d->policy) { case BUFFER_LINE: /** Variables: - * b - begin of line - * r - current position in buffer - * *len - length of remaining buffer - * c - pointer to current position (buffer->begin + r) - * res - result string - */ + * b - begin of line + * r - current position in buffer + * *len - length of remaining buffer + * c - pointer to current position (buffer->begin + r) + * res - result string + */ while (r < (ssize_t)len) { if (*c == '\n') { res.begin = b; @@ -404,7 +427,7 @@ read_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean skip_read) } else { /* Include EOL in reply */ - res.len ++; + res.len++; } /* Call callback for a line */ if (d->read_callback) { @@ -468,7 +491,8 @@ read_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean skip_read) d->in_buf->pos = d->in_buf->data->begin; } if (d->policy != saved_policy && (ssize_t)len != r) { - debug_ip("policy changed during callback, restart buffer's processing"); + debug_ip ( + "policy changed during callback, restart buffer's processing"); read_buffers (fd, d, TRUE); return; } @@ -484,13 +508,14 @@ read_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean skip_read) * Actually we do not want to send zero sized * buffers to a read callback */ - if (! (d->want_read && res.len == 0)) { + if (!(d->want_read && res.len == 0)) { if (!d->read_callback (&res, d->user_data)) { return; } } if (d->policy != saved_policy) { - debug_ip("policy changed during callback, restart buffer's processing"); + debug_ip ( + "policy changed during callback, restart buffer's processing"); read_buffers (fd, d, TRUE); return; } @@ -506,10 +531,10 @@ read_buffers (gint fd, rspamd_io_dispatcher_t * d, gboolean skip_read) static void dispatcher_cb (gint fd, short what, void *arg) { - rspamd_io_dispatcher_t *d = (rspamd_io_dispatcher_t *) arg; - GError *err = NULL; + rspamd_io_dispatcher_t *d = (rspamd_io_dispatcher_t *) arg; + GError *err = NULL; - debug_ip("in dispatcher callback, what: %d, fd: %d", (gint)what, fd); + debug_ip ("in dispatcher callback, what: %d, fd: %d", (gint)what, fd); if ((what & EV_TIMEOUT) != 0) { if (d->err_callback) { @@ -538,7 +563,8 @@ dispatcher_cb (gint fd, short what, void *arg) else { /* Want read again */ event_del (d->ev); - event_set (d->ev, fd, EV_READ | EV_PERSIST, dispatcher_cb, (void *)d); + event_set (d->ev, fd, EV_READ | EV_PERSIST, dispatcher_cb, + (void *)d); event_base_set (d->ev_base, d->ev); event_add (d->ev, d->tv); if (d->is_restored && d->write_callback) { @@ -558,11 +584,17 @@ dispatcher_cb (gint fd, short what, void *arg) } -rspamd_io_dispatcher_t * -rspamd_create_dispatcher (struct event_base *base, gint fd, enum io_policy policy, - dispatcher_read_callback_t read_cb, dispatcher_write_callback_t write_cb, dispatcher_err_callback_t err_cb, struct timeval *tv, void *user_data) +rspamd_io_dispatcher_t * +rspamd_create_dispatcher (struct event_base *base, + gint fd, + enum io_policy policy, + dispatcher_read_callback_t read_cb, + dispatcher_write_callback_t write_cb, + dispatcher_err_callback_t err_cb, + struct timeval *tv, + void *user_data) { - rspamd_io_dispatcher_t *new; + rspamd_io_dispatcher_t *new; if (fd == -1) { return NULL; @@ -608,7 +640,8 @@ rspamd_remove_dispatcher (rspamd_io_dispatcher_t * d) struct rspamd_out_buffer_s *cur, *tmp; if (d != NULL) { - DL_FOREACH_SAFE (d->out_buffers.buffers, cur, tmp) { + DL_FOREACH_SAFE (d->out_buffers.buffers, cur, tmp) + { DELETE_OUT_BUFFER (d, cur); } event_del (d->ev); @@ -618,10 +651,12 @@ rspamd_remove_dispatcher (rspamd_io_dispatcher_t * d) } void -rspamd_set_dispatcher_policy (rspamd_io_dispatcher_t * d, enum io_policy policy, size_t nchars) +rspamd_set_dispatcher_policy (rspamd_io_dispatcher_t * d, + enum io_policy policy, + size_t nchars) { - f_str_t *tmp; - gint t; + f_str_t *tmp; + gint t; if (d->policy != policy || nchars != d->nchars) { d->policy = policy; @@ -630,7 +665,8 @@ rspamd_set_dispatcher_policy (rspamd_io_dispatcher_t * d, enum io_policy policy, if (policy == BUFFER_CHARACTER && nchars != 0) { if (d->in_buf && d->in_buf->data->size < nchars) { tmp = fstralloc_tmp (d->pool, d->nchars + 1); - memcpy (tmp->begin, d->in_buf->data->begin, d->in_buf->data->len); + memcpy (tmp->begin, d->in_buf->data->begin, + d->in_buf->data->len); t = d->in_buf->pos - d->in_buf->data->begin; tmp->len = d->in_buf->data->len; d->in_buf->data = tmp; @@ -640,7 +676,8 @@ rspamd_set_dispatcher_policy (rspamd_io_dispatcher_t * d, enum io_policy policy, else if (policy == BUFFER_LINE || policy == BUFFER_ANY) { if (d->in_buf && d->nchars < d->default_buf_size) { tmp = fstralloc_tmp (d->pool, d->default_buf_size); - memcpy (tmp->begin, d->in_buf->data->begin, d->in_buf->data->len); + memcpy (tmp->begin, d->in_buf->data->begin, + d->in_buf->data->len); t = d->in_buf->pos - d->in_buf->data->begin; tmp->len = d->in_buf->data->len; d->in_buf->data = tmp; @@ -650,14 +687,14 @@ rspamd_set_dispatcher_policy (rspamd_io_dispatcher_t * d, enum io_policy policy, } } - debug_ip("new input length watermark is %uz", d->nchars); + debug_ip ("new input length watermark is %uz", d->nchars); } gboolean rspamd_dispatcher_write (rspamd_io_dispatcher_t * d, - const void *data, size_t len, gboolean delayed, gboolean allocated) + const void *data, size_t len, gboolean delayed, gboolean allocated) { - struct rspamd_out_buffer_s *newbuf; + struct rspamd_out_buffer_s *newbuf; newbuf = g_slice_alloc (sizeof (struct rspamd_out_buffer_s)); if (len == 0) { @@ -680,7 +717,7 @@ rspamd_dispatcher_write (rspamd_io_dispatcher_t * d, APPEND_OUT_BUFFER (d, newbuf); if (!delayed) { - debug_ip("plan write event"); + debug_ip ("plan write event"); return write_buffers (d->fd, d, FALSE); } /* Otherwise plan write event */ @@ -692,12 +729,13 @@ rspamd_dispatcher_write (rspamd_io_dispatcher_t * d, return TRUE; } -gboolean rspamd_dispatcher_write_string (rspamd_io_dispatcher_t *d, - GString *str, - gboolean delayed, - gboolean free_on_write) +gboolean +rspamd_dispatcher_write_string (rspamd_io_dispatcher_t *d, + GString *str, + gboolean delayed, + gboolean free_on_write) { - struct rspamd_out_buffer_s *newbuf; + struct rspamd_out_buffer_s *newbuf; newbuf = g_slice_alloc (sizeof (struct rspamd_out_buffer_s)); newbuf->data = str; @@ -706,7 +744,7 @@ gboolean rspamd_dispatcher_write_string (rspamd_io_dispatcher_t *d, APPEND_OUT_BUFFER (d, newbuf); if (!delayed) { - debug_ip("plan write event"); + debug_ip ("plan write event"); return write_buffers (d->fd, d, FALSE); } /* Otherwise plan write event */ @@ -718,7 +756,7 @@ gboolean rspamd_dispatcher_write_string (rspamd_io_dispatcher_t *d, return TRUE; } -gboolean +gboolean rspamd_dispatcher_sendfile (rspamd_io_dispatcher_t *d, gint fd, size_t len) { if (lseek (fd, 0, SEEK_SET) == -1) { @@ -733,9 +771,12 @@ rspamd_dispatcher_sendfile (rspamd_io_dispatcher_t *d, gint fd, size_t len) #ifndef HAVE_SENDFILE #ifdef HAVE_MMAP_NOCORE - if ((d->map = mmap (NULL, len, PROT_READ, MAP_SHARED | MAP_NOCORE, fd, 0)) == MAP_FAILED) { + if ((d->map = + mmap (NULL, len, PROT_READ, MAP_SHARED | MAP_NOCORE, fd, + 0)) == MAP_FAILED) { #else - if ((d->map = mmap (NULL, len, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { + if ((d->map = + mmap (NULL, len, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { #endif msg_warn ("mmap failed: %s", strerror (errno)); return FALSE; @@ -771,7 +812,8 @@ rspamd_dispacther_cleanup (rspamd_io_dispatcher_t *d) { struct rspamd_out_buffer_s *cur, *tmp; - DL_FOREACH_SAFE (d->out_buffers.buffers, cur, tmp) { + DL_FOREACH_SAFE (d->out_buffers.buffers, cur, tmp) + { DELETE_OUT_BUFFER (d, cur); } /* Cleanup temporary data */ @@ -781,6 +823,6 @@ rspamd_dispacther_cleanup (rspamd_io_dispatcher_t *d) #undef debug_ip -/* - * vi:ts=4 +/* + * vi:ts=4 */ diff --git a/src/libserver/buffer.h b/src/libserver/buffer.h index 5ed42bfb3..0a4c54462 100644 --- a/src/libserver/buffer.h +++ b/src/libserver/buffer.h @@ -7,8 +7,8 @@ #define RSPAMD_BUFFER_H #include "config.h" -#include "mem_pool.h" #include "fstring.h" +#include "mem_pool.h" typedef gboolean (*dispatcher_read_callback_t)(f_str_t *in, void *user_data); typedef gboolean (*dispatcher_write_callback_t)(void *user_data); @@ -18,17 +18,17 @@ typedef void (*dispatcher_err_callback_t)(GError *err, void *user_data); * Types of IO handling */ enum io_policy { - BUFFER_LINE, /**< call handler when we have line ready */ - BUFFER_CHARACTER, /**< call handler when we have some characters */ - BUFFER_ANY /**< call handler whenever we got data in buffer */ + BUFFER_LINE, /**< call handler when we have line ready */ + BUFFER_CHARACTER, /**< call handler when we have some characters */ + BUFFER_ANY /**< call handler whenever we got data in buffer */ }; /** * Buffer structure */ typedef struct rspamd_buffer_s { - f_str_t *data; /**< buffer logic */ - gchar *pos; /**< current position */ + f_str_t *data; /**< buffer logic */ + gchar *pos; /**< current position */ } rspamd_buffer_t; struct rspamd_out_buffer_s { @@ -38,33 +38,33 @@ struct rspamd_out_buffer_s { }; typedef struct rspamd_io_dispatcher_s { - rspamd_buffer_t *in_buf; /**< input buffer */ + rspamd_buffer_t *in_buf; /**< input buffer */ struct { guint pending; struct rspamd_out_buffer_s *buffers; - } out_buffers; /**< output buffers chain */ - struct timeval *tv; /**< io timeout */ - struct event *ev; /**< libevent io event */ - rspamd_mempool_t *pool; /**< where to store data */ - enum io_policy policy; /**< IO policy */ - size_t nchars; /**< how many chars to read */ - gint fd; /**< descriptor */ - guint32 peer_addr; /**< address of peer for debugging */ - gboolean wanna_die; /**< if dispatcher should be stopped */ - dispatcher_read_callback_t read_callback; /**< read callback */ - dispatcher_write_callback_t write_callback; /**< write callback */ - dispatcher_err_callback_t err_callback; /**< error callback */ - void *user_data; /**< user's data for callbacks */ - gulong default_buf_size; /**< default size for buffering */ - off_t offset; /**< for sendfile use */ + } out_buffers; /**< output buffers chain */ + struct timeval *tv; /**< io timeout */ + struct event *ev; /**< libevent io event */ + rspamd_mempool_t *pool; /**< where to store data */ + enum io_policy policy; /**< IO policy */ + size_t nchars; /**< how many chars to read */ + gint fd; /**< descriptor */ + guint32 peer_addr; /**< address of peer for debugging */ + gboolean wanna_die; /**< if dispatcher should be stopped */ + dispatcher_read_callback_t read_callback; /**< read callback */ + dispatcher_write_callback_t write_callback; /**< write callback */ + dispatcher_err_callback_t err_callback; /**< error callback */ + void *user_data; /**< user's data for callbacks */ + gulong default_buf_size; /**< default size for buffering */ + off_t offset; /**< for sendfile use */ size_t file_size; gint sendfile_fd; - gboolean in_sendfile; /**< whether buffer is in sendfile mode */ - gboolean strip_eol; /**< strip or not line ends in BUFFER_LINE policy */ - gboolean is_restored; /**< call a callback when dispatcher is restored */ - gboolean half_closed; /**< connection is half closed */ - gboolean want_read; /**< whether we want to read more data */ - struct event_base *ev_base; /**< event base for io operations */ + gboolean in_sendfile; /**< whether buffer is in sendfile mode */ + gboolean strip_eol; /**< strip or not line ends in BUFFER_LINE policy */ + gboolean is_restored; /**< call a callback when dispatcher is restored */ + gboolean half_closed; /**< connection is half closed */ + gboolean want_read; /**< whether we want to read more data */ + struct event_base *ev_base; /**< event base for io operations */ #ifndef HAVE_SENDFILE void *map; #endif @@ -81,13 +81,14 @@ typedef struct rspamd_io_dispatcher_s { * @param user_data pointer to user's data * @return new dispatcher object or NULL in case of failure */ -rspamd_io_dispatcher_t* rspamd_create_dispatcher (struct event_base *base, gint fd, - enum io_policy policy, - dispatcher_read_callback_t read_cb, - dispatcher_write_callback_t write_cb, - dispatcher_err_callback_t err_cb, - struct timeval *tv, - void *user_data); +rspamd_io_dispatcher_t * rspamd_create_dispatcher (struct event_base *base, + gint fd, + enum io_policy policy, + dispatcher_read_callback_t read_cb, + dispatcher_write_callback_t write_cb, + dispatcher_err_callback_t err_cb, + struct timeval *tv, + void *user_data); /** * Set new policy for dispatcher @@ -95,9 +96,9 @@ rspamd_io_dispatcher_t* rspamd_create_dispatcher (struct event_base *base, gint * @param policy IO policy * @param nchars number of characters in buffer for character policy */ -void rspamd_set_dispatcher_policy (rspamd_io_dispatcher_t *d, - enum io_policy policy, - size_t nchars); +void rspamd_set_dispatcher_policy (rspamd_io_dispatcher_t *d, + enum io_policy policy, + size_t nchars); /** * Write data when it would be possible @@ -106,9 +107,9 @@ void rspamd_set_dispatcher_policy (rspamd_io_dispatcher_t *d, * @param len length of data */ gboolean rspamd_dispatcher_write (rspamd_io_dispatcher_t *d, - const void *data, - size_t len, gboolean delayed, - gboolean allocated) G_GNUC_WARN_UNUSED_RESULT; + const void *data, + size_t len, gboolean delayed, + gboolean allocated) G_GNUC_WARN_UNUSED_RESULT; /** * Write a GString to dispatcher @@ -119,9 +120,9 @@ gboolean rspamd_dispatcher_write (rspamd_io_dispatcher_t *d, * @return TRUE if write has been queued successfully */ gboolean rspamd_dispatcher_write_string (rspamd_io_dispatcher_t *d, - GString *str, - gboolean delayed, - gboolean free_on_write) G_GNUC_WARN_UNUSED_RESULT; + GString *str, + gboolean delayed, + gboolean free_on_write) G_GNUC_WARN_UNUSED_RESULT; /** * Send specified descriptor to dispatcher @@ -129,7 +130,9 @@ gboolean rspamd_dispatcher_write_string (rspamd_io_dispatcher_t *d, * @param fd descriptor of file * @param len length of data */ -gboolean rspamd_dispatcher_sendfile (rspamd_io_dispatcher_t *d, gint fd, size_t len) G_GNUC_WARN_UNUSED_RESULT; +gboolean rspamd_dispatcher_sendfile (rspamd_io_dispatcher_t *d, + gint fd, + size_t len) G_GNUC_WARN_UNUSED_RESULT; /** * Pause IO events on dispatcher diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h index f502d5775..6626ecf35 100644 --- a/src/libserver/cfg_file.h +++ b/src/libserver/cfg_file.h @@ -6,13 +6,13 @@ #ifndef CFG_FILE_H #define CFG_FILE_H +#include "cfg_rcl.h" #include "config.h" #include "mem_pool.h" -#include "upstream.h" #include "symbols_cache.h" -#include "cfg_rcl.h" -#include "utlist.h" #include "ucl.h" +#include "upstream.h" +#include "utlist.h" #define DEFAULT_BIND_PORT 11333 #define DEFAULT_CONTROL_PORT 11334 @@ -72,22 +72,22 @@ enum rspamd_log_type { * Regexp structure */ struct rspamd_regexp { - enum rspamd_regexp_type type; /**< regexp type */ - gchar *regexp_text; /**< regexp text representation */ - GRegex *regexp; /**< glib regexp structure */ - GRegex *raw_regexp; /**< glib regexp structure for raw matching */ - gchar *header; /**< header name for header regexps */ - gboolean is_test; /**< true if this expression must be tested */ - gboolean is_raw; /**< true if this regexp is done by raw matching */ - gboolean is_strong; /**< true if headers search must be case sensitive */ + enum rspamd_regexp_type type; /**< regexp type */ + gchar *regexp_text; /**< regexp text representation */ + GRegex *regexp; /**< glib regexp structure */ + GRegex *raw_regexp; /**< glib regexp structure for raw matching */ + gchar *header; /**< header name for header regexps */ + gboolean is_test; /**< true if this expression must be tested */ + gboolean is_raw; /**< true if this regexp is done by raw matching */ + gboolean is_strong; /**< true if headers search must be case sensitive */ }; /** * script module list item */ struct script_module { - gchar *name; /**< name of module */ - gchar *path; /**< path to module */ + gchar *name; /**< name of module */ + gchar *path; /**< path to module */ }; /** @@ -104,13 +104,13 @@ enum lua_var_type { * Module option */ struct rspamd_module_opt { - gchar *param; /**< parameter name */ - gchar *value; /**< parameter value */ - gchar *description; /**< parameter description */ - gchar *group; /**< parameter group */ - gpointer actual_data; /**< parsed data */ - gboolean is_lua; /**< actually this is lua variable */ - enum lua_var_type lua_type; /**< type of lua variable */ + gchar *param; /**< parameter name */ + gchar *value; /**< parameter value */ + gchar *description; /**< parameter description */ + gchar *group; /**< parameter group */ + gpointer actual_data; /**< parsed data */ + gboolean is_lua; /**< actually this is lua variable */ + enum lua_var_type lua_type; /**< type of lua variable */ }; /** @@ -134,22 +134,22 @@ struct rspamd_symbols_group { * Statfile section definition */ struct rspamd_statfile_section { - guint32 code; /**< section's code */ - guint64 size; /**< size of section */ - double weight; /**< weight coefficient for section */ + guint32 code; /**< section's code */ + guint64 size; /**< size of section */ + double weight; /**< weight coefficient for section */ }; /** * Statfile autolearn parameters */ struct statfile_autolearn_params { - const gchar *metric; /**< metric name for autolearn triggering */ - double threshold_min; /**< threshold mark */ - double threshold_max; /**< threshold mark */ - GList *symbols; /**< list of symbols */ + const gchar *metric; /**< metric name for autolearn triggering */ + double threshold_min; /**< threshold mark */ + double threshold_max; /**< threshold mark */ + GList *symbols; /**< list of symbols */ }; -/** +/** * Sync affinity */ enum sync_affinity { @@ -168,38 +168,39 @@ struct statfile_binlog_params { guint16 master_port; }; -typedef double (*statfile_normalize_func)(struct rspamd_config *cfg, long double score, void *params); +typedef double (*statfile_normalize_func)(struct rspamd_config *cfg, + long double score, void *params); /** * Statfile config definition */ struct rspamd_statfile_config { - gchar *symbol; /**< symbol of statfile */ - gchar *path; /**< filesystem pattern (with %r or %f) */ - gchar *label; /**< label of this statfile */ - gsize size; /**< size of statfile */ - GList *sections; /**< list of sections in statfile */ - struct statfile_autolearn_params *autolearn; /**< autolearn params */ - struct statfile_binlog_params *binlog; /**< binlog params */ - statfile_normalize_func normalizer; /**< function that is used as normaliser */ - void *normalizer_data; /**< normalizer function params */ - gchar *normalizer_str; /**< source string (for dump) */ - ucl_object_t *opts; /**< other options */ - gboolean is_spam; /**< spam flag */ + gchar *symbol; /**< symbol of statfile */ + gchar *path; /**< filesystem pattern (with %r or %f) */ + gchar *label; /**< label of this statfile */ + gsize size; /**< size of statfile */ + GList *sections; /**< list of sections in statfile */ + struct statfile_autolearn_params *autolearn; /**< autolearn params */ + struct statfile_binlog_params *binlog; /**< binlog params */ + statfile_normalize_func normalizer; /**< function that is used as normaliser */ + void *normalizer_data; /**< normalizer function params */ + gchar *normalizer_str; /**< source string (for dump) */ + ucl_object_t *opts; /**< other options */ + gboolean is_spam; /**< spam flag */ }; /** * Classifier config definition */ struct rspamd_classifier_config { - GList *statfiles; /**< statfiles list */ - GHashTable *labels; /**< statfiles with labels */ - gchar *metric; /**< metric of this classifier */ - struct classifier *classifier; /**< classifier interface */ - struct tokenizer *tokenizer; /**< tokenizer used for classifier */ - GHashTable *opts; /**< other options */ - GList *pre_callbacks; /**< list of callbacks that are called before classification */ - GList *post_callbacks; /**< list of callbacks that are called after classification */ + GList *statfiles; /**< statfiles list */ + GHashTable *labels; /**< statfiles with labels */ + gchar *metric; /**< metric of this classifier */ + struct classifier *classifier; /**< classifier interface */ + struct tokenizer *tokenizer; /**< tokenizer used for classifier */ + GHashTable *opts; /**< other options */ + GList *pre_callbacks; /**< list of callbacks that are called before classification */ + GList *post_callbacks; /**< list of callbacks that are called after classification */ }; struct rspamd_worker_bind_conf { @@ -211,135 +212,135 @@ struct rspamd_worker_bind_conf { }; struct rspamd_worker_param_parser { - rspamd_rcl_handler_t handler; /**< handler function */ - struct rspamd_rcl_struct_parser parser; /**< parser attributes */ - const gchar *name; /**< parameter's name */ - UT_hash_handle hh; /**< hash by name */ + rspamd_rcl_handler_t handler; /**< handler function */ + struct rspamd_rcl_struct_parser parser; /**< parser attributes */ + const gchar *name; /**< parameter's name */ + UT_hash_handle hh; /**< hash by name */ }; struct rspamd_worker_cfg_parser { - struct rspamd_worker_param_parser *parsers; /**< parsers hash */ - gint type; /**< workers quark */ - gboolean (*def_obj_parser)(const ucl_object_t *obj, gpointer ud); /**< default object parser */ + struct rspamd_worker_param_parser *parsers; /**< parsers hash */ + gint type; /**< workers quark */ + gboolean (*def_obj_parser)(const ucl_object_t *obj, gpointer ud); /**< default object parser */ gpointer def_ud; - UT_hash_handle hh; /**< hash by type */ + UT_hash_handle hh; /**< hash by type */ }; /** * Config params for rspamd worker */ struct rspamd_worker_conf { - worker_t *worker; /**< pointer to worker type */ - GQuark type; /**< type of worker */ - struct rspamd_worker_bind_conf *bind_conf; /**< bind configuration */ - guint16 count; /**< number of workers */ - GList *listen_socks; /**< listening sockets desctiptors */ - guint32 rlimit_nofile; /**< max files limit */ - guint32 rlimit_maxcore; /**< maximum core file size */ - GHashTable *params; /**< params for worker */ - GQueue *active_workers; /**< linked list of spawned workers */ - gboolean has_socket; /**< whether we should make listening socket in main process */ - gpointer *ctx; /**< worker's context */ - ucl_object_t *options; /**< other worker's options */ + worker_t *worker; /**< pointer to worker type */ + GQuark type; /**< type of worker */ + struct rspamd_worker_bind_conf *bind_conf; /**< bind configuration */ + guint16 count; /**< number of workers */ + GList *listen_socks; /**< listening sockets desctiptors */ + guint32 rlimit_nofile; /**< max files limit */ + guint32 rlimit_maxcore; /**< maximum core file size */ + GHashTable *params; /**< params for worker */ + GQueue *active_workers; /**< linked list of spawned workers */ + gboolean has_socket; /**< whether we should make listening socket in main process */ + gpointer *ctx; /**< worker's context */ + ucl_object_t *options; /**< other worker's options */ }; /** * Structure that stores all config data */ struct rspamd_config { - gchar *rspamd_user; /**< user to run as */ - gchar *rspamd_group; /**< group to run as */ - rspamd_mempool_t *cfg_pool; /**< memory pool for config */ - gchar *cfg_name; /**< name of config file */ - gchar *pid_file; /**< name of pid file */ - gchar *temp_dir; /**< dir for temp files */ + gchar *rspamd_user; /**< user to run as */ + gchar *rspamd_group; /**< group to run as */ + rspamd_mempool_t *cfg_pool; /**< memory pool for config */ + gchar *cfg_name; /**< name of config file */ + gchar *pid_file; /**< name of pid file */ + gchar *temp_dir; /**< dir for temp files */ #ifdef WITH_GPERF_TOOLS gchar *profile_path; #endif - gboolean no_fork; /**< if 1 do not call daemon() */ - gboolean config_test; /**< if TRUE do only config file test */ - gboolean raw_mode; /**< work in raw mode instead of utf one */ - gboolean one_shot_mode; /**< rules add only one symbol */ - gboolean check_text_attachements; /**< check text attachements as text */ - gboolean convert_config; /**< convert config to XML format */ - gboolean strict_protocol_headers; /**< strictly check protocol headers */ - - gsize max_diff; /**< maximum diff size for text parts */ - - enum rspamd_log_type log_type; /**< log type */ - gint log_facility; /**< log facility in case of syslog */ - gint log_level; /**< log level trigger */ - gchar *log_file; /**< path to logfile in case of file logging */ - gboolean log_buffered; /**< whether logging is buffered */ - guint32 log_buf_size; /**< length of log buffer */ - gchar *debug_ip_map; /**< turn on debugging for specified ip addresses */ - gboolean log_urls; /**< whether we should log URLs */ - GList *debug_symbols; /**< symbols to debug */ - gboolean log_color; /**< output colors for console output */ - gboolean log_extended; /**< log extended information */ - - guint32 statfile_sync_interval; /**< synchronization interval */ - guint32 statfile_sync_timeout; /**< synchronization timeout */ - gboolean mlock_statfile_pool; /**< use mlock (2) for locking statfiles */ - - gboolean delivery_enable; /**< is delivery agent is enabled */ - gchar *deliver_host; /**< host for mail deliviring */ - struct in_addr deliver_addr; /**< its address */ - guint16 deliver_port; /**< port for deliviring */ - guint16 deliver_family; /**< socket family for delivirnig */ - gchar *deliver_agent_path; /**< deliver to pipe instead of socket */ - gboolean deliver_lmtp; /**< use LMTP instead of SMTP */ - - GList *script_modules; /**< linked list of script modules to load */ - - GList *filters; /**< linked list of all filters */ - GList *workers; /**< linked list of all workers params */ - struct rspamd_worker_cfg_parser *wrk_parsers; /**< hash for worker config parsers, indexed by worker quarks */ - gchar *filters_str; /**< string of filters */ - ucl_object_t *rcl_obj; /**< rcl object */ - GHashTable* metrics; /**< hash of metrics indexed by metric name */ - GList* symbols_groups; /**< groups of symbols */ - GList* metrics_list; /**< linked list of metrics */ - GHashTable* metrics_symbols; /**< hash table of metrics indexed by symbol */ - GHashTable* c_modules; /**< hash of c modules indexed by module name */ - GHashTable* composite_symbols; /**< hash of composite symbols indexed by its name */ + gboolean no_fork; /**< if 1 do not call daemon() */ + gboolean config_test; /**< if TRUE do only config file test */ + gboolean raw_mode; /**< work in raw mode instead of utf one */ + gboolean one_shot_mode; /**< rules add only one symbol */ + gboolean check_text_attachements; /**< check text attachements as text */ + gboolean convert_config; /**< convert config to XML format */ + gboolean strict_protocol_headers; /**< strictly check protocol headers */ + + gsize max_diff; /**< maximum diff size for text parts */ + + enum rspamd_log_type log_type; /**< log type */ + gint log_facility; /**< log facility in case of syslog */ + gint log_level; /**< log level trigger */ + gchar *log_file; /**< path to logfile in case of file logging */ + gboolean log_buffered; /**< whether logging is buffered */ + guint32 log_buf_size; /**< length of log buffer */ + gchar *debug_ip_map; /**< turn on debugging for specified ip addresses */ + gboolean log_urls; /**< whether we should log URLs */ + GList *debug_symbols; /**< symbols to debug */ + gboolean log_color; /**< output colors for console output */ + gboolean log_extended; /**< log extended information */ + + guint32 statfile_sync_interval; /**< synchronization interval */ + guint32 statfile_sync_timeout; /**< synchronization timeout */ + gboolean mlock_statfile_pool; /**< use mlock (2) for locking statfiles */ + + gboolean delivery_enable; /**< is delivery agent is enabled */ + gchar *deliver_host; /**< host for mail deliviring */ + struct in_addr deliver_addr; /**< its address */ + guint16 deliver_port; /**< port for deliviring */ + guint16 deliver_family; /**< socket family for delivirnig */ + gchar *deliver_agent_path; /**< deliver to pipe instead of socket */ + gboolean deliver_lmtp; /**< use LMTP instead of SMTP */ + + GList *script_modules; /**< linked list of script modules to load */ + + GList *filters; /**< linked list of all filters */ + GList *workers; /**< linked list of all workers params */ + struct rspamd_worker_cfg_parser *wrk_parsers; /**< hash for worker config parsers, indexed by worker quarks */ + gchar *filters_str; /**< string of filters */ + ucl_object_t *rcl_obj; /**< rcl object */ + GHashTable * metrics; /**< hash of metrics indexed by metric name */ + GList * symbols_groups; /**< groups of symbols */ + GList * metrics_list; /**< linked list of metrics */ + GHashTable * metrics_symbols; /**< hash table of metrics indexed by symbol */ + GHashTable * c_modules; /**< hash of c modules indexed by module name */ + GHashTable * composite_symbols; /**< hash of composite symbols indexed by its name */ GList *classifiers; /**< list of all classifiers defined */ GList *statfiles; /**< list of all statfiles in config file order */ GHashTable *classifiers_symbols; /**< hashtable indexed by symbol name of classifiers */ - GHashTable* cfg_params; /**< all cfg params indexed by its name in this structure */ - GList *pre_filters; /**< list of pre-processing lua filters */ - GList *post_filters; /**< list of post-processing lua filters */ - gchar *dynamic_conf; /**< path to dynamic configuration */ - GList *current_dynamic_conf; /**< currently loaded dynamic configuration */ - GHashTable* domain_settings; /**< settings per-domains */ - GHashTable* user_settings; /**< settings per-user */ - gchar* domain_settings_str; /**< string representation of settings */ - gchar* user_settings_str; - gint clock_res; /**< resolution of clock used */ - - GList *maps; /**< maps active */ - rspamd_mempool_t *map_pool; /**< static maps pool */ - gdouble map_timeout; /**< maps watch timeout */ - - struct symbols_cache *cache; /**< symbols cache object */ - gchar *cache_filename; /**< filename of cache file */ - struct metric *default_metric; /**< default metric */ - - gchar* checksum; /**< real checksum of config file */ - gchar* dump_checksum; /**< dump checksum of config file */ - gpointer lua_state; /**< pointer to lua state */ - - gchar* rrd_file; /**< rrd file to store statistics */ - - gchar* history_file; /**< file to save rolling history */ - - gdouble dns_timeout; /**< timeout in milliseconds for waiting for dns reply */ - guint32 dns_retransmits; /**< maximum retransmits count */ - guint32 dns_throttling_errors; /**< maximum errors for starting resolver throttling */ - guint32 dns_throttling_time; /**< time in seconds for DNS throttling */ - guint32 dns_io_per_server; /**< number of sockets per DNS server */ - GList *nameservers; /**< list of nameservers or NULL to parse resolv.conf */ + GHashTable * cfg_params; /**< all cfg params indexed by its name in this structure */ + GList *pre_filters; /**< list of pre-processing lua filters */ + GList *post_filters; /**< list of post-processing lua filters */ + gchar *dynamic_conf; /**< path to dynamic configuration */ + GList *current_dynamic_conf; /**< currently loaded dynamic configuration */ + GHashTable * domain_settings; /**< settings per-domains */ + GHashTable * user_settings; /**< settings per-user */ + gchar * domain_settings_str; /**< string representation of settings */ + gchar * user_settings_str; + gint clock_res; /**< resolution of clock used */ + + GList *maps; /**< maps active */ + rspamd_mempool_t *map_pool; /**< static maps pool */ + gdouble map_timeout; /**< maps watch timeout */ + + struct symbols_cache *cache; /**< symbols cache object */ + gchar *cache_filename; /**< filename of cache file */ + struct metric *default_metric; /**< default metric */ + + gchar * checksum; /**< real checksum of config file */ + gchar * dump_checksum; /**< dump checksum of config file */ + gpointer lua_state; /**< pointer to lua state */ + + gchar * rrd_file; /**< rrd file to store statistics */ + + gchar * history_file; /**< file to save rolling history */ + + gdouble dns_timeout; /**< timeout in milliseconds for waiting for dns reply */ + guint32 dns_retransmits; /**< maximum retransmits count */ + guint32 dns_throttling_errors; /**< maximum errors for starting resolver throttling */ + guint32 dns_throttling_time; /**< time in seconds for DNS throttling */ + guint32 dns_io_per_server; /**< number of sockets per DNS server */ + GList *nameservers; /**< list of nameservers or NULL to parse resolv.conf */ }; @@ -351,7 +352,7 @@ struct rspamd_config { * @return TRUE if string was parsed */ gboolean rspamd_parse_host_port_priority (rspamd_mempool_t *pool, - const gchar *str, gchar **addr, guint16 *port, guint *priority); + const gchar *str, gchar **addr, guint16 *port, guint *priority); /** * Parse host:port line @@ -360,7 +361,7 @@ gboolean rspamd_parse_host_port_priority (rspamd_mempool_t *pool, * @return TRUE if string was parsed */ gboolean rspamd_parse_host_port (rspamd_mempool_t *pool, const gchar *str, - gchar **addr, guint16 *port); + gchar **addr, guint16 *port); /** * Parse host:priority line @@ -369,7 +370,7 @@ gboolean rspamd_parse_host_port (rspamd_mempool_t *pool, const gchar *str, * @return TRUE if string was parsed */ gboolean rspamd_parse_host_priority (rspamd_mempool_t *pool, const gchar *str, - gchar **addr, guint *priority); + gchar **addr, guint *priority); /** * Parse bind credits @@ -379,7 +380,7 @@ gboolean rspamd_parse_host_priority (rspamd_mempool_t *pool, const gchar *str, * @return 1 if line was successfully parsed and 0 in case of error */ gboolean rspamd_parse_bind_line (struct rspamd_config *cfg, - struct rspamd_worker_conf *cf, const gchar *str); + struct rspamd_worker_conf *cf, const gchar *str); /** * Init default values @@ -400,9 +401,9 @@ void rspamd_config_free (struct rspamd_config *cfg); * @param opt_name name of option to get * @return module value or NULL if option does not defined */ -const ucl_object_t* rspamd_config_get_module_opt (struct rspamd_config *cfg, - const gchar *module_name, - const gchar *opt_name); +const ucl_object_t * rspamd_config_get_module_opt (struct rspamd_config *cfg, + const gchar *module_name, + const gchar *opt_name); /** * Parse limit @@ -440,36 +441,38 @@ void rspamd_config_unescape_quotes (gchar *line); /* * Convert comma separated string to a list of strings */ -GList* rspamd_config_parse_comma_list (rspamd_mempool_t *pool, - const gchar *line); +GList * rspamd_config_parse_comma_list (rspamd_mempool_t *pool, + const gchar *line); /* * Return a new classifier_config structure, setting default and non-conflicting attributes */ -struct rspamd_classifier_config* rspamd_config_new_classifier (struct rspamd_config *cfg, - struct rspamd_classifier_config *c); +struct rspamd_classifier_config * rspamd_config_new_classifier ( + struct rspamd_config *cfg, + struct rspamd_classifier_config *c); /* * Return a new worker_conf structure, setting default and non-conflicting attributes */ -struct rspamd_worker_conf* rspamd_config_new_worker (struct rspamd_config *cfg, - struct rspamd_worker_conf *c); +struct rspamd_worker_conf * rspamd_config_new_worker (struct rspamd_config *cfg, + struct rspamd_worker_conf *c); /* * Return a new metric structure, setting default and non-conflicting attributes */ -struct metric* rspamd_config_new_metric (struct rspamd_config *cfg, - struct metric *c); +struct metric * rspamd_config_new_metric (struct rspamd_config *cfg, + struct metric *c); /* * Return a new statfile structure, setting default and non-conflicting attributes */ -struct rspamd_statfile_config* rspamd_config_new_statfile (struct rspamd_config *cfg, - struct rspamd_statfile_config *c); +struct rspamd_statfile_config * rspamd_config_new_statfile ( + struct rspamd_config *cfg, + struct rspamd_statfile_config *c); /* * Read XML configuration file */ gboolean rspamd_config_read (struct rspamd_config *cfg, - const gchar *filename, const gchar *convert_to, - rspamd_rcl_section_fin_t logger_fin, gpointer logger_ud); + const gchar *filename, const gchar *convert_to, + rspamd_rcl_section_fin_t logger_fin, gpointer logger_ud); /* * Register symbols of classifiers inside metrics @@ -484,17 +487,17 @@ gboolean rspamd_config_check_statfiles (struct rspamd_classifier_config *cf); /* * Find classifier config by name */ -struct rspamd_classifier_config* rspamd_config_find_classifier ( - struct rspamd_config *cfg, - const gchar *name); +struct rspamd_classifier_config * rspamd_config_find_classifier ( + struct rspamd_config *cfg, + const gchar *name); /* * Parse input `ip_list` to radix tree `tree`. Now supports only IPv4 addresses. */ gboolean rspamd_config_parse_ip_list (const gchar *ip_list, - radix_tree_t **tree); + radix_tree_t **tree); #endif /* ifdef CFG_FILE_H */ -/* - * vi:ts=4 +/* + * vi:ts=4 */ diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c index f9524fd62..5021acfb2 100644 --- a/src/libserver/cfg_rcl.c +++ b/src/libserver/cfg_rcl.c @@ -21,12 +21,12 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "cfg_rcl.h" -#include "main.h" #include "cfg_file.h" -#include "lua/lua_common.h" -#include "expressions.h" +#include "cfg_rcl.h" #include "classifiers/classifiers.h" +#include "expressions.h" +#include "lua/lua_common.h" +#include "main.h" #include "tokenizers/tokenizers.h" /* @@ -34,7 +34,7 @@ */ static gboolean rspamd_rcl_logging_handler (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) + gpointer ud, struct rspamd_rcl_section *section, GError **err) { const ucl_object_t *val; const gchar *facility, *log_type, *log_level; @@ -45,11 +45,15 @@ rspamd_rcl_logging_handler (struct rspamd_config *cfg, const ucl_object_t *obj, /* Need to get filename */ val = ucl_object_find_key (obj, "filename"); if (val == NULL || val->type != UCL_STRING) { - g_set_error (err, CFG_RCL_ERROR, ENOENT, "filename attribute must be specified for file logging type"); + g_set_error (err, + CFG_RCL_ERROR, + ENOENT, + "filename attribute must be specified for file logging type"); return FALSE; } cfg->log_type = RSPAMD_LOG_FILE; - cfg->log_file = rspamd_mempool_strdup (cfg->cfg_pool, ucl_object_tostring (val)); + cfg->log_file = rspamd_mempool_strdup (cfg->cfg_pool, + ucl_object_tostring (val)); } else if (g_ascii_strcasecmp (log_type, "syslog") == 0) { /* Need to get facility */ @@ -58,74 +62,84 @@ rspamd_rcl_logging_handler (struct rspamd_config *cfg, const ucl_object_t *obj, val = ucl_object_find_key (obj, "facility"); if (val != NULL && ucl_object_tostring_safe (val, &facility)) { if (g_ascii_strcasecmp (facility, "LOG_AUTH") == 0 || - g_ascii_strcasecmp (facility, "auth") == 0 ) { + g_ascii_strcasecmp (facility, "auth") == 0 ) { cfg->log_facility = LOG_AUTH; } else if (g_ascii_strcasecmp (facility, "LOG_CRON") == 0 || - g_ascii_strcasecmp (facility, "cron") == 0 ) { + g_ascii_strcasecmp (facility, "cron") == 0 ) { cfg->log_facility = LOG_CRON; } else if (g_ascii_strcasecmp (facility, "LOG_DAEMON") == 0 || - g_ascii_strcasecmp (facility, "daemon") == 0 ) { + g_ascii_strcasecmp (facility, "daemon") == 0 ) { cfg->log_facility = LOG_DAEMON; } else if (g_ascii_strcasecmp (facility, "LOG_MAIL") == 0 || - g_ascii_strcasecmp (facility, "mail") == 0) { + g_ascii_strcasecmp (facility, "mail") == 0) { cfg->log_facility = LOG_MAIL; } else if (g_ascii_strcasecmp (facility, "LOG_USER") == 0 || - g_ascii_strcasecmp (facility, "user") == 0 ) { + g_ascii_strcasecmp (facility, "user") == 0 ) { cfg->log_facility = LOG_USER; } else if (g_ascii_strcasecmp (facility, "LOG_LOCAL0") == 0 || - g_ascii_strcasecmp (facility, "local0") == 0) { + g_ascii_strcasecmp (facility, "local0") == 0) { cfg->log_facility = LOG_LOCAL0; } else if (g_ascii_strcasecmp (facility, "LOG_LOCAL1") == 0 || - g_ascii_strcasecmp (facility, "local1") == 0) { + g_ascii_strcasecmp (facility, "local1") == 0) { cfg->log_facility = LOG_LOCAL1; } else if (g_ascii_strcasecmp (facility, "LOG_LOCAL2") == 0 || - g_ascii_strcasecmp (facility, "local2") == 0) { + g_ascii_strcasecmp (facility, "local2") == 0) { cfg->log_facility = LOG_LOCAL2; } else if (g_ascii_strcasecmp (facility, "LOG_LOCAL3") == 0 || - g_ascii_strcasecmp (facility, "local3") == 0) { + g_ascii_strcasecmp (facility, "local3") == 0) { cfg->log_facility = LOG_LOCAL3; } else if (g_ascii_strcasecmp (facility, "LOG_LOCAL4") == 0 || - g_ascii_strcasecmp (facility, "local4") == 0) { + g_ascii_strcasecmp (facility, "local4") == 0) { cfg->log_facility = LOG_LOCAL4; } else if (g_ascii_strcasecmp (facility, "LOG_LOCAL5") == 0 || - g_ascii_strcasecmp (facility, "local5") == 0) { + g_ascii_strcasecmp (facility, "local5") == 0) { cfg->log_facility = LOG_LOCAL5; } else if (g_ascii_strcasecmp (facility, "LOG_LOCAL6") == 0 || - g_ascii_strcasecmp (facility, "local6") == 0) { + g_ascii_strcasecmp (facility, "local6") == 0) { cfg->log_facility = LOG_LOCAL6; } else if (g_ascii_strcasecmp (facility, "LOG_LOCAL7") == 0 || - g_ascii_strcasecmp (facility, "local7") == 0) { + g_ascii_strcasecmp (facility, "local7") == 0) { cfg->log_facility = LOG_LOCAL7; } else { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "invalid log facility: %s", facility); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "invalid log facility: %s", + facility); return FALSE; } } } - else if (g_ascii_strcasecmp (log_type, "stderr") == 0 || g_ascii_strcasecmp (log_type, "console") == 0) { + else if (g_ascii_strcasecmp (log_type, + "stderr") == 0 || g_ascii_strcasecmp (log_type, "console") == 0) { cfg->log_type = RSPAMD_LOG_CONSOLE; } else { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "invalid log type: %s", log_type); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "invalid log type: %s", + log_type); return FALSE; } } else { /* No type specified */ - msg_warn ("logging type is not specified correctly, log output to the console"); + msg_warn ( + "logging type is not specified correctly, log output to the console"); } /* Handle log level */ @@ -144,7 +158,11 @@ rspamd_rcl_logging_handler (struct rspamd_config *cfg, const ucl_object_t *obj, cfg->log_level = G_LOG_LEVEL_DEBUG; } else { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "invalid log level: %s", log_level); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "invalid log level: %s", + log_level); return FALSE; } } @@ -154,7 +172,7 @@ rspamd_rcl_logging_handler (struct rspamd_config *cfg, const ucl_object_t *obj, static gboolean rspamd_rcl_options_handler (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) + gpointer ud, struct rspamd_rcl_section *section, GError **err) { const ucl_object_t *dns; struct rspamd_rcl_section *dns_section; @@ -163,7 +181,8 @@ rspamd_rcl_options_handler (struct rspamd_config *cfg, const ucl_object_t *obj, dns = ucl_object_find_key (obj, "dns"); if (dns_section != NULL && dns != NULL) { - if (!rspamd_rcl_section_parse_defaults (dns_section, cfg, dns, cfg, err)) { + if (!rspamd_rcl_section_parse_defaults (dns_section, cfg, dns, cfg, + err)) { return FALSE; } } @@ -174,8 +193,8 @@ rspamd_rcl_options_handler (struct rspamd_config *cfg, const ucl_object_t *obj, static gint rspamd_symbols_group_find_func (gconstpointer a, gconstpointer b) { - const struct rspamd_symbols_group *gr = a; - const gchar *uv = b; + const struct rspamd_symbols_group *gr = a; + const gchar *uv = b; return g_ascii_strcasecmp (gr->name, uv); } @@ -190,7 +209,7 @@ rspamd_symbols_group_find_func (gconstpointer a, gconstpointer b) */ static gboolean rspamd_rcl_insert_symbol (struct rspamd_config *cfg, struct metric *metric, - const ucl_object_t *obj, gboolean is_legacy, GError **err) + const ucl_object_t *obj, gboolean is_legacy, GError **err) { const gchar *group = "ungrouped", *description = NULL, *sym_name; gdouble symbol_score, *score_ptr; @@ -226,7 +245,11 @@ rspamd_rcl_insert_symbol (struct rspamd_config *cfg, struct metric *metric, else if (obj->type == UCL_OBJECT) { val = ucl_object_find_key (obj, "weight"); if (val == NULL || !ucl_object_todouble_safe (val, &symbol_score)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "invalid symbol score: %s", sym_name); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "invalid symbol score: %s", + sym_name); return FALSE; } val = ucl_object_find_key (obj, "description"); @@ -239,11 +262,16 @@ rspamd_rcl_insert_symbol (struct rspamd_config *cfg, struct metric *metric, } } else { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "invalid symbol type: %s", sym_name); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "invalid symbol type: %s", + sym_name); return FALSE; } - sym_def = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (struct rspamd_symbol_def)); + sym_def = + rspamd_mempool_alloc (cfg->cfg_pool, sizeof (struct rspamd_symbol_def)); score_ptr = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (gdouble)); *score_ptr = symbol_score; @@ -253,9 +281,12 @@ rspamd_rcl_insert_symbol (struct rspamd_config *cfg, struct metric *metric, g_hash_table_insert (metric->symbols, sym_def->name, score_ptr); - if ((metric_list = g_hash_table_lookup (cfg->metrics_symbols, sym_def->name)) == NULL) { + if ((metric_list = + g_hash_table_lookup (cfg->metrics_symbols, sym_def->name)) == NULL) { metric_list = g_list_prepend (NULL, metric); - rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)g_list_free, metric_list); + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t)g_list_free, + metric_list); g_hash_table_insert (cfg->metrics_symbols, sym_def->name, metric_list); } else { @@ -266,10 +297,14 @@ rspamd_rcl_insert_symbol (struct rspamd_config *cfg, struct metric *metric, } /* Search for symbol group */ - group_list = g_list_find_custom (cfg->symbols_groups, group, rspamd_symbols_group_find_func); + group_list = g_list_find_custom (cfg->symbols_groups, + group, + rspamd_symbols_group_find_func); if (group_list == NULL) { /* Create new group */ - sym_group = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (struct rspamd_symbols_group)); + sym_group = + rspamd_mempool_alloc (cfg->cfg_pool, + sizeof (struct rspamd_symbols_group)); sym_group->name = rspamd_mempool_strdup (cfg->cfg_pool, group); sym_group->symbols = NULL; cfg->symbols_groups = g_list_prepend (cfg->symbols_groups, sym_group); @@ -285,7 +320,7 @@ rspamd_rcl_insert_symbol (struct rspamd_config *cfg, struct metric *metric, static gboolean rspamd_rcl_metric_handler (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) + gpointer ud, struct rspamd_rcl_section *section, GError **err) { const ucl_object_t *val, *cur; const gchar *metric_name, *subject_name, *semicolon, *act_str; @@ -314,13 +349,18 @@ rspamd_rcl_metric_handler (struct rspamd_config *cfg, const ucl_object_t *obj, val = ucl_object_find_key (obj, "actions"); if (val != NULL) { if (val->type != UCL_OBJECT) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "actions must be an object"); + g_set_error (err, CFG_RCL_ERROR, EINVAL, + "actions must be an object"); return FALSE; } while ((cur = ucl_iterate_object (val, &it, true)) != NULL) { if (!check_action_str (ucl_object_key (cur), &action_value) || - !ucl_object_todouble_safe (cur, &action_score)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "invalid action definition: %s", ucl_object_key (cur)); + !ucl_object_todouble_safe (cur, &action_score)) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "invalid action definition: %s", + ucl_object_key (cur)); return FALSE; } action = &metric->actions[action_value]; @@ -338,7 +378,8 @@ rspamd_rcl_metric_handler (struct rspamd_config *cfg, const ucl_object_t *obj, have_actions = TRUE; } val = ucl_object_find_key (obj, "action"); - LL_FOREACH (val, cur) { + LL_FOREACH (val, cur) + { if (cur->type == UCL_STRING) { act_str = ucl_object_tostring (cur); semicolon = strchr (act_str, ':'); @@ -354,7 +395,11 @@ rspamd_rcl_metric_handler (struct rspamd_config *cfg, const ucl_object_t *obj, } } if (new && !have_actions) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "metric %s has no actions", metric_name); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "metric %s has no actions", + metric_name); return FALSE; } } @@ -366,7 +411,8 @@ rspamd_rcl_metric_handler (struct rspamd_config *cfg, const ucl_object_t *obj, val = val->value.ov; } if (val->type != UCL_OBJECT) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "symbols must be an object"); + g_set_error (err, CFG_RCL_ERROR, EINVAL, + "symbols must be an object"); return FALSE; } it = NULL; @@ -384,17 +430,25 @@ rspamd_rcl_metric_handler (struct rspamd_config *cfg, const ucl_object_t *obj, val = val->value.ov; } if (val->type != UCL_OBJECT) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "symbols must be an object"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "symbols must be an object"); return FALSE; } - LL_FOREACH (val, cur) { + LL_FOREACH (val, cur) + { if (!rspamd_rcl_insert_symbol (cfg, metric, cur, TRUE, err)) { return FALSE; } } } else if (new) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "metric %s has no symbols", metric_name); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "metric %s has no symbols", + metric_name); return FALSE; } } @@ -420,7 +474,7 @@ rspamd_rcl_metric_handler (struct rspamd_config *cfg, const ucl_object_t *obj, static gboolean rspamd_rcl_worker_handler (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) + gpointer ud, struct rspamd_rcl_section *section, GError **err) { const ucl_object_t *val, *cur; ucl_object_iter_t it = NULL; @@ -437,7 +491,11 @@ rspamd_rcl_worker_handler (struct rspamd_config *cfg, const ucl_object_t *obj, wrk = rspamd_config_new_worker (cfg, NULL); wrk->worker = rspamd_get_worker_by_type (qtype); if (wrk->worker == NULL) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "unknown worker type: %s", worker_type); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "unknown worker type: %s", + worker_type); return FALSE; } wrk->type = qtype; @@ -446,7 +504,11 @@ rspamd_rcl_worker_handler (struct rspamd_config *cfg, const ucl_object_t *obj, } } else { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "unknown worker type: %s", worker_type); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "unknown worker type: %s", + worker_type); return FALSE; } } @@ -460,12 +522,17 @@ rspamd_rcl_worker_handler (struct rspamd_config *cfg, const ucl_object_t *obj, if (val->type == UCL_ARRAY) { val = val->value.ov; } - LL_FOREACH (val, cur) { + LL_FOREACH (val, cur) + { if (!ucl_object_tostring_safe (cur, &worker_bind)) { continue; } if (!rspamd_parse_bind_line (cfg, wrk, worker_bind)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot parse bind line: %s", worker_bind); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot parse bind line: %s", + worker_bind); return FALSE; } } @@ -483,13 +550,14 @@ rspamd_rcl_worker_handler (struct rspamd_config *cfg, const ucl_object_t *obj, while ((cur = ucl_iterate_object (obj, &it, true)) != NULL) { HASH_FIND_STR (wparser->parsers, ucl_object_key (cur), whandler); if (whandler != NULL) { - if (!whandler->handler (cfg, cur, &whandler->parser, section, err)) { + if (!whandler->handler (cfg, cur, &whandler->parser, section, + err)) { return FALSE; } } } if (wparser->def_obj_parser != NULL) { - if (! wparser->def_obj_parser (obj, wparser->def_ud)) { + if (!wparser->def_obj_parser (obj, wparser->def_ud)) { return FALSE; } } @@ -503,7 +571,7 @@ rspamd_rcl_worker_handler (struct rspamd_config *cfg, const ucl_object_t *obj, static void rspamd_rcl_set_lua_globals (struct rspamd_config *cfg, lua_State *L) { - struct rspamd_config **pcfg; + struct rspamd_config **pcfg; /* First check for global variable 'config' */ lua_getglobal (L, "config"); @@ -542,9 +610,10 @@ rspamd_rcl_set_lua_globals (struct rspamd_config *cfg, lua_State *L) static gboolean rspamd_rcl_lua_handler (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) + gpointer ud, struct rspamd_rcl_section *section, GError **err) { - const gchar *lua_src = rspamd_mempool_strdup (cfg->cfg_pool, ucl_object_tostring (obj)); + const gchar *lua_src = rspamd_mempool_strdup (cfg->cfg_pool, + ucl_object_tostring (obj)); gchar *cur_dir, *lua_dir, *lua_file, *tmp1, *tmp2; lua_State *L = cfg->lua_state; @@ -557,10 +626,15 @@ rspamd_rcl_lua_handler (struct rspamd_config *cfg, const ucl_object_t *obj, if (getcwd (cur_dir, PATH_MAX) != NULL && chdir (lua_dir) != -1) { /* Load file */ if (luaL_loadfile (L, lua_file) != 0) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot load lua file %s: %s", - lua_src, lua_tostring (L, -1)); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot load lua file %s: %s", + lua_src, + lua_tostring (L, -1)); if (chdir (cur_dir) == -1) { - msg_err ("cannot chdir to %s: %s", cur_dir, strerror (errno));; + msg_err ("cannot chdir to %s: %s", cur_dir, + strerror (errno));; } g_free (cur_dir); g_free (tmp1); @@ -570,10 +644,15 @@ rspamd_rcl_lua_handler (struct rspamd_config *cfg, const ucl_object_t *obj, rspamd_rcl_set_lua_globals (cfg, L); /* Now do it */ if (lua_pcall (L, 0, LUA_MULTRET, 0) != 0) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot init lua file %s: %s", - lua_src, lua_tostring (L, -1)); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot init lua file %s: %s", + lua_src, + lua_tostring (L, -1)); if (chdir (cur_dir) == -1) { - msg_err ("cannot chdir to %s: %s", cur_dir, strerror (errno));; + msg_err ("cannot chdir to %s: %s", cur_dir, + strerror (errno));; } g_free (cur_dir); g_free (tmp1); @@ -583,7 +662,7 @@ rspamd_rcl_lua_handler (struct rspamd_config *cfg, const ucl_object_t *obj, } else { g_set_error (err, CFG_RCL_ERROR, ENOENT, "cannot chdir to %s: %s", - lua_src, strerror (errno)); + lua_src, strerror (errno)); if (chdir (cur_dir) == -1) { msg_err ("cannot chdir to %s: %s", cur_dir, strerror (errno));; } @@ -602,7 +681,7 @@ rspamd_rcl_lua_handler (struct rspamd_config *cfg, const ucl_object_t *obj, } else { g_set_error (err, CFG_RCL_ERROR, ENOENT, "cannot find to %s: %s", - lua_src, strerror (errno)); + lua_src, strerror (errno)); return FALSE; } @@ -610,7 +689,9 @@ rspamd_rcl_lua_handler (struct rspamd_config *cfg, const ucl_object_t *obj, } static gboolean -rspamd_rcl_add_module_path (struct rspamd_config *cfg, const gchar *path, GError **err) +rspamd_rcl_add_module_path (struct rspamd_config *cfg, + const gchar *path, + GError **err) { struct stat st; struct script_module *cur_mod; @@ -620,7 +701,12 @@ rspamd_rcl_add_module_path (struct rspamd_config *cfg, const gchar *path, GError guint i; if (stat (path, &st) == -1) { - g_set_error (err, CFG_RCL_ERROR, errno, "cannot stat path %s, %s", path, strerror (errno)); + g_set_error (err, + CFG_RCL_ERROR, + errno, + "cannot stat path %s, %s", + path, + strerror (errno)); return FALSE; } @@ -632,23 +718,33 @@ rspamd_rcl_add_module_path (struct rspamd_config *cfg, const gchar *path, GError snprintf (pattern, len, "%s%s", path, "*.lua"); if (glob (pattern, GLOB_DOOFFS, NULL, &globbuf) == 0) { - for (i = 0; i < globbuf.gl_pathc; i ++) { - cur_mod = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (struct script_module)); - cur_mod->path = rspamd_mempool_strdup (cfg->cfg_pool, globbuf.gl_pathv[i]); - cfg->script_modules = g_list_prepend (cfg->script_modules, cur_mod); + for (i = 0; i < globbuf.gl_pathc; i++) { + cur_mod = + rspamd_mempool_alloc (cfg->cfg_pool, + sizeof (struct script_module)); + cur_mod->path = rspamd_mempool_strdup (cfg->cfg_pool, + globbuf.gl_pathv[i]); + cfg->script_modules = g_list_prepend (cfg->script_modules, + cur_mod); } globfree (&globbuf); g_free (pattern); } else { - g_set_error (err, CFG_RCL_ERROR, errno, "glob failed for %s, %s", pattern, strerror (errno)); + g_set_error (err, + CFG_RCL_ERROR, + errno, + "glob failed for %s, %s", + pattern, + strerror (errno)); g_free (pattern); return FALSE; } } else { /* Handle single file */ - cur_mod = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (struct script_module)); + cur_mod = + rspamd_mempool_alloc (cfg->cfg_pool, sizeof (struct script_module)); cur_mod->path = rspamd_mempool_strdup (cfg->cfg_pool, path); cfg->script_modules = g_list_prepend (cfg->script_modules, cur_mod); } @@ -658,7 +754,7 @@ rspamd_rcl_add_module_path (struct rspamd_config *cfg, const gchar *path, GError static gboolean rspamd_rcl_modules_handler (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) + gpointer ud, struct rspamd_rcl_section *section, GError **err) { const ucl_object_t *val, *cur; const gchar *data; @@ -666,21 +762,27 @@ rspamd_rcl_modules_handler (struct rspamd_config *cfg, const ucl_object_t *obj, if (obj->type == UCL_OBJECT) { val = ucl_object_find_key (obj, "path"); - LL_FOREACH (val, cur) { + LL_FOREACH (val, cur) + { if (ucl_object_tostring_safe (cur, &data)) { - if (!rspamd_rcl_add_module_path (cfg, rspamd_mempool_strdup (cfg->cfg_pool, data), err)) { + if (!rspamd_rcl_add_module_path (cfg, + rspamd_mempool_strdup (cfg->cfg_pool, data), err)) { return FALSE; } } } } else if (ucl_object_tostring_safe (obj, &data)) { - if (!rspamd_rcl_add_module_path (cfg, rspamd_mempool_strdup (cfg->cfg_pool, data), err)) { + if (!rspamd_rcl_add_module_path (cfg, + rspamd_mempool_strdup (cfg->cfg_pool, data), err)) { return FALSE; } } else { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "module parameter has wrong type (must be an object or a string)"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "module parameter has wrong type (must be an object or a string)"); return FALSE; } @@ -689,7 +791,7 @@ rspamd_rcl_modules_handler (struct rspamd_config *cfg, const ucl_object_t *obj, static gboolean rspamd_rcl_statfile_handler (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) + gpointer ud, struct rspamd_rcl_section *section, GError **err) { struct rspamd_classifier_config *ccf = ud; const ucl_object_t *val; @@ -703,7 +805,9 @@ rspamd_rcl_statfile_handler (struct rspamd_config *cfg, const ucl_object_t *obj, val = ucl_object_find_key (obj, "binlog"); if (val != NULL && ucl_object_tostring_safe (val, &data)) { if (st->binlog == NULL) { - st->binlog = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct statfile_binlog_params)); + st->binlog = + rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (struct statfile_binlog_params)); } if (g_ascii_strcasecmp (data, "master") == 0) { st->binlog->affinity = AFFINITY_MASTER; @@ -721,7 +825,8 @@ rspamd_rcl_statfile_handler (struct rspamd_config *cfg, const ucl_object_t *obj, } val = ucl_object_find_key (obj, "binlog_master"); if (val != NULL && ucl_object_tostring_safe (val, &data)) { - if (!rspamd_parse_host_port (cfg->cfg_pool, data, &st->binlog->master_addr, &st->binlog->master_port)) { + if (!rspamd_parse_host_port (cfg->cfg_pool, data, + &st->binlog->master_addr, &st->binlog->master_port)) { msg_err ("cannot parse master address: %s", data); return FALSE; } @@ -737,19 +842,26 @@ rspamd_rcl_statfile_handler (struct rspamd_config *cfg, const ucl_object_t *obj, labels = g_list_append (labels, st); } else { - g_hash_table_insert (ccf->labels, st->label, g_list_prepend (NULL, st)); + g_hash_table_insert (ccf->labels, st->label, + g_list_prepend (NULL, st)); } } if (st->symbol != NULL) { g_hash_table_insert (cfg->classifiers_symbols, st->symbol, st); } else { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "statfile must have a symbol defined"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "statfile must have a symbol defined"); return FALSE; } if (st->path == NULL) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "statfile must have a path defined"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "statfile must have a path defined"); return FALSE; } @@ -757,19 +869,29 @@ rspamd_rcl_statfile_handler (struct rspamd_config *cfg, const ucl_object_t *obj, val = ucl_object_find_key (obj, "spam"); if (val == NULL) { - msg_info ("statfile %s has no explicit 'spam' setting, trying to guess by symbol", st->symbol); - if (rspamd_strncasestr (st->symbol, "spam", strlen (st->symbol)) != NULL) { + msg_info ( + "statfile %s has no explicit 'spam' setting, trying to guess by symbol", + st->symbol); + if (rspamd_strncasestr (st->symbol, "spam", + strlen (st->symbol)) != NULL) { st->is_spam = TRUE; } - else if (rspamd_strncasestr (st->symbol, "ham", strlen (st->symbol)) != NULL) { + else if (rspamd_strncasestr (st->symbol, "ham", + strlen (st->symbol)) != NULL) { st->is_spam = FALSE; } else { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot guess spam setting from %s", st->symbol); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot guess spam setting from %s", + st->symbol); return FALSE; } - msg_info ("guessed that statfile with symbol %s is %s", st->symbol, st->is_spam ? - "spam" : "ham"); + msg_info ("guessed that statfile with symbol %s is %s", + st->symbol, + st->is_spam ? + "spam" : "ham"); } return TRUE; } @@ -778,8 +900,11 @@ rspamd_rcl_statfile_handler (struct rspamd_config *cfg, const ucl_object_t *obj, } static gboolean -rspamd_rcl_classifier_handler (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) +rspamd_rcl_classifier_handler (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err) { const ucl_object_t *val, *cur; ucl_object_iter_t it = NULL; @@ -791,7 +916,10 @@ rspamd_rcl_classifier_handler (struct rspamd_config *cfg, const ucl_object_t *ob val = ucl_object_find_key (obj, "type"); if (val == NULL || !ucl_object_tostring_safe (val, &type)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "classifier should have type defined"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "classifier should have type defined"); return FALSE; } @@ -819,22 +947,31 @@ rspamd_rcl_classifier_handler (struct rspamd_config *cfg, const ucl_object_t *ob key = ucl_object_key (val); if (key != NULL) { if (g_ascii_strcasecmp (key, "statfile") == 0) { - LL_FOREACH (val, cur) { - res = rspamd_rcl_statfile_handler (cfg, cur, ccf, stat_section, err); + LL_FOREACH (val, cur) + { + res = rspamd_rcl_statfile_handler (cfg, + cur, + ccf, + stat_section, + err); if (!res) { return FALSE; } } } - else if (g_ascii_strcasecmp (key, "type") == 0 && val->type == UCL_STRING) { + else if (g_ascii_strcasecmp (key, + "type") == 0 && val->type == UCL_STRING) { continue; } - else if (g_ascii_strcasecmp (key, "tokenizer") == 0 && val->type == UCL_STRING) { + else if (g_ascii_strcasecmp (key, + "tokenizer") == 0 && val->type == UCL_STRING) { ccf->tokenizer = get_tokenizer (ucl_object_tostring (val)); } else { /* Just insert a value of option to the hash */ - g_hash_table_insert (ccf->opts, (gpointer)key, (gpointer)ucl_object_tostring_forced (val)); + g_hash_table_insert (ccf->opts, + (gpointer)key, + (gpointer)ucl_object_tostring_forced (val)); } } } @@ -848,8 +985,11 @@ rspamd_rcl_classifier_handler (struct rspamd_config *cfg, const ucl_object_t *ob } static gboolean -rspamd_rcl_composite_handler (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) +rspamd_rcl_composite_handler (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err) { const ucl_object_t *val; struct expression *expr; @@ -859,7 +999,10 @@ rspamd_rcl_composite_handler (struct rspamd_config *cfg, const ucl_object_t *obj val = ucl_object_find_key (obj, "name"); if (val == NULL || !ucl_object_tostring_safe (val, &composite_name)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "composite must have a name defined"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "composite must have a name defined"); return FALSE; } @@ -870,19 +1013,31 @@ rspamd_rcl_composite_handler (struct rspamd_config *cfg, const ucl_object_t *obj val = ucl_object_find_key (obj, "expression"); if (val == NULL || !ucl_object_tostring_safe (val, &composite_expression)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "composite must have an expression defined"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "composite must have an expression defined"); return FALSE; } - if ((expr = parse_expression (cfg->cfg_pool, (gchar *)composite_expression)) == NULL) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot parse composite expression: %s", composite_expression); + if ((expr = + parse_expression (cfg->cfg_pool, + (gchar *)composite_expression)) == NULL) { + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot parse composite expression: %s", + composite_expression); return FALSE; } - composite = rspamd_mempool_alloc (cfg->cfg_pool, sizeof (struct rspamd_composite)); + composite = + rspamd_mempool_alloc (cfg->cfg_pool, sizeof (struct rspamd_composite)); composite->expr = expr; composite->id = g_hash_table_size (cfg->composite_symbols) + 1; - g_hash_table_insert (cfg->composite_symbols, (gpointer)composite_name, composite); + g_hash_table_insert (cfg->composite_symbols, + (gpointer)composite_name, + composite); if (new) { register_virtual_symbol (&cfg->cache, composite_name, 1); @@ -897,7 +1052,7 @@ rspamd_rcl_composite_handler (struct rspamd_config *cfg, const ucl_object_t *obj */ static gboolean rspamd_rcl_empty_handler (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) + gpointer ud, struct rspamd_rcl_section *section, GError **err) { return rspamd_rcl_section_parse_defaults (section, cfg, obj, cfg, err); } @@ -912,10 +1067,10 @@ rspamd_rcl_empty_handler (struct rspamd_config *cfg, const ucl_object_t *obj, * @param strict_type turn on strict check for types for this section * @return newly created structure */ -static inline struct rspamd_rcl_section* +static inline struct rspamd_rcl_section * rspamd_rcl_add_section (struct rspamd_rcl_section **top, - const gchar *name, rspamd_rcl_handler_t handler, - enum ucl_type type, gboolean required, gboolean strict_type) + const gchar *name, rspamd_rcl_handler_t handler, + enum ucl_type type, gboolean required, gboolean strict_type) { struct rspamd_rcl_section *new; @@ -939,8 +1094,11 @@ rspamd_rcl_add_section (struct rspamd_rcl_section **top, * @return newly created structure */ static inline struct rspamd_rcl_default_handler_data * -rspamd_rcl_add_default_handler (struct rspamd_rcl_section *section, const gchar *name, - rspamd_rcl_handler_t handler, gsize offset, gint flags) +rspamd_rcl_add_default_handler (struct rspamd_rcl_section *section, + const gchar *name, + rspamd_rcl_handler_t handler, + gsize offset, + gint flags) { struct rspamd_rcl_default_handler_data *new; @@ -950,11 +1108,12 @@ rspamd_rcl_add_default_handler (struct rspamd_rcl_section *section, const gchar new->pd.offset = offset; new->pd.flags = flags; - HASH_ADD_KEYPTR (hh, section->default_parser, new->key, strlen (new->key), new); + HASH_ADD_KEYPTR (hh, section->default_parser, new->key, strlen ( + new->key), new); return new; } -struct rspamd_rcl_section* +struct rspamd_rcl_section * rspamd_rcl_config_init (void) { struct rspamd_rcl_section *new = NULL, *sub, *ssub; @@ -963,138 +1122,283 @@ rspamd_rcl_config_init (void) /** * Logging section */ - sub = rspamd_rcl_add_section (&new, "logging", rspamd_rcl_logging_handler, UCL_OBJECT, - FALSE, TRUE); + sub = rspamd_rcl_add_section (&new, + "logging", + rspamd_rcl_logging_handler, + UCL_OBJECT, + FALSE, + TRUE); /* Default handlers */ - rspamd_rcl_add_default_handler (sub, "log_buffer", rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, log_buf_size), 0); - rspamd_rcl_add_default_handler (sub, "log_urls", rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, log_urls), 0); - rspamd_rcl_add_default_handler (sub, "debug_ip", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, debug_ip_map), 0); - rspamd_rcl_add_default_handler (sub, "debug_symbols", rspamd_rcl_parse_struct_string_list, - G_STRUCT_OFFSET (struct rspamd_config, debug_symbols), 0); - rspamd_rcl_add_default_handler (sub, "log_color", rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, log_color), 0); + rspamd_rcl_add_default_handler (sub, + "log_buffer", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, log_buf_size), + 0); + rspamd_rcl_add_default_handler (sub, + "log_urls", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, log_urls), + 0); + rspamd_rcl_add_default_handler (sub, + "debug_ip", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, debug_ip_map), + 0); + rspamd_rcl_add_default_handler (sub, + "debug_symbols", + rspamd_rcl_parse_struct_string_list, + G_STRUCT_OFFSET (struct rspamd_config, debug_symbols), + 0); + rspamd_rcl_add_default_handler (sub, + "log_color", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, log_color), + 0); /** * Options section */ - sub = rspamd_rcl_add_section (&new, "options", rspamd_rcl_options_handler, UCL_OBJECT, - FALSE, TRUE); - rspamd_rcl_add_default_handler (sub, "cache_file", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, cache_filename), RSPAMD_CL_FLAG_STRING_PATH); + sub = rspamd_rcl_add_section (&new, + "options", + rspamd_rcl_options_handler, + UCL_OBJECT, + FALSE, + TRUE); + rspamd_rcl_add_default_handler (sub, + "cache_file", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, cache_filename), + RSPAMD_CL_FLAG_STRING_PATH); /* Old DNS configuration */ - rspamd_rcl_add_default_handler (sub, "dns_nameserver", rspamd_rcl_parse_struct_string_list, - G_STRUCT_OFFSET (struct rspamd_config, nameservers), 0); - rspamd_rcl_add_default_handler (sub, "dns_timeout", rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, dns_timeout), RSPAMD_CL_FLAG_TIME_FLOAT); - rspamd_rcl_add_default_handler (sub, "dns_retransmits", rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits), RSPAMD_CL_FLAG_INT_32); - rspamd_rcl_add_default_handler (sub, "dns_sockets", rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server), RSPAMD_CL_FLAG_INT_32); + rspamd_rcl_add_default_handler (sub, + "dns_nameserver", + rspamd_rcl_parse_struct_string_list, + G_STRUCT_OFFSET (struct rspamd_config, nameservers), + 0); + rspamd_rcl_add_default_handler (sub, + "dns_timeout", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, dns_timeout), + RSPAMD_CL_FLAG_TIME_FLOAT); + rspamd_rcl_add_default_handler (sub, + "dns_retransmits", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits), + RSPAMD_CL_FLAG_INT_32); + rspamd_rcl_add_default_handler (sub, + "dns_sockets", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server), + RSPAMD_CL_FLAG_INT_32); /* New DNS configiration */ ssub = rspamd_rcl_add_section (&sub->subsections, "dns", NULL, - UCL_OBJECT, FALSE, TRUE); - rspamd_rcl_add_default_handler (ssub, "nameserver", rspamd_rcl_parse_struct_string_list, - G_STRUCT_OFFSET (struct rspamd_config, nameservers), 0); - rspamd_rcl_add_default_handler (ssub, "timeout", rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, dns_timeout), RSPAMD_CL_FLAG_TIME_FLOAT); - rspamd_rcl_add_default_handler (ssub, "retransmits", rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits), RSPAMD_CL_FLAG_INT_32); - rspamd_rcl_add_default_handler (ssub, "sockets", rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server), RSPAMD_CL_FLAG_INT_32); - - rspamd_rcl_add_default_handler (sub, "raw_mode", rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, raw_mode), 0); - rspamd_rcl_add_default_handler (sub, "one_shot", rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, one_shot_mode), 0); - rspamd_rcl_add_default_handler (sub, "check_attachements", rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements), 0); - rspamd_rcl_add_default_handler (sub, "tempdir", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, temp_dir), RSPAMD_CL_FLAG_STRING_PATH); - rspamd_rcl_add_default_handler (sub, "pidfile", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, pid_file), RSPAMD_CL_FLAG_STRING_PATH); - rspamd_rcl_add_default_handler (sub, "filters", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, filters_str), 0); - rspamd_rcl_add_default_handler (sub, "sync_interval", rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, statfile_sync_interval), RSPAMD_CL_FLAG_TIME_INTEGER); - rspamd_rcl_add_default_handler (sub, "sync_timeout", rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, statfile_sync_timeout), RSPAMD_CL_FLAG_TIME_INTEGER); - rspamd_rcl_add_default_handler (sub, "max_diff", rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_config, max_diff), RSPAMD_CL_FLAG_INT_SIZE); - rspamd_rcl_add_default_handler (sub, "map_watch_interval", rspamd_rcl_parse_struct_time, - G_STRUCT_OFFSET (struct rspamd_config, map_timeout), RSPAMD_CL_FLAG_TIME_FLOAT); - rspamd_rcl_add_default_handler (sub, "dynamic_conf", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, dynamic_conf), 0); + UCL_OBJECT, FALSE, TRUE); + rspamd_rcl_add_default_handler (ssub, + "nameserver", + rspamd_rcl_parse_struct_string_list, + G_STRUCT_OFFSET (struct rspamd_config, nameservers), + 0); + rspamd_rcl_add_default_handler (ssub, + "timeout", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, dns_timeout), + RSPAMD_CL_FLAG_TIME_FLOAT); + rspamd_rcl_add_default_handler (ssub, + "retransmits", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, dns_retransmits), + RSPAMD_CL_FLAG_INT_32); + rspamd_rcl_add_default_handler (ssub, + "sockets", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, dns_io_per_server), + RSPAMD_CL_FLAG_INT_32); + + rspamd_rcl_add_default_handler (sub, + "raw_mode", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, raw_mode), + 0); + rspamd_rcl_add_default_handler (sub, + "one_shot", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, one_shot_mode), + 0); + rspamd_rcl_add_default_handler (sub, + "check_attachements", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, check_text_attachements), + 0); + rspamd_rcl_add_default_handler (sub, + "tempdir", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, temp_dir), + RSPAMD_CL_FLAG_STRING_PATH); + rspamd_rcl_add_default_handler (sub, + "pidfile", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, pid_file), + RSPAMD_CL_FLAG_STRING_PATH); + rspamd_rcl_add_default_handler (sub, + "filters", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, filters_str), + 0); + rspamd_rcl_add_default_handler (sub, + "sync_interval", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, statfile_sync_interval), + RSPAMD_CL_FLAG_TIME_INTEGER); + rspamd_rcl_add_default_handler (sub, + "sync_timeout", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, statfile_sync_timeout), + RSPAMD_CL_FLAG_TIME_INTEGER); + rspamd_rcl_add_default_handler (sub, + "max_diff", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_config, max_diff), + RSPAMD_CL_FLAG_INT_SIZE); + rspamd_rcl_add_default_handler (sub, + "map_watch_interval", + rspamd_rcl_parse_struct_time, + G_STRUCT_OFFSET (struct rspamd_config, map_timeout), + RSPAMD_CL_FLAG_TIME_FLOAT); + rspamd_rcl_add_default_handler (sub, + "dynamic_conf", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, dynamic_conf), + 0); rspamd_rcl_add_default_handler (sub, "rrd", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, rrd_file), RSPAMD_CL_FLAG_STRING_PATH); - rspamd_rcl_add_default_handler (sub, "history_file", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_config, history_file), RSPAMD_CL_FLAG_STRING_PATH); - rspamd_rcl_add_default_handler (sub, "use_mlock", rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, mlock_statfile_pool), 0); - rspamd_rcl_add_default_handler (sub, "strict_protocol_headers", rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_config, strict_protocol_headers), 0); + G_STRUCT_OFFSET (struct rspamd_config, + rrd_file), RSPAMD_CL_FLAG_STRING_PATH); + rspamd_rcl_add_default_handler (sub, + "history_file", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_config, history_file), + RSPAMD_CL_FLAG_STRING_PATH); + rspamd_rcl_add_default_handler (sub, + "use_mlock", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, mlock_statfile_pool), + 0); + rspamd_rcl_add_default_handler (sub, + "strict_protocol_headers", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_config, strict_protocol_headers), + 0); /** * Metric section */ - sub = rspamd_rcl_add_section (&new, "metric", rspamd_rcl_metric_handler, UCL_OBJECT, - FALSE, TRUE); + sub = rspamd_rcl_add_section (&new, + "metric", + rspamd_rcl_metric_handler, + UCL_OBJECT, + FALSE, + TRUE); /** * Worker section */ - sub = rspamd_rcl_add_section (&new, "worker", rspamd_rcl_worker_handler, UCL_OBJECT, - FALSE, TRUE); - rspamd_rcl_add_default_handler (sub, "count", rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_worker_conf, count), RSPAMD_CL_FLAG_INT_16); - rspamd_rcl_add_default_handler (sub, "max_files", rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_nofile), RSPAMD_CL_FLAG_INT_32); - rspamd_rcl_add_default_handler (sub, "max_core", rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_maxcore), RSPAMD_CL_FLAG_INT_32); + sub = rspamd_rcl_add_section (&new, + "worker", + rspamd_rcl_worker_handler, + UCL_OBJECT, + FALSE, + TRUE); + rspamd_rcl_add_default_handler (sub, + "count", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_worker_conf, count), + RSPAMD_CL_FLAG_INT_16); + rspamd_rcl_add_default_handler (sub, + "max_files", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_nofile), + RSPAMD_CL_FLAG_INT_32); + rspamd_rcl_add_default_handler (sub, + "max_core", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_worker_conf, rlimit_maxcore), + RSPAMD_CL_FLAG_INT_32); /** * Lua handler */ - sub = rspamd_rcl_add_section (&new, "lua", rspamd_rcl_lua_handler, UCL_STRING, - FALSE, TRUE); + sub = rspamd_rcl_add_section (&new, + "lua", + rspamd_rcl_lua_handler, + UCL_STRING, + FALSE, + TRUE); /** * Modules handler */ - sub = rspamd_rcl_add_section (&new, "modules", rspamd_rcl_modules_handler, UCL_OBJECT, - FALSE, FALSE); + sub = rspamd_rcl_add_section (&new, + "modules", + rspamd_rcl_modules_handler, + UCL_OBJECT, + FALSE, + FALSE); /** * Classifiers handler */ - sub = rspamd_rcl_add_section (&new, "classifier", rspamd_rcl_classifier_handler, UCL_OBJECT, - FALSE, TRUE); - ssub = rspamd_rcl_add_section (&sub->subsections, "statfile", rspamd_rcl_statfile_handler, - UCL_OBJECT, TRUE, TRUE); - rspamd_rcl_add_default_handler (ssub, "symbol", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_statfile_config, symbol), 0); - rspamd_rcl_add_default_handler (ssub, "path", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_statfile_config, path), RSPAMD_CL_FLAG_STRING_PATH); - rspamd_rcl_add_default_handler (ssub, "label", rspamd_rcl_parse_struct_string, - G_STRUCT_OFFSET (struct rspamd_statfile_config, label), 0); - rspamd_rcl_add_default_handler (ssub, "size", rspamd_rcl_parse_struct_integer, - G_STRUCT_OFFSET (struct rspamd_statfile_config, size), RSPAMD_CL_FLAG_INT_SIZE); - rspamd_rcl_add_default_handler (ssub, "spam", rspamd_rcl_parse_struct_boolean, - G_STRUCT_OFFSET (struct rspamd_statfile_config, is_spam), 0); + sub = rspamd_rcl_add_section (&new, + "classifier", + rspamd_rcl_classifier_handler, + UCL_OBJECT, + FALSE, + TRUE); + ssub = rspamd_rcl_add_section (&sub->subsections, + "statfile", + rspamd_rcl_statfile_handler, + UCL_OBJECT, + TRUE, + TRUE); + rspamd_rcl_add_default_handler (ssub, + "symbol", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_statfile_config, symbol), + 0); + rspamd_rcl_add_default_handler (ssub, + "path", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_statfile_config, path), + RSPAMD_CL_FLAG_STRING_PATH); + rspamd_rcl_add_default_handler (ssub, + "label", + rspamd_rcl_parse_struct_string, + G_STRUCT_OFFSET (struct rspamd_statfile_config, label), + 0); + rspamd_rcl_add_default_handler (ssub, + "size", + rspamd_rcl_parse_struct_integer, + G_STRUCT_OFFSET (struct rspamd_statfile_config, size), + RSPAMD_CL_FLAG_INT_SIZE); + rspamd_rcl_add_default_handler (ssub, + "spam", + rspamd_rcl_parse_struct_boolean, + G_STRUCT_OFFSET (struct rspamd_statfile_config, is_spam), + 0); /** * Composites handler */ - sub = rspamd_rcl_add_section (&new, "composite", rspamd_rcl_composite_handler, UCL_OBJECT, - FALSE, TRUE); + sub = rspamd_rcl_add_section (&new, + "composite", + rspamd_rcl_composite_handler, + UCL_OBJECT, + FALSE, + TRUE); return new; } struct rspamd_rcl_section * rspamd_rcl_config_get_section (struct rspamd_rcl_section *top, - const char *path) + const char *path) { struct rspamd_rcl_section *cur, *found; char **path_components; @@ -1109,7 +1413,7 @@ rspamd_rcl_config_get_section (struct rspamd_rcl_section *top, ncomponents = g_strv_length (path_components); cur = top; - for (i = 0; i < ncomponents; i ++) { + for (i = 0; i < ncomponents; i++) { if (cur == NULL) { g_strfreev (path_components); return NULL; @@ -1128,23 +1432,27 @@ rspamd_rcl_config_get_section (struct rspamd_rcl_section *top, gboolean rspamd_read_rcl_config (struct rspamd_rcl_section *top, - struct rspamd_config *cfg, const ucl_object_t *obj, GError **err) + struct rspamd_config *cfg, const ucl_object_t *obj, GError **err) { const ucl_object_t *found, *cur_obj; struct rspamd_rcl_section *cur, *tmp; if (obj->type != UCL_OBJECT) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "top configuration must be an object"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "top configuration must be an object"); return FALSE; } /* Iterate over known sections and ignore unknown ones */ - HASH_ITER (hh, top, cur, tmp) { + HASH_ITER (hh, top, cur, tmp) + { found = ucl_object_find_key (obj, cur->name); if (found == NULL) { if (cur->required) { g_set_error (err, CFG_RCL_ERROR, ENOENT, - "required section %s is missing", cur->name); + "required section %s is missing", cur->name); return FALSE; } } @@ -1153,18 +1461,23 @@ rspamd_read_rcl_config (struct rspamd_rcl_section *top, if (cur->strict_type) { if (cur->type != found->type) { g_set_error (err, CFG_RCL_ERROR, EINVAL, - "object in section %s has invalid type", cur->name); + "object in section %s has invalid type", cur->name); return FALSE; } } - LL_FOREACH (found, cur_obj) { + LL_FOREACH (found, cur_obj) + { if (cur->handler != NULL) { if (!cur->handler (cfg, cur_obj, NULL, cur, err)) { return FALSE; } } else { - rspamd_rcl_section_parse_defaults (cur, cfg, cur_obj, cfg, err); + rspamd_rcl_section_parse_defaults (cur, + cfg, + cur_obj, + cfg, + err); } } } @@ -1178,19 +1491,24 @@ rspamd_read_rcl_config (struct rspamd_rcl_section *top, return TRUE; } -gboolean rspamd_rcl_section_parse_defaults (struct rspamd_rcl_section *section, - struct rspamd_config *cfg, const ucl_object_t *obj, gpointer ptr, - GError **err) +gboolean +rspamd_rcl_section_parse_defaults (struct rspamd_rcl_section *section, + struct rspamd_config *cfg, const ucl_object_t *obj, gpointer ptr, + GError **err) { const ucl_object_t *found; struct rspamd_rcl_default_handler_data *cur, *tmp; if (obj->type != UCL_OBJECT) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "default configuration must be an object"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "default configuration must be an object"); return FALSE; } - HASH_ITER (hh, section->default_parser, cur, tmp) { + HASH_ITER (hh, section->default_parser, cur, tmp) + { found = ucl_object_find_key (obj, cur->key); if (found != NULL) { cur->pd.user_struct = ptr; @@ -1204,8 +1522,11 @@ gboolean rspamd_rcl_section_parse_defaults (struct rspamd_rcl_section *section, } gboolean -rspamd_rcl_parse_struct_string (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) +rspamd_rcl_parse_struct_string (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err) { struct rspamd_rcl_struct_parser *pd = ud; gchar **target; @@ -1214,7 +1535,8 @@ rspamd_rcl_parse_struct_string (struct rspamd_config *cfg, const ucl_object_t *o target = (gchar **)(((gchar *)pd->user_struct) + pd->offset); switch (obj->type) { case UCL_STRING: - *target = rspamd_mempool_strdup (cfg->cfg_pool, ucl_copy_value_trash (obj)); + *target = + rspamd_mempool_strdup (cfg->cfg_pool, ucl_copy_value_trash (obj)); break; case UCL_INT: *target = rspamd_mempool_alloc (cfg->cfg_pool, num_str_len); @@ -1229,7 +1551,10 @@ rspamd_rcl_parse_struct_string (struct rspamd_config *cfg, const ucl_object_t *o rspamd_snprintf (*target, num_str_len, "%b", (gboolean)obj->value.iv); break; default: - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot convert object or array to string"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot convert object or array to string"); return FALSE; } @@ -1237,8 +1562,11 @@ rspamd_rcl_parse_struct_string (struct rspamd_config *cfg, const ucl_object_t *o } gboolean -rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) +rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err) { struct rspamd_rcl_struct_parser *pd = ud; union { @@ -1253,7 +1581,10 @@ rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, const ucl_object_t * if (pd->flags == RSPAMD_CL_FLAG_INT_32) { target.i32p = (gint32 *)(((gchar *)pd->user_struct) + pd->offset); if (!ucl_object_toint_safe (obj, &val)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot convert param to integer"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot convert param to integer"); return FALSE; } *target.i32p = val; @@ -1261,7 +1592,10 @@ rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, const ucl_object_t * else if (pd->flags == RSPAMD_CL_FLAG_INT_64) { target.i64p = (gint64 *)(((gchar *)pd->user_struct) + pd->offset); if (!ucl_object_toint_safe (obj, &val)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot convert param to integer"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot convert param to integer"); return FALSE; } *target.i64p = val; @@ -1269,7 +1603,10 @@ rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, const ucl_object_t * else if (pd->flags == RSPAMD_CL_FLAG_INT_SIZE) { target.sp = (gsize *)(((gchar *)pd->user_struct) + pd->offset); if (!ucl_object_toint_safe (obj, &val)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot convert param to integer"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot convert param to integer"); return FALSE; } *target.sp = val; @@ -1277,7 +1614,10 @@ rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, const ucl_object_t * else if (pd->flags == RSPAMD_CL_FLAG_INT_16) { target.i16p = (gint16 *)(((gchar *)pd->user_struct) + pd->offset); if (!ucl_object_toint_safe (obj, &val)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot convert param to integer"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot convert param to integer"); return FALSE; } *target.i16p = val; @@ -1285,7 +1625,10 @@ rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, const ucl_object_t * else { target.ip = (gint *)(((gchar *)pd->user_struct) + pd->offset); if (!ucl_object_toint_safe (obj, &val)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot convert param to integer"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot convert param to integer"); return FALSE; } *target.ip = val; @@ -1295,8 +1638,11 @@ rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, const ucl_object_t * } gboolean -rspamd_rcl_parse_struct_double (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) +rspamd_rcl_parse_struct_double (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err) { struct rspamd_rcl_struct_parser *pd = ud; gdouble *target; @@ -1304,7 +1650,10 @@ rspamd_rcl_parse_struct_double (struct rspamd_config *cfg, const ucl_object_t *o target = (gdouble *)(((gchar *)pd->user_struct) + pd->offset); if (!ucl_object_todouble_safe (obj, target)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot convert param to double"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot convert param to double"); return FALSE; } @@ -1312,8 +1661,11 @@ rspamd_rcl_parse_struct_double (struct rspamd_config *cfg, const ucl_object_t *o } gboolean -rspamd_rcl_parse_struct_time (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) +rspamd_rcl_parse_struct_time (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err) { struct rspamd_rcl_struct_parser *pd = ud; union { @@ -1326,17 +1678,22 @@ rspamd_rcl_parse_struct_time (struct rspamd_config *cfg, const ucl_object_t *obj gdouble val; if (!ucl_object_todouble_safe (obj, &val)) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot convert param to double"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot convert param to double"); return FALSE; } if (pd->flags == RSPAMD_CL_FLAG_TIME_TIMEVAL) { - target.ptv = (struct timeval *)(((gchar *)pd->user_struct) + pd->offset); + target.ptv = + (struct timeval *)(((gchar *)pd->user_struct) + pd->offset); target.ptv->tv_sec = (glong)val; target.ptv->tv_usec = (val - (glong)val) * 1000000; } else if (pd->flags == RSPAMD_CL_FLAG_TIME_TIMESPEC) { - target.pts = (struct timespec *)(((gchar *)pd->user_struct) + pd->offset); + target.pts = + (struct timespec *)(((gchar *)pd->user_struct) + pd->offset); target.pts->tv_sec = (glong)val; target.pts->tv_nsec = (val - (glong)val) * 1000000000000LL; } @@ -1353,7 +1710,10 @@ rspamd_rcl_parse_struct_time (struct rspamd_config *cfg, const ucl_object_t *obj *target.pu32 = val * 1000; } else { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "invalid flags to parse time value"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "invalid flags to parse time value"); return FALSE; } @@ -1361,8 +1721,11 @@ rspamd_rcl_parse_struct_time (struct rspamd_config *cfg, const ucl_object_t *obj } gboolean -rspamd_rcl_parse_struct_string_list (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) +rspamd_rcl_parse_struct_string_list (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err) { struct rspamd_rcl_struct_parser *pd = ud; GList **target; @@ -1374,14 +1737,19 @@ rspamd_rcl_parse_struct_string_list (struct rspamd_config *cfg, const ucl_object target = (GList **)(((gchar *)pd->user_struct) + pd->offset); if (obj->type != UCL_ARRAY) { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "an array of strings is expected"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "an array of strings is expected"); return FALSE; } while ((cur = ucl_iterate_object (obj, &iter, true)) != NULL) { switch (cur->type) { case UCL_STRING: - val = rspamd_mempool_strdup (cfg->cfg_pool, ucl_copy_value_trash (cur)); + val = + rspamd_mempool_strdup (cfg->cfg_pool, + ucl_copy_value_trash (cur)); break; case UCL_INT: val = rspamd_mempool_alloc (cfg->cfg_pool, num_str_len); @@ -1396,21 +1764,29 @@ rspamd_rcl_parse_struct_string_list (struct rspamd_config *cfg, const ucl_object rspamd_snprintf (val, num_str_len, "%b", (gboolean)cur->value.iv); break; default: - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot convert an object or array to string"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot convert an object or array to string"); return FALSE; } *target = g_list_prepend (*target, val); } /* Add a destructor */ - rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)g_list_free, *target); + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t)g_list_free, + *target); return TRUE; } gboolean -rspamd_rcl_parse_struct_boolean (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err) +rspamd_rcl_parse_struct_boolean (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err) { struct rspamd_rcl_struct_parser *pd = ud; gboolean *target; @@ -1424,7 +1800,10 @@ rspamd_rcl_parse_struct_boolean (struct rspamd_config *cfg, const ucl_object_t * *target = obj->value.iv; } else { - g_set_error (err, CFG_RCL_ERROR, EINVAL, "cannot convert an object to boolean"); + g_set_error (err, + CFG_RCL_ERROR, + EINVAL, + "cannot convert an object to boolean"); return FALSE; } @@ -1432,8 +1811,13 @@ rspamd_rcl_parse_struct_boolean (struct rspamd_config *cfg, const ucl_object_t * } void -rspamd_rcl_register_worker_option (struct rspamd_config *cfg, gint type, const gchar *name, - rspamd_rcl_handler_t handler, gpointer target, gsize offset, gint flags) +rspamd_rcl_register_worker_option (struct rspamd_config *cfg, + gint type, + const gchar *name, + rspamd_rcl_handler_t handler, + gpointer target, + gsize offset, + gint flags) { struct rspamd_worker_param_parser *nhandler; struct rspamd_worker_cfg_parser *nparser; @@ -1441,18 +1825,24 @@ rspamd_rcl_register_worker_option (struct rspamd_config *cfg, gint type, const g HASH_FIND_INT (cfg->wrk_parsers, &type, nparser); if (nparser == NULL) { /* Allocate new parser for this worker */ - nparser = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_worker_cfg_parser)); + nparser = + rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (struct rspamd_worker_cfg_parser)); nparser->type = type; HASH_ADD_INT (cfg->wrk_parsers, type, nparser); } HASH_FIND_STR (nparser->parsers, name, nhandler); if (nhandler != NULL) { - msg_warn ("handler for parameter %s is already registered for worker type %s", - name, g_quark_to_string (type)); + msg_warn ( + "handler for parameter %s is already registered for worker type %s", + name, + g_quark_to_string (type)); return; } - nhandler = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_worker_param_parser)); + nhandler = + rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (struct rspamd_worker_param_parser)); nhandler->name = name; nhandler->parser.flags = flags; nhandler->parser.offset = offset; @@ -1464,13 +1854,15 @@ rspamd_rcl_register_worker_option (struct rspamd_config *cfg, gint type, const g void rspamd_rcl_register_worker_parser (struct rspamd_config *cfg, gint type, - gboolean (*func)(ucl_object_t *, gpointer), gpointer ud) + gboolean (*func)(ucl_object_t *, gpointer), gpointer ud) { struct rspamd_worker_cfg_parser *nparser; HASH_FIND_INT (cfg->wrk_parsers, &type, nparser); if (nparser == NULL) { /* Allocate new parser for this worker */ - nparser = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_worker_cfg_parser)); + nparser = + rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (struct rspamd_worker_cfg_parser)); nparser->type = type; HASH_ADD_INT (cfg->wrk_parsers, type, nparser); } diff --git a/src/libserver/cfg_rcl.h b/src/libserver/cfg_rcl.h index 6db0b0d81..6ba197be8 100644 --- a/src/libserver/cfg_rcl.h +++ b/src/libserver/cfg_rcl.h @@ -63,15 +63,17 @@ struct rspamd_rcl_struct_parser { * @param err error object * @return TRUE if a section has been parsed */ -typedef gboolean (*rspamd_rcl_handler_t) (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err); +typedef gboolean (*rspamd_rcl_handler_t) (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, struct rspamd_rcl_section *section, GError **err); /** * A handler type that is called at the end of section parsing * @param cfg configuration * @param ud user data */ -typedef void (*rspamd_rcl_section_fin_t)(struct rspamd_config *cfg, gpointer ud); +typedef void (*rspamd_rcl_section_fin_t)(struct rspamd_config *cfg, + gpointer ud); struct rspamd_rcl_default_handler_data { struct rspamd_rcl_struct_parser pd; @@ -81,12 +83,12 @@ struct rspamd_rcl_default_handler_data { }; struct rspamd_rcl_section { - const gchar *name; /**< name of section */ - rspamd_rcl_handler_t handler; /**< handler of section attributes */ - enum ucl_type type; /**< type of attribute */ - gboolean required; /**< whether this param is required */ - gboolean strict_type; /**< whether we need strict type */ - UT_hash_handle hh; /** hash handle */ + const gchar *name; /**< name of section */ + rspamd_rcl_handler_t handler; /**< handler of section attributes */ + enum ucl_type type; /**< type of attribute */ + gboolean required; /**< whether this param is required */ + gboolean strict_type; /**< whether we need strict type */ + UT_hash_handle hh; /** hash handle */ struct rspamd_rcl_section *subsections; /**< hash table of subsections */ struct rspamd_rcl_default_handler_data *default_parser; /**< generic parsing fields */ rspamd_rcl_section_fin_t fin; /** called at the end of section parsing */ @@ -97,7 +99,7 @@ struct rspamd_rcl_section { * Init common sections known to rspamd * @return top section */ -struct rspamd_rcl_section* rspamd_rcl_config_init (void); +struct rspamd_rcl_section * rspamd_rcl_config_init (void); /** * Get a section specified by path, it understand paths separated by '/' character @@ -105,8 +107,9 @@ struct rspamd_rcl_section* rspamd_rcl_config_init (void); * @param path '/' divided path * @return */ -struct rspamd_rcl_section *rspamd_rcl_config_get_section (struct rspamd_rcl_section *top, - const char *path); +struct rspamd_rcl_section * rspamd_rcl_config_get_section ( + struct rspamd_rcl_section *top, + const char *path); /** * Read RCL configuration and parse it to a config file @@ -116,7 +119,7 @@ struct rspamd_rcl_section *rspamd_rcl_config_get_section (struct rspamd_rcl_sect * @return TRUE if an object can be parsed */ gboolean rspamd_read_rcl_config (struct rspamd_rcl_section *top, - struct rspamd_config *cfg, const ucl_object_t *obj, GError **err); + struct rspamd_config *cfg, const ucl_object_t *obj, GError **err); /** @@ -129,8 +132,8 @@ gboolean rspamd_read_rcl_config (struct rspamd_rcl_section *top, * @return TRUE if the object has been parsed */ gboolean rspamd_rcl_section_parse_defaults (struct rspamd_rcl_section *section, - struct rspamd_config *cfg, const ucl_object_t *obj, gpointer ptr, - GError **err); + struct rspamd_config *cfg, const ucl_object_t *obj, gpointer ptr, + GError **err); /** * Here is a section of common handlers that accepts rcl_struct_parser * which itself contains a struct pointer and the offset of a member in a @@ -146,8 +149,11 @@ gboolean rspamd_rcl_section_parse_defaults (struct rspamd_rcl_section *section, * @param err error pointer * @return TRUE if a string value has been successfully parsed */ -gboolean rspamd_rcl_parse_struct_string (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err); +gboolean rspamd_rcl_parse_struct_string (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err); /** * Parse an integer field of a structure @@ -158,8 +164,11 @@ gboolean rspamd_rcl_parse_struct_string (struct rspamd_config *cfg, const ucl_ob * @param err error pointer * @return TRUE if a value has been successfully parsed */ -gboolean rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err); +gboolean rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err); /** @@ -171,8 +180,11 @@ gboolean rspamd_rcl_parse_struct_integer (struct rspamd_config *cfg, const ucl_o * @param err error pointer * @return TRUE if a value has been successfully parsed */ -gboolean rspamd_rcl_parse_struct_double (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err); +gboolean rspamd_rcl_parse_struct_double (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err); /** * Parse a time field of a structure @@ -183,8 +195,11 @@ gboolean rspamd_rcl_parse_struct_double (struct rspamd_config *cfg, const ucl_ob * @param err error pointer * @return TRUE if a value has been successfully parsed */ -gboolean rspamd_rcl_parse_struct_time (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err); +gboolean rspamd_rcl_parse_struct_time (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err); /** * Parse a string list field of a structure presented by a GList* object @@ -195,8 +210,11 @@ gboolean rspamd_rcl_parse_struct_time (struct rspamd_config *cfg, const ucl_obje * @param err error pointer * @return TRUE if a value has been successfully parsed */ -gboolean rspamd_rcl_parse_struct_string_list (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err); +gboolean rspamd_rcl_parse_struct_string_list (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err); /** * Parse a boolean field of a structure @@ -207,8 +225,11 @@ gboolean rspamd_rcl_parse_struct_string_list (struct rspamd_config *cfg, const u * @param err error pointer * @return TRUE if a value has been successfully parsed */ -gboolean rspamd_rcl_parse_struct_boolean (struct rspamd_config *cfg, const ucl_object_t *obj, - gpointer ud, struct rspamd_rcl_section *section, GError **err); +gboolean rspamd_rcl_parse_struct_boolean (struct rspamd_config *cfg, + const ucl_object_t *obj, + gpointer ud, + struct rspamd_rcl_section *section, + GError **err); /** * Utility functions @@ -223,8 +244,13 @@ gboolean rspamd_rcl_parse_struct_boolean (struct rspamd_config *cfg, const ucl_o * @param target opaque target structure * @param offset offset inside a structure */ -void rspamd_rcl_register_worker_option (struct rspamd_config *cfg, gint type, const gchar *name, - rspamd_rcl_handler_t handler, gpointer target, gsize offset, gint flags); +void rspamd_rcl_register_worker_option (struct rspamd_config *cfg, + gint type, + const gchar *name, + rspamd_rcl_handler_t handler, + gpointer target, + gsize offset, + gint flags); /** * Regiester a default parser for a worker @@ -234,5 +260,5 @@ void rspamd_rcl_register_worker_option (struct rspamd_config *cfg, gint type, co * @param ud userdata for handler function */ void rspamd_rcl_register_worker_parser (struct rspamd_config *cfg, gint type, - gboolean (*func)(ucl_object_t *, gpointer), gpointer ud); + gboolean (*func)(ucl_object_t *, gpointer), gpointer ud); #endif /* CFG_RCL_H_ */ diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index e1652ac55..15afb9945 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -26,13 +26,13 @@ #include "config.h" #include "cfg_file.h" -#include "main.h" -#include "filter.h" #include "classifiers/classifiers.h" -#include "lua/lua_common.h" +#include "dynamic_cfg.h" +#include "filter.h" #include "kvstorage_config.h" +#include "lua/lua_common.h" +#include "main.h" #include "map.h" -#include "dynamic_cfg.h" #define DEFAULT_SCORE 10.0 @@ -44,18 +44,22 @@ struct rspamd_ucl_map_cbdata { struct rspamd_config *cfg; GString *buf; }; -static gchar* rspamd_ucl_read_cb (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map_cb_data *data); -static void rspamd_ucl_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data); +static gchar * rspamd_ucl_read_cb (rspamd_mempool_t * pool, + gchar * chunk, + gint len, + struct map_cb_data *data); +static void rspamd_ucl_fin_cb (rspamd_mempool_t * pool, + struct map_cb_data *data); static gboolean parse_host_port_priority_strv (rspamd_mempool_t *pool, gchar **tokens, - gchar **addr, guint16 *port, guint *priority, guint default_port) + gchar **addr, guint16 *port, guint *priority, guint default_port) { - gchar *err_str, portbuf[8]; - const gchar *cur_tok, *cur_port; - struct addrinfo hints, *res; - guint port_parsed, priority_parsed, saved_errno = errno; - gint r; + gchar *err_str, portbuf[8]; + const gchar *cur_tok, *cur_port; + struct addrinfo hints, *res; + guint port_parsed, priority_parsed, saved_errno = errno; + gint r; union { struct sockaddr_in v4; struct sockaddr_in6 v6; @@ -90,12 +94,18 @@ parse_host_port_priority_strv (rspamd_mempool_t *pool, gchar **tokens, errno = 0; port_parsed = strtoul (tokens[1], &err_str, 10); if (*err_str != '\0' || errno != 0) { - msg_warn ("cannot parse port: %s, at symbol %c, error: %s", tokens[1], *err_str, strerror (errno)); + msg_warn ("cannot parse port: %s, at symbol %c, error: %s", + tokens[1], + *err_str, + strerror (errno)); hints.ai_flags ^= AI_NUMERICSERV; } else if (port_parsed > G_MAXUINT16) { errno = ERANGE; - msg_warn ("cannot parse port: %s, error: %s", tokens[1], *err_str, strerror (errno)); + msg_warn ("cannot parse port: %s, error: %s", + tokens[1], + *err_str, + strerror (errno)); hints.ai_flags ^= AI_NUMERICSERV; } else { @@ -114,7 +124,11 @@ parse_host_port_priority_strv (rspamd_mempool_t *pool, gchar **tokens, errno = 0; priority_parsed = strtoul (cur_tok, &err_str, 10); if (*err_str != '\0' || errno != 0) { - msg_warn ("cannot parse priority: %s, at symbol %c, error: %s", tokens[1], *err_str, strerror (errno)); + msg_warn ( + "cannot parse priority: %s, at symbol %c, error: %s", + tokens[1], + *err_str, + strerror (errno)); } else { *priority = priority_parsed; @@ -131,23 +145,32 @@ parse_host_port_priority_strv (rspamd_mempool_t *pool, gchar **tokens, } if ((r = getaddrinfo (cur_tok, cur_port, &hints, &res)) == 0) { - memcpy (&addr_holder, res->ai_addr, MIN (sizeof (addr_holder), res->ai_addrlen)); + memcpy (&addr_holder, res->ai_addr, + MIN (sizeof (addr_holder), res->ai_addrlen)); if (res->ai_family == AF_INET) { if (pool != NULL) { *addr = rspamd_mempool_alloc (pool, INET_ADDRSTRLEN + 1); } - inet_ntop (res->ai_family, &addr_holder.v4.sin_addr, *addr, INET_ADDRSTRLEN + 1); + inet_ntop (res->ai_family, + &addr_holder.v4.sin_addr, + *addr, + INET_ADDRSTRLEN + 1); } else { if (pool != NULL) { *addr = rspamd_mempool_alloc (pool, INET6_ADDRSTRLEN + 1); } - inet_ntop (res->ai_family, &addr_holder.v6.sin6_addr, *addr, INET6_ADDRSTRLEN + 1); + inet_ntop (res->ai_family, + &addr_holder.v6.sin6_addr, + *addr, + INET6_ADDRSTRLEN + 1); } freeaddrinfo (res); } else { - msg_err ("address resolution for %s failed: %s", tokens[0], gai_strerror (r)); + msg_err ("address resolution for %s failed: %s", + tokens[0], + gai_strerror (r)); goto err; } @@ -161,10 +184,14 @@ err: } gboolean -rspamd_parse_host_port_priority (rspamd_mempool_t *pool, const gchar *str, gchar **addr, guint16 *port, guint *priority) +rspamd_parse_host_port_priority (rspamd_mempool_t *pool, + const gchar *str, + gchar **addr, + guint16 *port, + guint *priority) { - gchar **tokens; - gboolean ret; + gchar **tokens; + gboolean ret; tokens = g_strsplit_set (str, ":", 0); if (!tokens || !tokens[0]) { @@ -179,19 +206,27 @@ rspamd_parse_host_port_priority (rspamd_mempool_t *pool, const gchar *str, gchar } gboolean -rspamd_parse_host_port (rspamd_mempool_t *pool, const gchar *str, gchar **addr, guint16 *port) +rspamd_parse_host_port (rspamd_mempool_t *pool, + const gchar *str, + gchar **addr, + guint16 *port) { return rspamd_parse_host_port_priority (pool, str, addr, port, NULL); } gboolean -rspamd_parse_host_priority (rspamd_mempool_t *pool, const gchar *str, gchar **addr, guint *priority) +rspamd_parse_host_priority (rspamd_mempool_t *pool, + const gchar *str, + gchar **addr, + guint *priority) { return rspamd_parse_host_port_priority (pool, str, addr, NULL, priority); } gboolean -rspamd_parse_bind_line (struct rspamd_config *cfg, struct rspamd_worker_conf *cf, const gchar *str) +rspamd_parse_bind_line (struct rspamd_config *cfg, + struct rspamd_worker_conf *cf, + const gchar *str) { struct rspamd_worker_bind_conf *cnf; gchar **tokens, *tmp, *err; @@ -206,7 +241,9 @@ rspamd_parse_bind_line (struct rspamd_config *cfg, struct rspamd_worker_conf *cf return FALSE; } - cnf = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_worker_bind_conf)); + cnf = + rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (struct rspamd_worker_bind_conf)); cnf->bind_port = DEFAULT_BIND_PORT; cnf->bind_host = rspamd_mempool_strdup (cfg->cfg_pool, str); cnf->ai = AF_UNSPEC; @@ -221,16 +258,18 @@ rspamd_parse_bind_line (struct rspamd_config *cfg, struct rspamd_worker_conf *cf tokens[0] = "*v4"; cnf->ai = AF_INET; if ((ret = parse_host_port_priority_strv (cfg->cfg_pool, tokens, - &cnf->bind_host, &cnf->bind_port, NULL, DEFAULT_BIND_PORT))) { + &cnf->bind_host, &cnf->bind_port, NULL, DEFAULT_BIND_PORT))) { LL_PREPEND (cf->bind_conf, cnf); } - cnf = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_worker_bind_conf)); + cnf = + rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (struct rspamd_worker_bind_conf)); cnf->bind_port = DEFAULT_BIND_PORT; cnf->bind_host = rspamd_mempool_strdup (cfg->cfg_pool, str); cnf->ai = AF_INET6; tokens[0] = "*v6"; if ((ret &= parse_host_port_priority_strv (cfg->cfg_pool, tokens, - &cnf->bind_host, &cnf->bind_port, NULL, DEFAULT_BIND_PORT))) { + &cnf->bind_host, &cnf->bind_port, NULL, DEFAULT_BIND_PORT))) { LL_PREPEND (cf->bind_conf, cnf); } tokens[0] = tmp; @@ -287,8 +326,10 @@ rspamd_config_defaults (struct rspamd_config *cfg) cfg->metrics = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cfg->c_modules = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); - cfg->composite_symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); - cfg->classifiers_symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); + cfg->composite_symbols = + g_hash_table_new (rspamd_str_hash, rspamd_str_equal); + cfg->classifiers_symbols = g_hash_table_new (rspamd_str_hash, + rspamd_str_equal); cfg->cfg_params = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); cfg->metrics_symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); @@ -301,8 +342,8 @@ rspamd_config_defaults (struct rspamd_config *cfg) void rspamd_config_free (struct rspamd_config *cfg) { - GList *cur; - struct rspamd_symbols_group *gr; + GList *cur; + struct rspamd_symbols_group *gr; remove_all_maps (cfg); ucl_obj_unref (cfg->rcl_obj); @@ -337,8 +378,10 @@ rspamd_config_free (struct rspamd_config *cfg) rspamd_mempool_delete (cfg->cfg_pool); } -const ucl_object_t * -rspamd_config_get_module_opt (struct rspamd_config *cfg, const gchar *module_name, const gchar *opt_name) +const ucl_object_t * +rspamd_config_get_module_opt (struct rspamd_config *cfg, + const gchar *module_name, + const gchar *opt_name) { const ucl_object_t *res = NULL, *sec; @@ -353,8 +396,8 @@ rspamd_config_get_module_opt (struct rspamd_config *cfg, const gchar *module_nam guint64 rspamd_config_parse_limit (const gchar *limit, guint len) { - guint64 result = 0; - const gchar *err_str; + guint64 result = 0; + const gchar *err_str; if (!limit || *limit == '\0' || len == 0) { return 0; @@ -377,7 +420,9 @@ rspamd_config_parse_limit (const gchar *limit, guint len) result *= 1073741824L; } else if (len > 0 && err_str - limit != (gint)len) { - msg_warn ("invalid limit value '%s' at position '%s'", limit, err_str); + msg_warn ("invalid limit value '%s' at position '%s'", + limit, + err_str); result = 0; } } @@ -388,8 +433,8 @@ rspamd_config_parse_limit (const gchar *limit, guint len) gchar rspamd_config_parse_flag (const gchar *str) { - guint len; - gchar c; + guint len; + gchar c; if (!str || !*str) { return -1; @@ -439,15 +484,16 @@ rspamd_config_parse_flag (const gchar *str) } gboolean -rspamd_config_calculate_checksum (struct rspamd_config *cfg) +rspamd_config_calculate_checksum (struct rspamd_config *cfg) { - gint fd; - void *map; - struct stat st; + gint fd; + void *map; + struct stat st; /* Compute checksum for config file that should be used by xml dumper */ if ((fd = open (cfg->cfg_name, O_RDONLY)) == -1) { - msg_err ("config file %s is no longer available, cannot calculate checksum"); + msg_err ( + "config file %s is no longer available, cannot calculate checksum"); return FALSE; } if (stat (cfg->cfg_name, &st) == -1) { @@ -456,37 +502,40 @@ rspamd_config_calculate_checksum (struct rspamd_config *cfg) } /* Now mmap this file to simplify reading process */ - if ((map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { + if ((map = + mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { msg_err ("cannot mmap %s: %s", cfg->cfg_name, strerror (errno)); close (fd); return FALSE; } close (fd); - + /* Get checksum for a file */ - cfg->checksum = g_compute_checksum_for_string (G_CHECKSUM_MD5, map, st.st_size); + cfg->checksum = g_compute_checksum_for_string (G_CHECKSUM_MD5, + map, + st.st_size); munmap (map, st.st_size); - + return TRUE; } -/* +/* * Perform post load actions */ void rspamd_config_post_load (struct rspamd_config *cfg) { #ifdef HAVE_CLOCK_GETTIME - struct timespec ts; + struct timespec ts; #endif - struct metric *def_metric; + struct metric *def_metric; #ifdef HAVE_CLOCK_GETTIME #ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID clock_getres (CLOCK_PROCESS_CPUTIME_ID, &ts); # elif defined(HAVE_CLOCK_VIRTUAL) - clock_getres (CLOCK_VIRTUAL, &ts); + clock_getres (CLOCK_VIRTUAL, &ts); # else - clock_getres (CLOCK_REALTIME, &ts); + clock_getres (CLOCK_REALTIME, &ts); # endif cfg->clock_res = (gint)log10 (1000000 / ts.tv_nsec); @@ -496,12 +545,13 @@ rspamd_config_post_load (struct rspamd_config *cfg) if (cfg->clock_res > 3) { cfg->clock_res = 3; } -#else +#else /* For gettimeofday */ cfg->clock_res = 1; #endif - if ((def_metric = g_hash_table_lookup (cfg->metrics, DEFAULT_METRIC)) == NULL) { + if ((def_metric = + g_hash_table_lookup (cfg->metrics, DEFAULT_METRIC)) == NULL) { def_metric = rspamd_config_new_metric (cfg, NULL); def_metric->name = DEFAULT_METRIC; def_metric->actions[METRIC_ACTION_REJECT].score = DEFAULT_SCORE; @@ -520,14 +570,18 @@ rspamd_config_post_load (struct rspamd_config *cfg) void parse_err (const gchar *fmt, ...) { - va_list aq; - gchar logbuf[BUFSIZ], readbuf[32]; - gint r; + va_list aq; + gchar logbuf[BUFSIZ], readbuf[32]; + gint r; va_start (aq, fmt); rspamd_strlcpy (readbuf, yytext, sizeof (readbuf)); - r = snprintf (logbuf, sizeof (logbuf), "config file parse error! line: %d, text: %s, reason: ", yylineno, readbuf); + r = snprintf (logbuf, + sizeof (logbuf), + "config file parse error! line: %d, text: %s, reason: ", + yylineno, + readbuf); r += vsnprintf (logbuf + r, sizeof (logbuf) - r, fmt, aq); va_end (aq); @@ -537,14 +591,18 @@ parse_err (const gchar *fmt, ...) void parse_warn (const gchar *fmt, ...) { - va_list aq; - gchar logbuf[BUFSIZ], readbuf[32]; - gint r; + va_list aq; + gchar logbuf[BUFSIZ], readbuf[32]; + gint r; va_start (aq, fmt); rspamd_strlcpy (readbuf, yytext, sizeof (readbuf)); - r = snprintf (logbuf, sizeof (logbuf), "config file parse warning! line: %d, text: %s, reason: ", yylineno, readbuf); + r = snprintf (logbuf, + sizeof (logbuf), + "config file parse warning! line: %d, text: %s, reason: ", + yylineno, + readbuf); r += vsnprintf (logbuf + r, sizeof (logbuf) - r, fmt, aq); va_end (aq); @@ -555,7 +613,7 @@ parse_warn (const gchar *fmt, ...) void rspamd_config_unescape_quotes (gchar *line) { - gchar *c = line, *t; + gchar *c = line, *t; while (*c) { if (*c == '\\' && *(c + 1) == '"') { @@ -569,12 +627,12 @@ rspamd_config_unescape_quotes (gchar *line) } } -GList * +GList * rspamd_config_parse_comma_list (rspamd_mempool_t * pool, const gchar *line) { - GList *res = NULL; - const gchar *c, *p; - gchar *str; + GList *res = NULL; + const gchar *c, *p; + gchar *str; c = line; p = c; @@ -585,42 +643,57 @@ rspamd_config_parse_comma_list (rspamd_mempool_t * pool, const gchar *line) rspamd_strlcpy (str, c, p - c + 1); res = g_list_prepend (res, str); /* Skip spaces */ - while (g_ascii_isspace (*(++p))); + while (g_ascii_isspace (*(++p))) ; c = p; continue; } p++; } if (res != NULL) { - rspamd_mempool_add_destructor (pool, (rspamd_mempool_destruct_t) g_list_free, res); + rspamd_mempool_add_destructor (pool, + (rspamd_mempool_destruct_t) g_list_free, + res); } return res; } -struct rspamd_classifier_config * -rspamd_config_new_classifier (struct rspamd_config *cfg, struct rspamd_classifier_config *c) +struct rspamd_classifier_config * +rspamd_config_new_classifier (struct rspamd_config *cfg, + struct rspamd_classifier_config *c) { if (c == NULL) { - c = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_classifier_config)); + c = + rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (struct rspamd_classifier_config)); } if (c->opts == NULL) { c->opts = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); - rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_destroy, c->opts); + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t) g_hash_table_destroy, + c->opts); } if (c->labels == NULL) { - c->labels = g_hash_table_new_full (rspamd_str_hash, rspamd_str_equal, NULL, (GDestroyNotify)g_list_free); - rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_destroy, c->labels); + c->labels = g_hash_table_new_full (rspamd_str_hash, + rspamd_str_equal, + NULL, + (GDestroyNotify)g_list_free); + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t) g_hash_table_destroy, + c->labels); } return c; } -struct rspamd_statfile_config* -rspamd_config_new_statfile (struct rspamd_config *cfg, struct rspamd_statfile_config *c) +struct rspamd_statfile_config * +rspamd_config_new_statfile (struct rspamd_config *cfg, + struct rspamd_statfile_config *c) { if (c == NULL) { - c = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_statfile_config)); + c = + rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (struct rspamd_statfile_config)); } return c; @@ -635,25 +708,36 @@ rspamd_config_new_metric (struct rspamd_config *cfg, struct metric *c) c->grow_factor = 1.0; c->symbols = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); c->descriptions = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); - for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i ++) { + for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) { c->actions[i].score = -1.0; } - rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_destroy, c->symbols); - rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t) g_hash_table_destroy, c->descriptions); + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t) g_hash_table_destroy, + c->symbols); + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t) g_hash_table_destroy, + c->descriptions); } return c; } struct rspamd_worker_conf * -rspamd_config_new_worker (struct rspamd_config *cfg, struct rspamd_worker_conf *c) +rspamd_config_new_worker (struct rspamd_config *cfg, + struct rspamd_worker_conf *c) { if (c == NULL) { - c = rspamd_mempool_alloc0 (cfg->cfg_pool, sizeof (struct rspamd_worker_conf)); + c = + rspamd_mempool_alloc0 (cfg->cfg_pool, + sizeof (struct rspamd_worker_conf)); c->params = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); c->active_workers = g_queue_new (); - rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)g_hash_table_destroy, c->params); - rspamd_mempool_add_destructor (cfg->cfg_pool, (rspamd_mempool_destruct_t)g_queue_free, c->active_workers); + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t)g_hash_table_destroy, + c->params); + rspamd_mempool_add_destructor (cfg->cfg_pool, + (rspamd_mempool_destruct_t)g_queue_free, + c->active_workers); #ifdef HAVE_SC_NPROCESSORS_ONLN c->count = sysconf (_SC_NPROCESSORS_ONLN); #else @@ -662,13 +746,13 @@ rspamd_config_new_worker (struct rspamd_config *cfg, struct rspamd_worker_conf * c->rlimit_nofile = DEFAULT_RLIMIT_NOFILE; c->rlimit_maxcore = DEFAULT_RLIMIT_MAXCORE; } - + return c; } static bool -rspamd_include_map_handler (const guchar *data, gsize len, void* ud) +rspamd_include_map_handler (const guchar *data, gsize len, void * ud) { struct rspamd_config *cfg = (struct rspamd_config *)ud; struct rspamd_ucl_map_cbdata *cbdata, **pcbdata; @@ -683,7 +767,12 @@ rspamd_include_map_handler (const guchar *data, gsize len, void* ud) cbdata->cfg = cfg; *pcbdata = cbdata; - return add_map (cfg, map_line, "ucl include", rspamd_ucl_read_cb, rspamd_ucl_fin_cb, (void **)pcbdata); + return add_map (cfg, + map_line, + "ucl include", + rspamd_ucl_read_cb, + rspamd_ucl_fin_cb, + (void **)pcbdata); } /* @@ -709,32 +798,45 @@ rspamd_include_map_handler (const guchar *data, gsize len, void* ud) static void rspamd_ucl_add_conf_variables (struct ucl_parser *parser) { - ucl_parser_register_variable (parser, RSPAMD_CONFDIR_MACRO, RSPAMD_CONFDIR); - ucl_parser_register_variable (parser, RSPAMD_RUNDIR_MACRO, RSPAMD_RUNDIR); - ucl_parser_register_variable (parser, RSPAMD_DBDIR_MACRO, RSPAMD_DBDIR); - ucl_parser_register_variable (parser, RSPAMD_LOGDIR_MACRO, RSPAMD_LOGDIR); - ucl_parser_register_variable (parser, RSPAMD_PLUGINSDIR_MACRO, RSPAMD_PLUGINSDIR); - ucl_parser_register_variable (parser, RSPAMD_WWWDIR_MACRO, RSPAMD_WWWDIR); - ucl_parser_register_variable (parser, RSPAMD_PREFIX_MACRO, RSPAMD_PREFIX); + ucl_parser_register_variable (parser, + RSPAMD_CONFDIR_MACRO, + RSPAMD_CONFDIR); + ucl_parser_register_variable (parser, RSPAMD_RUNDIR_MACRO, + RSPAMD_RUNDIR); + ucl_parser_register_variable (parser, RSPAMD_DBDIR_MACRO, + RSPAMD_DBDIR); + ucl_parser_register_variable (parser, RSPAMD_LOGDIR_MACRO, + RSPAMD_LOGDIR); + ucl_parser_register_variable (parser, + RSPAMD_PLUGINSDIR_MACRO, + RSPAMD_PLUGINSDIR); + ucl_parser_register_variable (parser, RSPAMD_WWWDIR_MACRO, + RSPAMD_WWWDIR); + ucl_parser_register_variable (parser, RSPAMD_PREFIX_MACRO, + RSPAMD_PREFIX); ucl_parser_register_variable (parser, RSPAMD_VERSION_MACRO, RVERSION); } static void -rspamd_ucl_add_conf_macros (struct ucl_parser *parser, struct rspamd_config *cfg) +rspamd_ucl_add_conf_macros (struct ucl_parser *parser, + struct rspamd_config *cfg) { - ucl_parser_register_macro (parser, "include_map", rspamd_include_map_handler, cfg); + ucl_parser_register_macro (parser, + "include_map", + rspamd_include_map_handler, + cfg); } gboolean rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, - const gchar *convert_to, rspamd_rcl_section_fin_t logger_fin, - gpointer logger_ud) + const gchar *convert_to, rspamd_rcl_section_fin_t logger_fin, + gpointer logger_ud) { - struct stat st; - gint fd; - gchar *data; - GError *err = NULL; - struct rspamd_rcl_section *top, *logger; + struct stat st; + gint fd; + gchar *data; + GError *err = NULL; + struct rspamd_rcl_section *top, *logger; gboolean res; struct ucl_parser *parser; @@ -745,10 +847,11 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, if ((fd = open (filename, O_RDONLY)) == -1) { msg_err ("cannot open %s: %s", filename, strerror (errno)); return FALSE; - + } /* Now mmap this file to simplify reading process */ - if ((data = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { + if ((data = + mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { msg_err ("cannot mmap %s: %s", filename, strerror (errno)); close (fd); return FALSE; @@ -776,7 +879,7 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, top = rspamd_rcl_config_init (); err = NULL; - HASH_FIND_STR(top, "logging", logger); + HASH_FIND_STR (top, "logging", logger); if (logger != NULL) { logger->fin = logger_fin; logger->fin_ud = logger_ud; @@ -793,7 +896,7 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename, static void symbols_classifiers_callback (gpointer key, gpointer value, gpointer ud) { - struct rspamd_config *cfg = ud; + struct rspamd_config *cfg = ud; register_virtual_symbol (&cfg->cache, key, 1.0); } @@ -801,14 +904,16 @@ symbols_classifiers_callback (gpointer key, gpointer value, gpointer ud) void rspamd_config_insert_classify_symbols (struct rspamd_config *cfg) { - g_hash_table_foreach (cfg->classifiers_symbols, symbols_classifiers_callback, cfg); + g_hash_table_foreach (cfg->classifiers_symbols, + symbols_classifiers_callback, + cfg); } -struct rspamd_classifier_config* +struct rspamd_classifier_config * rspamd_config_find_classifier (struct rspamd_config *cfg, const gchar *name) { - GList *cur; - struct rspamd_classifier_config *cf; + GList *cur; + struct rspamd_classifier_config *cf; if (name == NULL) { return NULL; @@ -831,9 +936,9 @@ rspamd_config_find_classifier (struct rspamd_config *cfg, const gchar *name) gboolean rspamd_config_check_statfiles (struct rspamd_classifier_config *cf) { - struct rspamd_statfile_config *st; - gboolean has_other = FALSE, res = FALSE, cur_class; - GList *cur; + struct rspamd_statfile_config *st; + gboolean has_other = FALSE, res = FALSE, cur_class; + GList *cur; /* First check classes directly */ cur = cf->statfiles; @@ -884,8 +989,11 @@ rspamd_config_check_statfiles (struct rspamd_classifier_config *cf) return res; } -static gchar* -rspamd_ucl_read_cb (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map_cb_data *data) +static gchar * +rspamd_ucl_read_cb (rspamd_mempool_t * pool, + gchar * chunk, + gint len, + struct map_cb_data *data) { struct rspamd_ucl_map_cbdata *cbdata = data->cur_data, *prev; @@ -905,7 +1013,8 @@ rspamd_ucl_read_cb (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map static void rspamd_ucl_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) { - struct rspamd_ucl_map_cbdata *cbdata = data->cur_data, *prev = data->prev_data; + struct rspamd_ucl_map_cbdata *cbdata = data->cur_data, *prev = + data->prev_data; ucl_object_t *obj; struct ucl_parser *parser; guint32 checksum; @@ -926,8 +1035,11 @@ rspamd_ucl_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) if (data->map->checksum != checksum) { /* New data available */ parser = ucl_parser_new (0); - if (!ucl_parser_add_chunk (parser, cbdata->buf->str, cbdata->buf->len)) { - msg_err ("cannot parse map %s: %s", data->map->uri, ucl_parser_get_error (parser)); + if (!ucl_parser_add_chunk (parser, cbdata->buf->str, + cbdata->buf->len)) { + msg_err ("cannot parse map %s: %s", + data->map->uri, + ucl_parser_get_error (parser)); ucl_parser_free (parser); } else { @@ -939,16 +1051,18 @@ rspamd_ucl_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) } } else { - msg_info ("do not reload map %s, checksum is the same: %d", data->map->uri, checksum); + msg_info ("do not reload map %s, checksum is the same: %d", + data->map->uri, + checksum); } } gboolean rspamd_config_parse_ip_list (const gchar *ip_list, radix_tree_t **tree) { - gchar **strvec, **cur; - struct in_addr ina; - guint32 mask; + gchar **strvec, **cur; + struct in_addr ina; + guint32 mask; strvec = g_strsplit_set (ip_list, ",", 0); cur = strvec; @@ -961,7 +1075,7 @@ rspamd_config_parse_ip_list (const gchar *ip_list, radix_tree_t **tree) } radix32tree_add (*tree, htonl (ina.s_addr), mask, 1); } - cur ++; + cur++; } return (*tree != NULL); diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c index abc8ddc05..2498b4003 100644 --- a/src/libserver/dkim.c +++ b/src/libserver/dkim.c @@ -22,26 +22,63 @@ */ #include "config.h" -#include "main.h" -#include "message.h" #include "dkim.h" #include "dns.h" +#include "main.h" +#include "message.h" /* Parser of dkim params */ -typedef gboolean (*dkim_parse_param_f) (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); - -static gboolean rspamd_dkim_parse_signature (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_signalg (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_domain (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_canonalg (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_ignore (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_selector (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_hdrlist (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_version (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_timestamp (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_expiration (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_bodyhash (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); -static gboolean rspamd_dkim_parse_bodylength (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err); +typedef gboolean (*dkim_parse_param_f) (rspamd_dkim_context_t * ctx, + const gchar *param, gsize len, GError **err); + +static gboolean rspamd_dkim_parse_signature (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_signalg (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_domain (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_canonalg (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_ignore (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_selector (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_hdrlist (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_version (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_timestamp (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_expiration (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_bodyhash (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); +static gboolean rspamd_dkim_parse_bodylength (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err); static const dkim_parse_param_f parser_funcs[] = { @@ -75,7 +112,10 @@ dkim_error_quark (void) /* Parsers implementation */ static gboolean -rspamd_dkim_parse_signature (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_signature (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { ctx->b = rspamd_mempool_alloc (ctx->pool, len + 1); rspamd_strlcpy (ctx->b, param, len + 1); @@ -93,7 +133,10 @@ rspamd_dkim_parse_signature (rspamd_dkim_context_t* ctx, const gchar *param, gsi } static gboolean -rspamd_dkim_parse_signalg (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_signalg (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { if (len == 8) { if (memcmp (param, "rsa-sha1", len) == 0) { @@ -108,12 +151,18 @@ rspamd_dkim_parse_signalg (rspamd_dkim_context_t* ctx, const gchar *param, gsize } } - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_INVALID_A, "invalid dkim sign algorithm"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_INVALID_A, + "invalid dkim sign algorithm"); return FALSE; } static gboolean -rspamd_dkim_parse_domain (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_domain (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { ctx->domain = rspamd_mempool_alloc (ctx->pool, len + 1); rspamd_strlcpy (ctx->domain, param, len + 1); @@ -121,10 +170,13 @@ rspamd_dkim_parse_domain (rspamd_dkim_context_t* ctx, const gchar *param, gsize } static gboolean -rspamd_dkim_parse_canonalg (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_canonalg (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { - const gchar *p, *slash = NULL, *end = param + len; - gsize sl = 0; + const gchar *p, *slash = NULL, *end = param + len; + gsize sl = 0; p = param; while (p != end) { @@ -132,8 +184,8 @@ rspamd_dkim_parse_canonalg (rspamd_dkim_context_t* ctx, const gchar *param, gsiz slash = p; break; } - p ++; - sl ++; + p++; + sl++; } if (slash == NULL) { @@ -160,7 +212,7 @@ rspamd_dkim_parse_canonalg (rspamd_dkim_context_t* ctx, const gchar *param, gsiz } /* Check body */ len -= sl + 1; - slash ++; + slash++; if (len == 6 && memcmp (slash, "simple", len) == 0) { ctx->body_canon_type = DKIM_CANON_SIMPLE; return TRUE; @@ -172,32 +224,41 @@ rspamd_dkim_parse_canonalg (rspamd_dkim_context_t* ctx, const gchar *param, gsiz } err: - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_INVALID_A, "invalid dkim canonization algorithm"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_INVALID_A, + "invalid dkim canonization algorithm"); return FALSE; } static gboolean -rspamd_dkim_parse_ignore (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_ignore (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { /* Just ignore unused params */ return TRUE; } static gboolean -rspamd_dkim_parse_selector (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_selector (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { ctx->selector = rspamd_mempool_alloc (ctx->pool, len + 1); rspamd_strlcpy (ctx->selector, param, len + 1); return TRUE; } -static struct rspamd_dkim_header* +static struct rspamd_dkim_header * rspamd_dkim_find_header (GPtrArray *arr, const gchar *name, gsize len) { - guint i; - struct rspamd_dkim_header *h; + guint i; + struct rspamd_dkim_header *h; - for (i = 0; i < arr->len; i ++) { + for (i = 0; i < arr->len; i++) { h = g_ptr_array_index (arr, i); if (g_ascii_strncasecmp (h->name, name, len) == 0) { return h; @@ -210,26 +271,29 @@ rspamd_dkim_find_header (GPtrArray *arr, const gchar *name, gsize len) static void rspamd_dkim_hlist_free (void *ud) { - GPtrArray *a = ud; + GPtrArray *a = ud; g_ptr_array_free (a, TRUE); } static gboolean -rspamd_dkim_parse_hdrlist (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_hdrlist (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { - const gchar *c, *p, *end = param + len; - gchar *h; - gboolean from_found = FALSE; - guint count = 0; - struct rspamd_dkim_header *new; + const gchar *c, *p, *end = param + len; + gchar *h; + gboolean from_found = FALSE; + guint count = 0; + struct rspamd_dkim_header *new; p = param; while (p <= end) { if ((*p == ':' || p == end)) { - count ++; + count++; } - p ++; + p++; } if (count > 0) { @@ -243,12 +307,15 @@ rspamd_dkim_parse_hdrlist (rspamd_dkim_context_t* ctx, const gchar *param, gsize p = param; while (p <= end) { if ((*p == ':' || p == end) && p - c > 0) { - if ((new = rspamd_dkim_find_header (ctx->hlist, c, p - c)) != NULL) { - new->count ++; + if ((new = + rspamd_dkim_find_header (ctx->hlist, c, p - c)) != NULL) { + new->count++; } else { /* Insert new header to the list */ - new = rspamd_mempool_alloc (ctx->pool, sizeof (struct rspamd_dkim_header)); + new = + rspamd_mempool_alloc (ctx->pool, + sizeof (struct rspamd_dkim_header)); h = rspamd_mempool_alloc (ctx->pool, p - c + 1); rspamd_strlcpy (h, c, p - c + 1); g_strstrip (h); @@ -261,35 +328,49 @@ rspamd_dkim_parse_hdrlist (rspamd_dkim_context_t* ctx, const gchar *param, gsize g_ptr_array_add (ctx->hlist, new); } c = p + 1; - p ++; + p++; } else { - p ++; + p++; } } if (!ctx->hlist) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_INVALID_H, "invalid dkim header list"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_INVALID_H, + "invalid dkim header list"); return FALSE; } else { if (!from_found) { g_ptr_array_free (ctx->hlist, TRUE); - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_INVALID_H, "invalid dkim header list, from header is missing"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_INVALID_H, + "invalid dkim header list, from header is missing"); return FALSE; } /* Reverse list */ - rspamd_mempool_add_destructor (ctx->pool, (rspamd_mempool_destruct_t)rspamd_dkim_hlist_free, ctx->hlist); + rspamd_mempool_add_destructor (ctx->pool, + (rspamd_mempool_destruct_t)rspamd_dkim_hlist_free, + ctx->hlist); } return TRUE; } static gboolean -rspamd_dkim_parse_version (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_version (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { if (len != 1 || *param != '1') { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_VERSION, "invalid dkim version"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_VERSION, + "invalid dkim version"); return FALSE; } @@ -298,12 +379,18 @@ rspamd_dkim_parse_version (rspamd_dkim_context_t* ctx, const gchar *param, gsize } static gboolean -rspamd_dkim_parse_timestamp (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_timestamp (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { - gulong val; + gulong val; if (!rspamd_strtoul (param, len, &val)) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_UNKNOWN, "invalid dkim timestamp"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_UNKNOWN, + "invalid dkim timestamp"); return FALSE; } ctx->timestamp = val; @@ -312,12 +399,18 @@ rspamd_dkim_parse_timestamp (rspamd_dkim_context_t* ctx, const gchar *param, gsi } static gboolean -rspamd_dkim_parse_expiration (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_expiration (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { - gulong val; + gulong val; if (!rspamd_strtoul (param, len, &val)) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_UNKNOWN, "invalid dkim expiration"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_UNKNOWN, + "invalid dkim expiration"); return FALSE; } ctx->expiration = val; @@ -326,7 +419,10 @@ rspamd_dkim_parse_expiration (rspamd_dkim_context_t* ctx, const gchar *param, gs } static gboolean -rspamd_dkim_parse_bodyhash (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_bodyhash (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { ctx->bh = rspamd_mempool_alloc (ctx->pool, len + 1); rspamd_strlcpy (ctx->bh, param, len + 1); @@ -344,12 +440,18 @@ rspamd_dkim_parse_bodyhash (rspamd_dkim_context_t* ctx, const gchar *param, gsiz } static gboolean -rspamd_dkim_parse_bodylength (rspamd_dkim_context_t* ctx, const gchar *param, gsize len, GError **err) +rspamd_dkim_parse_bodylength (rspamd_dkim_context_t * ctx, + const gchar *param, + gsize len, + GError **err) { - gulong val; + gulong val; if (!rspamd_strtoul (param, len, &val)) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_INVALID_L, "invalid dkim body length"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_INVALID_L, + "invalid dkim body length"); return FALSE; } ctx->len = val; @@ -364,21 +466,24 @@ rspamd_dkim_parse_bodylength (rspamd_dkim_context_t* ctx, const gchar *param, gs * @param err pointer to error object * @return new context or NULL */ -rspamd_dkim_context_t* -rspamd_create_dkim_context (const gchar *sig, rspamd_mempool_t *pool, guint time_jitter, GError **err) +rspamd_dkim_context_t * +rspamd_create_dkim_context (const gchar *sig, + rspamd_mempool_t *pool, + guint time_jitter, + GError **err) { - const gchar *p, *c, *tag = NULL, *end; - gsize taglen; - gint param = DKIM_PARAM_UNKNOWN; - time_t now; - rspamd_dkim_context_t *new; + const gchar *p, *c, *tag = NULL, *end; + gsize taglen; + gint param = DKIM_PARAM_UNKNOWN; + time_t now; + rspamd_dkim_context_t *new; enum { DKIM_STATE_TAG = 0, DKIM_STATE_AFTER_TAG, DKIM_STATE_VALUE, DKIM_STATE_SKIP_SPACES = 99, DKIM_STATE_ERROR = 100 - } state, next_state; + } state, next_state; new = rspamd_mempool_alloc0 (pool, sizeof (rspamd_dkim_context_t)); @@ -400,17 +505,20 @@ rspamd_create_dkim_context (const gchar *sig, rspamd_mempool_t *pool, guint time taglen = p - c; while (*p && g_ascii_isspace (*p)) { /* Skip spaces before '=' sign */ - p ++; + p++; } if (*p != '=') { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_UNKNOWN, "invalid dkim param"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_UNKNOWN, + "invalid dkim param"); state = DKIM_STATE_ERROR; } else { state = DKIM_STATE_SKIP_SPACES; next_state = DKIM_STATE_AFTER_TAG; param = DKIM_PARAM_UNKNOWN; - p ++; + p++; tag = c; } } @@ -418,19 +526,22 @@ rspamd_create_dkim_context (const gchar *sig, rspamd_mempool_t *pool, guint time state = DKIM_STATE_SKIP_SPACES; next_state = DKIM_STATE_AFTER_TAG; param = DKIM_PARAM_UNKNOWN; - p ++; + p++; tag = c; } else { - taglen ++; - p ++; + taglen++; + p++; } break; case DKIM_STATE_AFTER_TAG: /* We got tag at tag and len at taglen */ switch (taglen) { case 0: - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_UNKNOWN, "zero length dkim param"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_UNKNOWN, + "zero length dkim param"); state = DKIM_STATE_ERROR; break; case 1: @@ -476,7 +587,11 @@ rspamd_create_dkim_context (const gchar *sig, rspamd_mempool_t *pool, guint time param = DKIM_PARAM_COPIEDHDRS; break; default: - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_UNKNOWN, "invalid dkim param: %c", *tag); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_UNKNOWN, + "invalid dkim param: %c", + *tag); state = DKIM_STATE_ERROR; break; } @@ -486,12 +601,21 @@ rspamd_create_dkim_context (const gchar *sig, rspamd_mempool_t *pool, guint time param = DKIM_PARAM_BODYHASH; } else { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_UNKNOWN, "invalid dkim param: %c%c", tag[0], tag[1]); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_UNKNOWN, + "invalid dkim param: %c%c", + tag[0], + tag[1]); state = DKIM_STATE_ERROR; } break; default: - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_UNKNOWN, "invalid dkim param length: %zd", taglen); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_UNKNOWN, + "invalid dkim param length: %zd", + taglen); state = DKIM_STATE_ERROR; break; } @@ -503,32 +627,34 @@ rspamd_create_dkim_context (const gchar *sig, rspamd_mempool_t *pool, guint time break; case DKIM_STATE_VALUE: if (*p == ';') { - if (param == DKIM_PARAM_UNKNOWN || !parser_funcs[param](new, c, p - c, err)) { + if (param == DKIM_PARAM_UNKNOWN || + !parser_funcs[param](new, c, p - c, err)) { state = DKIM_STATE_ERROR; } else { state = DKIM_STATE_SKIP_SPACES; next_state = DKIM_STATE_TAG; - p ++; + p++; taglen = 0; } } else if (p == end) { - if (param == DKIM_PARAM_UNKNOWN || !parser_funcs[param](new, c, p - c + 1, err)) { + if (param == DKIM_PARAM_UNKNOWN || + !parser_funcs[param](new, c, p - c + 1, err)) { state = DKIM_STATE_ERROR; } else { /* Finish processing */ - p ++; + p++; } } else { - p ++; + p++; } break; case DKIM_STATE_SKIP_SPACES: if (g_ascii_isspace (*p)) { - p ++; + p++; } else { c = p; @@ -550,62 +676,105 @@ rspamd_create_dkim_context (const gchar *sig, rspamd_mempool_t *pool, guint time /* Now check validity of signature */ if (new->b == NULL) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_EMPTY_B, "b parameter missing"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_EMPTY_B, + "b parameter missing"); return NULL; } if (new->bh == NULL) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_EMPTY_BH, "bh parameter missing"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_EMPTY_BH, + "bh parameter missing"); return NULL; } if (new->domain == NULL) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_EMPTY_D, "domain parameter missing"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_EMPTY_D, + "domain parameter missing"); return NULL; } if (new->selector == NULL) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_EMPTY_S, "selector parameter missing"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_EMPTY_S, + "selector parameter missing"); return NULL; } if (new->ver == 0) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_EMPTY_V, "v parameter missing"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_EMPTY_V, + "v parameter missing"); return NULL; } if (new->hlist == NULL) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_EMPTY_H, "h parameter missing"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_EMPTY_H, + "h parameter missing"); return NULL; } if (new->sig_alg == DKIM_SIGN_UNKNOWN) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_EMPTY_S, "s parameter missing"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_EMPTY_S, + "s parameter missing"); return NULL; } if (new->sig_alg == DKIM_SIGN_RSASHA1) { /* Check bh length */ if (new->bhlen != (guint)g_checksum_type_get_length (G_CHECKSUM_SHA1)) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_BADSIG, "signature has incorrect length: %ud", new->bhlen); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_BADSIG, + "signature has incorrect length: %ud", + new->bhlen); return NULL; } } else if (new->sig_alg == DKIM_SIGN_RSASHA256) { - if (new->bhlen != (guint)g_checksum_type_get_length (G_CHECKSUM_SHA256)) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_BADSIG, "signature has incorrect length: %ud", new->bhlen); + if (new->bhlen != + (guint)g_checksum_type_get_length (G_CHECKSUM_SHA256)) { + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_BADSIG, + "signature has incorrect length: %ud", + new->bhlen); return NULL; } } /* Check expiration */ now = time (NULL); - if (new->timestamp && now < new->timestamp && new->timestamp - now > (gint)time_jitter) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_FUTURE, "signature was made in future, ignoring"); + if (new->timestamp && now < new->timestamp && new->timestamp - now > + (gint)time_jitter) { + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_FUTURE, + "signature was made in future, ignoring"); return NULL; } if (new->expiration && new->expiration < now) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_EXPIRED, "signature has expired"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_EXPIRED, + "signature has expired"); return NULL; } /* Now create dns key to request further */ - taglen = strlen (new->domain) + strlen (new->selector) + sizeof (DKIM_DNSKEYNAME) + 2; + taglen = strlen (new->domain) + strlen (new->selector) + + sizeof (DKIM_DNSKEYNAME) + 2; new->dns_key = rspamd_mempool_alloc (new->pool, taglen); - rspamd_snprintf (new->dns_key, taglen, "%s.%s.%s", new->selector, DKIM_DNSKEYNAME, new->domain); + rspamd_snprintf (new->dns_key, + taglen, + "%s.%s.%s", + new->selector, + DKIM_DNSKEYNAME, + new->domain); /* Create checksums for further operations */ if (new->sig_alg == DKIM_SIGN_RSASHA1) { @@ -617,12 +786,19 @@ rspamd_create_dkim_context (const gchar *sig, rspamd_mempool_t *pool, guint time new->headers_hash = g_checksum_new (G_CHECKSUM_SHA256); } else { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_BADSIG, "signature has unsupported signature algorithm"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_BADSIG, + "signature has unsupported signature algorithm"); return NULL; } - rspamd_mempool_add_destructor (new->pool, (rspamd_mempool_destruct_t)g_checksum_free, new->body_hash); - rspamd_mempool_add_destructor (new->pool, (rspamd_mempool_destruct_t)g_checksum_free, new->headers_hash); + rspamd_mempool_add_destructor (new->pool, + (rspamd_mempool_destruct_t)g_checksum_free, + new->body_hash); + rspamd_mempool_add_destructor (new->pool, + (rspamd_mempool_destruct_t)g_checksum_free, + new->headers_hash); return new; } @@ -633,10 +809,10 @@ struct rspamd_dkim_key_cbdata { gpointer ud; }; -static rspamd_dkim_key_t* +static rspamd_dkim_key_t * rspamd_dkim_make_key (const gchar *keydata, guint keylen, GError **err) { - rspamd_dkim_key_t *key = NULL; + rspamd_dkim_key_t *key = NULL; if (keylen < 3) { msg_err ("DKIM key is too short to be valid"); @@ -660,21 +836,30 @@ rspamd_dkim_make_key (const gchar *keydata, guint keylen, GError **err) #ifdef HAVE_OPENSSL key->key_bio = BIO_new_mem_buf (key->keydata, key->decoded_len); if (key->key_bio == NULL) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_KEYFAIL, "cannot make ssl bio from key"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_KEYFAIL, + "cannot make ssl bio from key"); rspamd_dkim_key_free (key); return NULL; } key->key_evp = d2i_PUBKEY_bio (key->key_bio, NULL); if (key->key_evp == NULL) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_KEYFAIL, "cannot extract pubkey from bio"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_KEYFAIL, + "cannot extract pubkey from bio"); rspamd_dkim_key_free (key); return NULL; } key->key_rsa = EVP_PKEY_get1_RSA (key->key_evp); if (key->key_rsa == NULL) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_KEYFAIL, "cannot extract rsa key from evp key"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_KEYFAIL, + "cannot extract rsa key from evp key"); rspamd_dkim_key_free (key); return NULL; } @@ -699,16 +884,16 @@ rspamd_dkim_key_free (rspamd_dkim_key_t *key) BIO_free (key->key_bio); } #endif - g_slice_free1 (key->keylen, key->keydata); + g_slice_free1 (key->keylen, key->keydata); g_slice_free1 (sizeof (rspamd_dkim_key_t), key); } -static rspamd_dkim_key_t* +static rspamd_dkim_key_t * rspamd_dkim_parse_key (const gchar *txt, gsize *keylen, GError **err) { - const gchar *c, *p, *end; - gint state = 0; - gsize len; + const gchar *c, *p, *end; + gint state = 0; + gsize len; c = txt; p = txt; @@ -725,7 +910,7 @@ rspamd_dkim_parse_key (const gchar *txt, gsize *keylen, GError **err) } else { /* Ignore everything */ - p ++; + p++; } break; case 1: @@ -735,17 +920,21 @@ rspamd_dkim_parse_key (const gchar *txt, gsize *keylen, GError **err) return rspamd_dkim_make_key (c, len, err); } else { - p ++; + p++; } break; } } if (p - c == 0) { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_KEYREVOKED, "key was revoked"); + g_set_error (err, + DKIM_ERROR, + DKIM_SIGERROR_KEYREVOKED, + "key was revoked"); } else { - g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_KEYFAIL, "key was not found"); + g_set_error (err, DKIM_ERROR, DKIM_SIGERROR_KEYFAIL, + "key was not found"); } return NULL; @@ -755,21 +944,28 @@ rspamd_dkim_parse_key (const gchar *txt, gsize *keylen, GError **err) static void rspamd_dkim_dns_cb (struct rdns_reply *reply, gpointer arg) { - struct rspamd_dkim_key_cbdata *cbdata = arg; - rspamd_dkim_key_t *key = NULL; - GError *err = NULL; - struct rdns_reply_entry *elt; - gsize keylen = 0; + struct rspamd_dkim_key_cbdata *cbdata = arg; + rspamd_dkim_key_t *key = NULL; + GError *err = NULL; + struct rdns_reply_entry *elt; + gsize keylen = 0; if (reply->code != RDNS_RC_NOERROR) { - g_set_error (&err, DKIM_ERROR, DKIM_SIGERROR_NOKEY, "dns request to %s failed: %s", cbdata->ctx->dns_key, - rdns_strerror (reply->code)); + g_set_error (&err, + DKIM_ERROR, + DKIM_SIGERROR_NOKEY, + "dns request to %s failed: %s", + cbdata->ctx->dns_key, + rdns_strerror (reply->code)); cbdata->handler (NULL, 0, cbdata->ctx, cbdata->ud, err); } else { - LL_FOREACH (reply->entries, elt) { + LL_FOREACH (reply->entries, elt) + { if (elt->type == RDNS_REQUEST_TXT) { - key = rspamd_dkim_parse_key (elt->content.txt.data, &keylen, &err); + key = rspamd_dkim_parse_key (elt->content.txt.data, + &keylen, + &err); if (key) { key->ttl = elt->ttl; break; @@ -793,30 +989,41 @@ rspamd_dkim_dns_cb (struct rdns_reply *reply, gpointer arg) * @return */ gboolean -rspamd_get_dkim_key (rspamd_dkim_context_t *ctx, struct rspamd_dns_resolver *resolver, - struct rspamd_async_session *s, dkim_key_handler_f handler, gpointer ud) +rspamd_get_dkim_key (rspamd_dkim_context_t *ctx, + struct rspamd_dns_resolver *resolver, + struct rspamd_async_session *s, + dkim_key_handler_f handler, + gpointer ud) { - struct rspamd_dkim_key_cbdata *cbdata; + struct rspamd_dkim_key_cbdata *cbdata; - g_return_val_if_fail (ctx != NULL, FALSE); + g_return_val_if_fail (ctx != NULL, FALSE); g_return_val_if_fail (ctx->dns_key != NULL, FALSE); - cbdata = rspamd_mempool_alloc (ctx->pool, sizeof (struct rspamd_dkim_key_cbdata)); + cbdata = + rspamd_mempool_alloc (ctx->pool, + sizeof (struct rspamd_dkim_key_cbdata)); cbdata->ctx = ctx; cbdata->handler = handler; cbdata->ud = ud; - return make_dns_request (resolver, s, ctx->pool, rspamd_dkim_dns_cb, cbdata, RDNS_REQUEST_TXT, ctx->dns_key); + return make_dns_request (resolver, + s, + ctx->pool, + rspamd_dkim_dns_cb, + cbdata, + RDNS_REQUEST_TXT, + ctx->dns_key); } static gboolean rspamd_dkim_relaxed_body_step (GChecksum *ck, const gchar **start, guint remain) { - const gchar *h; - static gchar buf[BUFSIZ]; - gchar *t; - guint len, inlen; - gboolean got_sp, finished = FALSE; + const gchar *h; + static gchar buf[BUFSIZ]; + gchar *t; + guint len, inlen; + gboolean got_sp, finished = FALSE; if (remain > sizeof (buf)) { len = sizeof (buf); @@ -835,16 +1042,16 @@ rspamd_dkim_relaxed_body_step (GChecksum *ck, const gchar **start, guint remain) /* Ignore spaces at the end of line */ if (got_sp) { got_sp = FALSE; - t --; + t--; } /* Replace a single \n or \r with \r\n */ if (*h == '\n' && *(h - 1) != '\r') { - *t ++ = '\r'; - inlen --; + *t++ = '\r'; + inlen--; } else if (*h == '\r' && *(h + 1) != '\n') { - *t ++ = *h ++; - *t ++ = '\n'; + *t++ = *h++; + *t++ = '\n'; if (inlen > 1) { inlen -= 2; } @@ -852,22 +1059,22 @@ rspamd_dkim_relaxed_body_step (GChecksum *ck, const gchar **start, guint remain) /* It is safe as inlen = sizeof (buf) - 1 */ inlen = 0; } - len --; + len--; continue; } } else if (g_ascii_isspace (*h)) { if (got_sp) { /* Ignore multiply spaces */ - h ++; - len --; + h++; + len--; continue; } else { *t++ = ' '; - h ++; - inlen --; - len --; + h++; + inlen--; + len--; got_sp = TRUE; continue; } @@ -876,15 +1083,15 @@ rspamd_dkim_relaxed_body_step (GChecksum *ck, const gchar **start, guint remain) got_sp = FALSE; } *t++ = *h++; - inlen --; - len --; + inlen--; + len--; } *start = h; if (!finished && *(t - 1) == ' ' && g_ascii_isspace (*h)) { /* Avoid border problems */ - t --; + t--; } #if 0 msg_debug ("update signature with buffer: %*s", t - buf, buf); @@ -897,11 +1104,11 @@ rspamd_dkim_relaxed_body_step (GChecksum *ck, const gchar **start, guint remain) static gboolean rspamd_dkim_simple_body_step (GChecksum *ck, const gchar **start, guint remain) { - const gchar *h; - static gchar buf[BUFSIZ]; - gchar *t; - guint len, inlen; - gboolean finished = FALSE; + const gchar *h; + static gchar buf[BUFSIZ]; + gchar *t; + guint len, inlen; + gboolean finished = FALSE; if (remain > sizeof (buf)) { len = sizeof (buf); @@ -918,12 +1125,12 @@ rspamd_dkim_simple_body_step (GChecksum *ck, const gchar **start, guint remain) if (*h == '\r' || *h == '\n') { /* Replace a single \n or \r with \r\n */ if (*h == '\n' && *(h - 1) != '\r') { - *t ++ = '\r'; - inlen --; + *t++ = '\r'; + inlen--; } else if (*h == '\r' && *(h + 1) != '\n') { - *t ++ = *h ++; - *t ++ = '\n'; + *t++ = *h++; + *t++ = '\n'; if (inlen > 1) { inlen -= 2; } @@ -931,13 +1138,13 @@ rspamd_dkim_simple_body_step (GChecksum *ck, const gchar **start, guint remain) /* It is safe as inlen = sizeof (buf) - 1 */ inlen = 0; } - len --; + len--; continue; } } *t++ = *h++; - inlen --; - len --; + inlen--; + len--; } *start = h; @@ -951,7 +1158,9 @@ rspamd_dkim_simple_body_step (GChecksum *ck, const gchar **start, guint remain) } static gboolean -rspamd_dkim_canonize_body (rspamd_dkim_context_t *ctx, const gchar *start, const gchar *end) +rspamd_dkim_canonize_body (rspamd_dkim_context_t *ctx, + const gchar *start, + const gchar *end) { const gchar *p; @@ -972,10 +1181,10 @@ rspamd_dkim_canonize_body (rspamd_dkim_context_t *ctx, const gchar *start, const p -= 2; } else if (*p == '\n' && *(p - 1) == '\n') { - p --; + p--; } else if (*p == '\r' && *(p - 1) == '\r') { - p --; + p--; } else { break; @@ -994,10 +1203,12 @@ rspamd_dkim_canonize_body (rspamd_dkim_context_t *ctx, const gchar *start, const else { if (ctx->body_canon_type == DKIM_CANON_SIMPLE) { /* Simple canonization */ - while (rspamd_dkim_simple_body_step (ctx->body_hash, &start, end - start)); + while (rspamd_dkim_simple_body_step (ctx->body_hash, &start, + end - start)) ; } else { - while (rspamd_dkim_relaxed_body_step (ctx->body_hash, &start, end - start)); + while (rspamd_dkim_relaxed_body_step (ctx->body_hash, &start, + end - start)) ; } } return TRUE; @@ -1011,29 +1222,29 @@ rspamd_dkim_canonize_body (rspamd_dkim_context_t *ctx, const gchar *start, const static void rspamd_dkim_hash_update (GChecksum *ck, const gchar *begin, gsize len) { - const gchar *p, *c, *end; + const gchar *p, *c, *end; end = begin + len; p = begin; c = p; while (p != end) { if (*p == '\r') { - g_checksum_update (ck, c, p - c); + g_checksum_update (ck, c, p - c); g_checksum_update (ck, CRLF, sizeof (CRLF) - 1); - p ++; + p++; if (*p == '\n') { - p ++; + p++; } c = p; } else if (*p == '\n') { - g_checksum_update (ck, c, p - c); + g_checksum_update (ck, c, p - c); g_checksum_update (ck, CRLF, sizeof (CRLF) - 1); - p ++; + p++; c = p; } else { - p ++; + p++; } } if (p != c) { @@ -1043,10 +1254,12 @@ rspamd_dkim_hash_update (GChecksum *ck, const gchar *begin, gsize len) /* Update hash by signature value (ignoring b= tag) */ static void -rspamd_dkim_signature_update (rspamd_dkim_context_t *ctx, const gchar *begin, guint len) +rspamd_dkim_signature_update (rspamd_dkim_context_t *ctx, + const gchar *begin, + guint len) { - const gchar *p, *c, *end; - gboolean tag, skip; + const gchar *p, *c, *end; + gboolean tag, skip; end = begin + len; p = begin; @@ -1057,7 +1270,9 @@ rspamd_dkim_signature_update (rspamd_dkim_context_t *ctx, const gchar *begin, gu while (p < end) { if (tag && p[0] == 'b' && p[1] == '=') { /* Add to signature */ - msg_debug ("initial update hash with signature part: %*s", p - c + 2, c); + msg_debug ("initial update hash with signature part: %*s", + p - c + 2, + c); rspamd_dkim_hash_update (ctx->headers_hash, c, p - c + 2); skip = TRUE; } @@ -1071,13 +1286,13 @@ rspamd_dkim_signature_update (rspamd_dkim_context_t *ctx, const gchar *begin, gu else if (tag && *p == '=') { tag = FALSE; } - p ++; + p++; } - p --; + p--; /* Skip \r\n at the end */ while ((*p == '\r' || *p == '\n') && p >= c) { - p --; + p--; } if (p - c + 1 > 0) { @@ -1087,12 +1302,15 @@ rspamd_dkim_signature_update (rspamd_dkim_context_t *ctx, const gchar *begin, gu } static gboolean -rspamd_dkim_canonize_header_relaxed (rspamd_dkim_context_t *ctx, const gchar *header, const gchar *header_name, gboolean is_sign) +rspamd_dkim_canonize_header_relaxed (rspamd_dkim_context_t *ctx, + const gchar *header, + const gchar *header_name, + gboolean is_sign) { - const gchar *h; - gchar *t, *buf; - guint inlen; - gboolean got_sp, allocated = FALSE; + const gchar *h; + gchar *t, *buf; + guint inlen; + gboolean got_sp, allocated = FALSE; inlen = strlen (header) + strlen (header_name) + sizeof (":" CRLF); if (inlen > BUFSIZ) { @@ -1108,7 +1326,7 @@ rspamd_dkim_canonize_header_relaxed (rspamd_dkim_context_t *ctx, const gchar *he t = buf; h = header_name; while (*h) { - *t ++ = g_ascii_tolower (*h++); + *t++ = g_ascii_tolower (*h++); } *t++ = ':'; @@ -1116,30 +1334,30 @@ rspamd_dkim_canonize_header_relaxed (rspamd_dkim_context_t *ctx, const gchar *he h = header; /* Skip spaces at the beginning */ while (g_ascii_isspace (*h)) { - h ++; + h++; } got_sp = FALSE; while (*h) { if (g_ascii_isspace (*h)) { if (got_sp) { - h ++; + h++; continue; } else { got_sp = TRUE; - *t ++ = ' '; - h ++; + *t++ = ' '; + h++; continue; } } else { got_sp = FALSE; } - *t ++ = *h ++; + *t++ = *h++; } if (g_ascii_isspace (*(t - 1))) { - t --; + t--; } *t++ = '\r'; *t++ = '\n'; @@ -1167,18 +1385,25 @@ struct rspamd_dkim_sign_chunk { }; static gboolean -rspamd_dkim_canonize_header_simple (rspamd_dkim_context_t *ctx, const gchar *headers, - const gchar *header_name, guint count, gboolean is_sign) +rspamd_dkim_canonize_header_simple (rspamd_dkim_context_t *ctx, + const gchar *headers, + const gchar *header_name, + guint count, + gboolean is_sign) { - const gchar *p, *c; - gint state = 0, hlen; - gboolean found = FALSE; - GArray *to_sign; - struct rspamd_dkim_sign_chunk chunk, *elt; - gint i; + const gchar *p, *c; + gint state = 0, hlen; + gboolean found = FALSE; + GArray *to_sign; + struct rspamd_dkim_sign_chunk chunk, *elt; + gint i; /* This process is very similar to raw headers processing */ - to_sign = g_array_sized_new (FALSE, FALSE, sizeof (struct rspamd_dkim_sign_chunk), count); + to_sign = + g_array_sized_new (FALSE, + FALSE, + sizeof (struct rspamd_dkim_sign_chunk), + count); p = headers; c = p; hlen = strlen (header_name); @@ -1204,7 +1429,7 @@ rspamd_dkim_canonize_header_simple (rspamd_dkim_context_t *ctx, const gchar *hea state = 1; } } - p ++; + p++; break; case 1: /* Skip header state */ @@ -1213,7 +1438,7 @@ rspamd_dkim_canonize_header_simple (rspamd_dkim_context_t *ctx, const gchar *hea state = 0; c = p + 1; } - p ++; + p++; break; case 2: /* c contains the beginning of header */ @@ -1233,7 +1458,7 @@ rspamd_dkim_canonize_header_simple (rspamd_dkim_context_t *ctx, const gchar *hea state = 0; found = TRUE; } - p ++; + p++; break; } } @@ -1241,16 +1466,25 @@ rspamd_dkim_canonize_header_simple (rspamd_dkim_context_t *ctx, const gchar *hea if (found) { if (!is_sign) { - for (i = to_sign->len - 1; i >= 0 && count > 0; i --, count --) { - elt = &g_array_index (to_sign, struct rspamd_dkim_sign_chunk, i); + for (i = to_sign->len - 1; i >= 0 && count > 0; i--, count--) { + elt = + &g_array_index (to_sign, struct rspamd_dkim_sign_chunk, i); if (!chunk.append_crlf) { - msg_debug ("update signature with header: %*s", elt->len, elt->begin); - rspamd_dkim_hash_update (ctx->headers_hash, elt->begin, elt->len); + msg_debug ("update signature with header: %*s", + elt->len, + elt->begin); + rspamd_dkim_hash_update (ctx->headers_hash, + elt->begin, + elt->len); } else { - msg_debug ("update signature with header: %*s", elt->len + 1, elt->begin); - rspamd_dkim_hash_update (ctx->headers_hash, elt->begin, elt->len + 1); + msg_debug ("update signature with header: %*s", + elt->len + 1, + elt->begin); + rspamd_dkim_hash_update (ctx->headers_hash, + elt->begin, + elt->len + 1); } } } @@ -1271,15 +1505,22 @@ rspamd_dkim_canonize_header_simple (rspamd_dkim_context_t *ctx, const gchar *hea } static gboolean -rspamd_dkim_canonize_header (rspamd_dkim_context_t *ctx, struct rspamd_task *task, const gchar *header_name, - guint count, gboolean is_sig) +rspamd_dkim_canonize_header (rspamd_dkim_context_t *ctx, + struct rspamd_task *task, + const gchar *header_name, + guint count, + gboolean is_sig) { - struct raw_header *rh, *rh_iter; - guint rh_num = 0; - GList *nh = NULL, *cur; + struct raw_header *rh, *rh_iter; + guint rh_num = 0; + GList *nh = NULL, *cur; if (ctx->header_canon_type == DKIM_CANON_SIMPLE) { - return rspamd_dkim_canonize_header_simple (ctx, task->raw_headers_str, header_name, count, is_sig); + return rspamd_dkim_canonize_header_simple (ctx, + task->raw_headers_str, + header_name, + count, + is_sig); } else { rh = g_hash_table_lookup (task->raw_headers, header_name); @@ -1287,7 +1528,7 @@ rspamd_dkim_canonize_header (rspamd_dkim_context_t *ctx, struct rspamd_task *tas if (!is_sig) { rh_iter = rh; while (rh_iter) { - rh_num ++; + rh_num++; rh_iter = rh_iter->next; } @@ -1301,7 +1542,7 @@ rspamd_dkim_canonize_header (rspamd_dkim_context_t *ctx, struct rspamd_task *tas rh_iter = rh; while (rh_num) { rh_iter = rh_iter->next; - rh_num --; + rh_num--; } /* Now insert required headers */ while (rh_iter) { @@ -1311,7 +1552,8 @@ rspamd_dkim_canonize_header (rspamd_dkim_context_t *ctx, struct rspamd_task *tas cur = nh; while (cur) { rh = cur->data; - if (! rspamd_dkim_canonize_header_relaxed (ctx, rh->value, header_name, is_sig)) { + if (!rspamd_dkim_canonize_header_relaxed (ctx, rh->value, + header_name, is_sig)) { g_list_free (nh); return FALSE; } @@ -1323,7 +1565,10 @@ rspamd_dkim_canonize_header (rspamd_dkim_context_t *ctx, struct rspamd_task *tas } else { /* For signature check just use the first dkim header */ - rspamd_dkim_canonize_header_relaxed (ctx, rh->value, header_name, is_sig); + rspamd_dkim_canonize_header_relaxed (ctx, + rh->value, + header_name, + is_sig); } return TRUE; } @@ -1341,21 +1586,23 @@ rspamd_dkim_canonize_header (rspamd_dkim_context_t *ctx, struct rspamd_task *tas * @return */ gint -rspamd_dkim_check (rspamd_dkim_context_t *ctx, rspamd_dkim_key_t *key, struct rspamd_task *task) +rspamd_dkim_check (rspamd_dkim_context_t *ctx, + rspamd_dkim_key_t *key, + struct rspamd_task *task) { - const gchar *p, *headers_end = NULL, *end, *body_end; - gboolean got_cr = FALSE, got_crlf = FALSE, got_lf = FALSE; - gchar *digest; - gsize dlen; - gint res = DKIM_CONTINUE; - guint i; - struct rspamd_dkim_header *dh; + const gchar *p, *headers_end = NULL, *end, *body_end; + gboolean got_cr = FALSE, got_crlf = FALSE, got_lf = FALSE; + gchar *digest; + gsize dlen; + gint res = DKIM_CONTINUE; + guint i; + struct rspamd_dkim_header *dh; #ifdef HAVE_OPENSSL - gint nid; + gint nid; #endif - g_return_val_if_fail (ctx != NULL, DKIM_ERROR); - g_return_val_if_fail (key != NULL, DKIM_ERROR); + g_return_val_if_fail (ctx != NULL, DKIM_ERROR); + g_return_val_if_fail (key != NULL, DKIM_ERROR); g_return_val_if_fail (task->msg != NULL, DKIM_ERROR); /* First of all find place of body */ @@ -1422,7 +1669,7 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, rspamd_dkim_key_t *key, struct rs got_cr = FALSE; got_crlf = FALSE; } - p ++; + p++; } /* Start canonization of body part */ @@ -1442,7 +1689,7 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, rspamd_dkim_key_t *key, struct rs return DKIM_RECORD_ERROR; } /* Now canonize headers */ - for (i = 0; i < ctx->hlist->len; i ++) { + for (i = 0; i < ctx->hlist->len; i++) { dh = g_ptr_array_index (ctx->hlist, i); rspamd_dkim_canonize_header (ctx, task, dh->name, dh->count, FALSE); } diff --git a/src/libserver/dkim.h b/src/libserver/dkim.h index 29ec479b7..3340c2432 100644 --- a/src/libserver/dkim.h +++ b/src/libserver/dkim.h @@ -26,109 +26,109 @@ #define DKIM_H_ #include "config.h" -#include "event.h" #include "dns.h" +#include "event.h" #ifdef HAVE_OPENSSL -#include <openssl/rsa.h> #include <openssl/engine.h> +#include <openssl/rsa.h> #endif /* Main types and definitions */ -#define DKIM_SIGNHEADER "DKIM-Signature" - /* DKIM signature header */ +#define DKIM_SIGNHEADER "DKIM-Signature" +/* DKIM signature header */ /* special DNS tokens */ -#define DKIM_DNSKEYNAME "_domainkey" - /* reserved DNS sub-zone */ -#define DKIM_DNSPOLICYNAME "_adsp" /* reserved DNS sub-zone */ +#define DKIM_DNSKEYNAME "_domainkey" +/* reserved DNS sub-zone */ +#define DKIM_DNSPOLICYNAME "_adsp" /* reserved DNS sub-zone */ /* Canonization methods */ -#define DKIM_CANON_UNKNOWN (-1) /* unknown method */ -#define DKIM_CANON_SIMPLE 0 /* as specified in DKIM spec */ -#define DKIM_CANON_RELAXED 1 /* as specified in DKIM spec */ +#define DKIM_CANON_UNKNOWN (-1) /* unknown method */ +#define DKIM_CANON_SIMPLE 0 /* as specified in DKIM spec */ +#define DKIM_CANON_RELAXED 1 /* as specified in DKIM spec */ -#define DKIM_CANON_DEFAULT DKIM_CANON_SIMPLE +#define DKIM_CANON_DEFAULT DKIM_CANON_SIMPLE /* Signature methods */ -#define DKIM_SIGN_UNKNOWN (-2) /* unknown method */ -#define DKIM_SIGN_DEFAULT (-1) /* use internal default */ -#define DKIM_SIGN_RSASHA1 0 /* an RSA-signed SHA1 digest */ -#define DKIM_SIGN_RSASHA256 1 /* an RSA-signed SHA256 digest */ +#define DKIM_SIGN_UNKNOWN (-2) /* unknown method */ +#define DKIM_SIGN_DEFAULT (-1) /* use internal default */ +#define DKIM_SIGN_RSASHA1 0 /* an RSA-signed SHA1 digest */ +#define DKIM_SIGN_RSASHA256 1 /* an RSA-signed SHA256 digest */ /* Params */ -#define DKIM_PARAM_UNKNOWN (-1) /* unknown */ -#define DKIM_PARAM_SIGNATURE 0 /* b */ -#define DKIM_PARAM_SIGNALG 1 /* a */ -#define DKIM_PARAM_DOMAIN 2 /* d */ -#define DKIM_PARAM_CANONALG 3 /* c */ -#define DKIM_PARAM_QUERYMETHOD 4 /* q */ -#define DKIM_PARAM_SELECTOR 5 /* s */ -#define DKIM_PARAM_HDRLIST 6 /* h */ -#define DKIM_PARAM_VERSION 7 /* v */ -#define DKIM_PARAM_IDENTITY 8 /* i */ -#define DKIM_PARAM_TIMESTAMP 9 /* t */ -#define DKIM_PARAM_EXPIRATION 10 /* x */ -#define DKIM_PARAM_COPIEDHDRS 11 /* z */ -#define DKIM_PARAM_BODYHASH 12 /* bh */ -#define DKIM_PARAM_BODYLENGTH 13 /* l */ +#define DKIM_PARAM_UNKNOWN (-1) /* unknown */ +#define DKIM_PARAM_SIGNATURE 0 /* b */ +#define DKIM_PARAM_SIGNALG 1 /* a */ +#define DKIM_PARAM_DOMAIN 2 /* d */ +#define DKIM_PARAM_CANONALG 3 /* c */ +#define DKIM_PARAM_QUERYMETHOD 4 /* q */ +#define DKIM_PARAM_SELECTOR 5 /* s */ +#define DKIM_PARAM_HDRLIST 6 /* h */ +#define DKIM_PARAM_VERSION 7 /* v */ +#define DKIM_PARAM_IDENTITY 8 /* i */ +#define DKIM_PARAM_TIMESTAMP 9 /* t */ +#define DKIM_PARAM_EXPIRATION 10 /* x */ +#define DKIM_PARAM_COPIEDHDRS 11 /* z */ +#define DKIM_PARAM_BODYHASH 12 /* bh */ +#define DKIM_PARAM_BODYLENGTH 13 /* l */ /* Errors (from OpenDKIM) */ -#define DKIM_SIGERROR_UNKNOWN (-1) /* unknown error */ -#define DKIM_SIGERROR_OK 0 /* no error */ -#define DKIM_SIGERROR_VERSION 1 /* unsupported version */ -#define DKIM_SIGERROR_DOMAIN 2 /* invalid domain (d=/i=) */ -#define DKIM_SIGERROR_EXPIRED 3 /* signature expired */ -#define DKIM_SIGERROR_FUTURE 4 /* signature in the future */ -#define DKIM_SIGERROR_TIMESTAMPS 5 /* x= < t= */ -#define DKIM_SIGERROR_UNUSED 6 /* OBSOLETE */ -#define DKIM_SIGERROR_INVALID_HC 7 /* c= invalid (header) */ -#define DKIM_SIGERROR_INVALID_BC 8 /* c= invalid (body) */ -#define DKIM_SIGERROR_MISSING_A 9 /* a= missing */ -#define DKIM_SIGERROR_INVALID_A 10 /* a= invalid */ -#define DKIM_SIGERROR_MISSING_H 11 /* h= missing */ -#define DKIM_SIGERROR_INVALID_L 12 /* l= invalid */ -#define DKIM_SIGERROR_INVALID_Q 13 /* q= invalid */ -#define DKIM_SIGERROR_INVALID_QO 14 /* q= option invalid */ -#define DKIM_SIGERROR_MISSING_D 15 /* d= missing */ -#define DKIM_SIGERROR_EMPTY_D 16 /* d= empty */ -#define DKIM_SIGERROR_MISSING_S 17 /* s= missing */ -#define DKIM_SIGERROR_EMPTY_S 18 /* s= empty */ -#define DKIM_SIGERROR_MISSING_B 19 /* b= missing */ -#define DKIM_SIGERROR_EMPTY_B 20 /* b= empty */ -#define DKIM_SIGERROR_CORRUPT_B 21 /* b= corrupt */ -#define DKIM_SIGERROR_NOKEY 22 /* no key found in DNS */ -#define DKIM_SIGERROR_DNSSYNTAX 23 /* DNS reply corrupt */ -#define DKIM_SIGERROR_KEYFAIL 24 /* DNS query failed */ -#define DKIM_SIGERROR_MISSING_BH 25 /* bh= missing */ -#define DKIM_SIGERROR_EMPTY_BH 26 /* bh= empty */ -#define DKIM_SIGERROR_CORRUPT_BH 27 /* bh= corrupt */ -#define DKIM_SIGERROR_BADSIG 28 /* signature mismatch */ -#define DKIM_SIGERROR_SUBDOMAIN 29 /* unauthorized subdomain */ -#define DKIM_SIGERROR_MULTIREPLY 30 /* multiple records returned */ -#define DKIM_SIGERROR_EMPTY_H 31 /* h= empty */ -#define DKIM_SIGERROR_INVALID_H 32 /* h= missing req'd entries */ -#define DKIM_SIGERROR_TOOLARGE_L 33 /* l= value exceeds body size */ -#define DKIM_SIGERROR_MBSFAILED 34 /* "must be signed" failure */ -#define DKIM_SIGERROR_KEYVERSION 35 /* unknown key version */ -#define DKIM_SIGERROR_KEYUNKNOWNHASH 36 /* unknown key hash */ -#define DKIM_SIGERROR_KEYHASHMISMATCH 37 /* sig-key hash mismatch */ -#define DKIM_SIGERROR_NOTEMAILKEY 38 /* not an e-mail key */ -#define DKIM_SIGERROR_UNUSED2 39 /* OBSOLETE */ -#define DKIM_SIGERROR_KEYTYPEMISSING 40 /* key type missing */ -#define DKIM_SIGERROR_KEYTYPEUNKNOWN 41 /* key type unknown */ -#define DKIM_SIGERROR_KEYREVOKED 42 /* key revoked */ -#define DKIM_SIGERROR_KEYDECODE 43 /* key couldn't be decoded */ -#define DKIM_SIGERROR_MISSING_V 44 /* v= tag missing */ -#define DKIM_SIGERROR_EMPTY_V 45 /* v= tag empty */ +#define DKIM_SIGERROR_UNKNOWN (-1) /* unknown error */ +#define DKIM_SIGERROR_OK 0 /* no error */ +#define DKIM_SIGERROR_VERSION 1 /* unsupported version */ +#define DKIM_SIGERROR_DOMAIN 2 /* invalid domain (d=/i=) */ +#define DKIM_SIGERROR_EXPIRED 3 /* signature expired */ +#define DKIM_SIGERROR_FUTURE 4 /* signature in the future */ +#define DKIM_SIGERROR_TIMESTAMPS 5 /* x= < t= */ +#define DKIM_SIGERROR_UNUSED 6 /* OBSOLETE */ +#define DKIM_SIGERROR_INVALID_HC 7 /* c= invalid (header) */ +#define DKIM_SIGERROR_INVALID_BC 8 /* c= invalid (body) */ +#define DKIM_SIGERROR_MISSING_A 9 /* a= missing */ +#define DKIM_SIGERROR_INVALID_A 10 /* a= invalid */ +#define DKIM_SIGERROR_MISSING_H 11 /* h= missing */ +#define DKIM_SIGERROR_INVALID_L 12 /* l= invalid */ +#define DKIM_SIGERROR_INVALID_Q 13 /* q= invalid */ +#define DKIM_SIGERROR_INVALID_QO 14 /* q= option invalid */ +#define DKIM_SIGERROR_MISSING_D 15 /* d= missing */ +#define DKIM_SIGERROR_EMPTY_D 16 /* d= empty */ +#define DKIM_SIGERROR_MISSING_S 17 /* s= missing */ +#define DKIM_SIGERROR_EMPTY_S 18 /* s= empty */ +#define DKIM_SIGERROR_MISSING_B 19 /* b= missing */ +#define DKIM_SIGERROR_EMPTY_B 20 /* b= empty */ +#define DKIM_SIGERROR_CORRUPT_B 21 /* b= corrupt */ +#define DKIM_SIGERROR_NOKEY 22 /* no key found in DNS */ +#define DKIM_SIGERROR_DNSSYNTAX 23 /* DNS reply corrupt */ +#define DKIM_SIGERROR_KEYFAIL 24 /* DNS query failed */ +#define DKIM_SIGERROR_MISSING_BH 25 /* bh= missing */ +#define DKIM_SIGERROR_EMPTY_BH 26 /* bh= empty */ +#define DKIM_SIGERROR_CORRUPT_BH 27 /* bh= corrupt */ +#define DKIM_SIGERROR_BADSIG 28 /* signature mismatch */ +#define DKIM_SIGERROR_SUBDOMAIN 29 /* unauthorized subdomain */ +#define DKIM_SIGERROR_MULTIREPLY 30 /* multiple records returned */ +#define DKIM_SIGERROR_EMPTY_H 31 /* h= empty */ +#define DKIM_SIGERROR_INVALID_H 32 /* h= missing req'd entries */ +#define DKIM_SIGERROR_TOOLARGE_L 33 /* l= value exceeds body size */ +#define DKIM_SIGERROR_MBSFAILED 34 /* "must be signed" failure */ +#define DKIM_SIGERROR_KEYVERSION 35 /* unknown key version */ +#define DKIM_SIGERROR_KEYUNKNOWNHASH 36 /* unknown key hash */ +#define DKIM_SIGERROR_KEYHASHMISMATCH 37 /* sig-key hash mismatch */ +#define DKIM_SIGERROR_NOTEMAILKEY 38 /* not an e-mail key */ +#define DKIM_SIGERROR_UNUSED2 39 /* OBSOLETE */ +#define DKIM_SIGERROR_KEYTYPEMISSING 40 /* key type missing */ +#define DKIM_SIGERROR_KEYTYPEUNKNOWN 41 /* key type unknown */ +#define DKIM_SIGERROR_KEYREVOKED 42 /* key revoked */ +#define DKIM_SIGERROR_KEYDECODE 43 /* key couldn't be decoded */ +#define DKIM_SIGERROR_MISSING_V 44 /* v= tag missing */ +#define DKIM_SIGERROR_EMPTY_V 45 /* v= tag empty */ /* Check results */ -#define DKIM_CONTINUE 0 /* continue */ -#define DKIM_REJECT 1 /* reject */ -#define DKIM_TRYAGAIN 2 /* try again later */ -#define DKIM_NOTFOUND 3 /* requested record not found */ -#define DKIM_RECORD_ERROR 4 /* error requesting record */ +#define DKIM_CONTINUE 0 /* continue */ +#define DKIM_REJECT 1 /* reject */ +#define DKIM_TRYAGAIN 2 /* try again later */ +#define DKIM_NOTFOUND 3 /* requested record not found */ +#define DKIM_RECORD_ERROR 4 /* error requesting record */ typedef struct rspamd_dkim_context_s { rspamd_mempool_t *pool; @@ -167,7 +167,8 @@ rspamd_dkim_key_t; struct rspamd_task; /* Err MUST be freed if it is not NULL, key is allocated by slice allocator */ -typedef void (*dkim_key_handler_f)(rspamd_dkim_key_t *key, gsize keylen, rspamd_dkim_context_t *ctx, gpointer ud, GError *err); +typedef void (*dkim_key_handler_f)(rspamd_dkim_key_t *key, gsize keylen, + rspamd_dkim_context_t *ctx, gpointer ud, GError *err); /** * Create new dkim context from signature @@ -177,7 +178,10 @@ typedef void (*dkim_key_handler_f)(rspamd_dkim_key_t *key, gsize keylen, rspamd_ * @param err pointer to error object * @return new context or NULL */ -rspamd_dkim_context_t* rspamd_create_dkim_context (const gchar *sig, rspamd_mempool_t *pool, guint time_jitter, GError **err); +rspamd_dkim_context_t * rspamd_create_dkim_context (const gchar *sig, + rspamd_mempool_t *pool, + guint time_jitter, + GError **err); /** * Make DNS request for specified context and obtain and parse key @@ -186,8 +190,11 @@ rspamd_dkim_context_t* rspamd_create_dkim_context (const gchar *sig, rspamd_memp * @param s async session to make request * @return */ -gboolean rspamd_get_dkim_key (rspamd_dkim_context_t *ctx, struct rspamd_dns_resolver *resolver, - struct rspamd_async_session *s, dkim_key_handler_f handler, gpointer ud); +gboolean rspamd_get_dkim_key (rspamd_dkim_context_t *ctx, + struct rspamd_dns_resolver *resolver, + struct rspamd_async_session *s, + dkim_key_handler_f handler, + gpointer ud); /** * Check task for dkim context using dkim key @@ -196,7 +203,9 @@ gboolean rspamd_get_dkim_key (rspamd_dkim_context_t *ctx, struct rspamd_dns_reso * @param task task to check * @return */ -gint rspamd_dkim_check (rspamd_dkim_context_t *ctx, rspamd_dkim_key_t *key, struct rspamd_task *task); +gint rspamd_dkim_check (rspamd_dkim_context_t *ctx, + rspamd_dkim_key_t *key, + struct rspamd_task *task); /** * Free DKIM key diff --git a/src/libserver/dns.c b/src/libserver/dns.c index c1303e3ea..a95232fca 100644 --- a/src/libserver/dns.c +++ b/src/libserver/dns.c @@ -26,9 +26,9 @@ #include "config.h" #include "dns.h" #include "main.h" -#include "utlist.h" -#include "uthash.h" #include "rdns_event.h" +#include "uthash.h" +#include "utlist.h" struct rspamd_dns_resolver { struct rdns_resolver *r; @@ -48,7 +48,7 @@ static void rspamd_dns_fin_cb (gpointer arg) { struct rdns_request *req = arg; - + rdns_request_release (req); } @@ -67,25 +67,33 @@ rspamd_dns_callback (struct rdns_reply *reply, gpointer ud) remove_normal_event (reqdata->session, rspamd_dns_fin_cb, reqdata->req); } -gboolean +gboolean make_dns_request (struct rspamd_dns_resolver *resolver, - struct rspamd_async_session *session, rspamd_mempool_t *pool, dns_callback_type cb, - gpointer ud, enum rdns_request_type type, const char *name) + struct rspamd_async_session *session, + rspamd_mempool_t *pool, + dns_callback_type cb, + gpointer ud, + enum rdns_request_type type, + const char *name) { struct rdns_request *req; struct rspamd_dns_request_ud *reqdata; - - reqdata = rspamd_mempool_alloc (pool, sizeof (struct rspamd_dns_request_ud)); + + reqdata = + rspamd_mempool_alloc (pool, sizeof (struct rspamd_dns_request_ud)); reqdata->session = session; reqdata->cb = cb; reqdata->ud = ud; req = rdns_make_request_full (resolver->r, rspamd_dns_callback, reqdata, - resolver->request_timeout, resolver->max_retransmits, 1, name, type); + resolver->request_timeout, resolver->max_retransmits, 1, name, + type); if (req != NULL) { - register_async_event (session, (event_finalizer_t)rspamd_dns_fin_cb, req, - g_quark_from_static_string ("dns resolver")); + register_async_event (session, + (event_finalizer_t)rspamd_dns_fin_cb, + req, + g_quark_from_static_string ("dns resolver")); reqdata->req = req; } else { @@ -97,13 +105,15 @@ make_dns_request (struct rspamd_dns_resolver *resolver, struct rspamd_dns_resolver * -dns_resolver_init (rspamd_logger_t *logger, struct event_base *ev_base, struct rspamd_config *cfg) +dns_resolver_init (rspamd_logger_t *logger, + struct event_base *ev_base, + struct rspamd_config *cfg) { - GList *cur; - struct rspamd_dns_resolver *new; - gchar *begin, *p, *err; - gint priority; - + GList *cur; + struct rspamd_dns_resolver *new; + gchar *begin, *p, *err; + gint priority; + new = g_slice_alloc0 (sizeof (struct rspamd_dns_resolver)); new->ev_base = ev_base; new->request_timeout = cfg->dns_timeout; @@ -112,12 +122,15 @@ dns_resolver_init (rspamd_logger_t *logger, struct event_base *ev_base, struct r new->r = rdns_resolver_new (); rdns_bind_libevent (new->r, new->ev_base); rdns_resolver_set_log_level (new->r, cfg->log_level); - rdns_resolver_set_logger (new->r, (rdns_log_function)rspamd_common_logv, logger); + rdns_resolver_set_logger (new->r, + (rdns_log_function)rspamd_common_logv, + logger); if (cfg->nameservers == NULL) { /* Parse resolv.conf */ if (!rdns_resolver_parse_resolv_conf (new->r, "/etc/resolv.conf")) { - msg_err ("cannot parse resolv.conf and no nameservers defined, so no ways to resolve addresses"); + msg_err ( + "cannot parse resolv.conf and no nameservers defined, so no ways to resolve addresses"); return new; } } @@ -128,16 +141,19 @@ dns_resolver_init (rspamd_logger_t *logger, struct event_base *ev_base, struct r p = strchr (begin, ':'); if (p != NULL) { *p = '\0'; - p ++; + p++; priority = strtoul (p, &err, 10); if (err != NULL && *err != '\0') { - msg_info ("bad character '%x', must be 'm' or 's' or a numeric priority", *err); + msg_info ( + "bad character '%x', must be 'm' or 's' or a numeric priority", + *err); } } else { priority = 0; } - if (!rdns_resolver_add_server (new->r, begin, 53, priority, cfg->dns_io_per_server)) { + if (!rdns_resolver_add_server (new->r, begin, 53, priority, + cfg->dns_io_per_server)) { msg_warn ("cannot parse ip address of nameserver: %s", begin); cur = g_list_next (cur); continue; diff --git a/src/libserver/dns.h b/src/libserver/dns.h index 7b3bbd208..147a641d3 100644 --- a/src/libserver/dns.h +++ b/src/libserver/dns.h @@ -27,9 +27,9 @@ #define RSPAMD_DNS_H #include "config.h" -#include "mem_pool.h" #include "events.h" #include "logger.h" +#include "mem_pool.h" #include "rdns.h" struct rspamd_dns_resolver; @@ -39,8 +39,8 @@ struct rspamd_dns_resolver; /** * Init DNS resolver, params are obtained from a config file or system file /etc/resolv.conf */ -struct rspamd_dns_resolver *dns_resolver_init (rspamd_logger_t *logger, - struct event_base *ev_base, struct rspamd_config *cfg); +struct rspamd_dns_resolver * dns_resolver_init (rspamd_logger_t *logger, + struct event_base *ev_base, struct rspamd_config *cfg); /** * Make a DNS request @@ -53,8 +53,12 @@ struct rspamd_dns_resolver *dns_resolver_init (rspamd_logger_t *logger, * @param ... string or ip address based on a request type * @return TRUE if request was sent. */ -gboolean make_dns_request (struct rspamd_dns_resolver *resolver, - struct rspamd_async_session *session, rspamd_mempool_t *pool, - dns_callback_type cb, gpointer ud, enum rdns_request_type type, const char *name); +gboolean make_dns_request (struct rspamd_dns_resolver *resolver, + struct rspamd_async_session *session, + rspamd_mempool_t *pool, + dns_callback_type cb, + gpointer ud, + enum rdns_request_type type, + const char *name); #endif diff --git a/src/libserver/dynamic_cfg.c b/src/libserver/dynamic_cfg.c index d0dabef67..1ba2b2baf 100644 --- a/src/libserver/dynamic_cfg.c +++ b/src/libserver/dynamic_cfg.c @@ -22,34 +22,34 @@ */ #include "config.h" -#include "main.h" -#include "map.h" -#include "filter.h" #include "dynamic_cfg.h" +#include "filter.h" #include "json/jansson.h" +#include "main.h" +#include "map.h" struct dynamic_cfg_symbol { - gchar *name; - gdouble value; + gchar *name; + gdouble value; }; struct dynamic_cfg_action { - enum rspamd_metric_action action; - gdouble value; + enum rspamd_metric_action action; + gdouble value; }; struct dynamic_cfg_metric { - GList *symbols; - struct dynamic_cfg_action actions[METRIC_ACTION_MAX]; - gchar *name; + GList *symbols; + struct dynamic_cfg_action actions[METRIC_ACTION_MAX]; + gchar *name; }; struct config_json_buf { - gchar *buf; - gchar *pos; - size_t buflen; - struct rspamd_config *cfg; - GList *config_metrics; + gchar *buf; + gchar *pos; + size_t buflen; + struct rspamd_config *cfg; + GList *config_metrics; }; /** @@ -59,9 +59,9 @@ struct config_json_buf { static void dynamic_cfg_free (GList *conf_metrics) { - GList *cur, *cur_elt; - struct dynamic_cfg_metric *metric; - struct dynamic_cfg_symbol *sym; + GList *cur, *cur_elt; + struct dynamic_cfg_metric *metric; + struct dynamic_cfg_symbol *sym; if (conf_metrics) { cur = conf_metrics; @@ -91,44 +91,50 @@ dynamic_cfg_free (GList *conf_metrics) static void apply_dynamic_conf (GList *conf_metrics, struct rspamd_config *cfg) { - GList *cur, *cur_elt; - struct dynamic_cfg_metric *metric; - struct dynamic_cfg_symbol *sym; - struct dynamic_cfg_action *act; - struct metric *real_metric; - struct metric_action *real_act; - gdouble *w; - gint i, j; + GList *cur, *cur_elt; + struct dynamic_cfg_metric *metric; + struct dynamic_cfg_symbol *sym; + struct dynamic_cfg_action *act; + struct metric *real_metric; + struct metric_action *real_act; + gdouble *w; + gint i, j; cur = conf_metrics; while (cur) { metric = cur->data; - if ((real_metric = g_hash_table_lookup (cfg->metrics, metric->name)) != NULL) { + if ((real_metric = + g_hash_table_lookup (cfg->metrics, metric->name)) != NULL) { cur_elt = metric->symbols; while (cur_elt) { sym = cur_elt->data; - if ((w = g_hash_table_lookup (real_metric->symbols, sym->name)) != NULL) { + if ((w = + g_hash_table_lookup (real_metric->symbols, + sym->name)) != NULL) { *w = sym->value; } else { - msg_info ("symbol %s is not found in the main configuration", sym->name); + msg_info ( + "symbol %s is not found in the main configuration", + sym->name); } cur_elt = g_list_next (cur_elt); } - for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i ++) { + for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) { act = &metric->actions[i]; if (act->value < 0) { continue; } - for (j = METRIC_ACTION_REJECT; j < METRIC_ACTION_MAX; j ++) { + for (j = METRIC_ACTION_REJECT; j < METRIC_ACTION_MAX; j++) { real_act = &real_metric->actions[j]; if (real_act->action == act->action) { real_act->score = act->value; } /* Update required score accordingly to metric's action */ if (act->action == METRIC_ACTION_REJECT) { - real_metric->actions[METRIC_ACTION_REJECT].score = act->value; + real_metric->actions[METRIC_ACTION_REJECT].score = + act->value; } } } @@ -138,11 +144,14 @@ apply_dynamic_conf (GList *conf_metrics, struct rspamd_config *cfg) } /* Callbacks for reading json dynamic rules */ -gchar * -json_config_read_cb (rspamd_mempool_t * pool, gchar * chunk, gint len, struct map_cb_data *data) +gchar * +json_config_read_cb (rspamd_mempool_t * pool, + gchar * chunk, + gint len, + struct map_cb_data *data) { - struct config_json_buf *jb; - gint free, off; + struct config_json_buf *jb; + gint free, off; if (data->cur_data == NULL) { jb = g_malloc (sizeof (struct config_json_buf)); @@ -182,14 +191,14 @@ json_config_read_cb (rspamd_mempool_t * pool, gchar * chunk, gint len, struct ma void json_config_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) { - struct config_json_buf *jb; - guint nelts, i, j, selts; - gint test_act; - json_t *js, *cur_elt, *cur_nm, *it_val; - json_error_t je; - struct dynamic_cfg_metric *cur_metric; - struct dynamic_cfg_symbol *cur_symbol; - struct dynamic_cfg_action *cur_action; + struct config_json_buf *jb; + guint nelts, i, j, selts; + gint test_act; + json_t *js, *cur_elt, *cur_nm, *it_val; + json_error_t je; + struct dynamic_cfg_metric *cur_metric; + struct dynamic_cfg_symbol *cur_symbol; + struct dynamic_cfg_action *cur_action; if (data->prev_data) { jb = data->prev_data; @@ -217,7 +226,9 @@ json_config_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) js = json_loads (jb->buf, &je); if (!js) { - msg_err ("cannot load json data: parse error %s, on line %d", je.text, je.line); + msg_err ("cannot load json data: parse error %s, on line %d", + je.text, + je.line); return; } @@ -242,11 +253,12 @@ json_config_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) cur_nm = json_object_get (cur_elt, "metric"); if (!cur_nm || !json_is_string (cur_nm)) { - msg_err ("loaded json metric object element has no 'metric' attribute"); + msg_err ( + "loaded json metric object element has no 'metric' attribute"); continue; } cur_metric = g_slice_alloc0 (sizeof (struct dynamic_cfg_metric)); - for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i ++) { + for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) { cur_metric->actions[i].value = -1.0; } cur_metric->name = g_strdup (json_string_value (cur_nm)); @@ -254,18 +266,27 @@ json_config_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) /* Parse symbols */ if (cur_nm && json_is_array (cur_nm)) { selts = json_array_size (cur_nm); - for (j = 0; j < selts; j ++) { + for (j = 0; j < selts; j++) { it_val = json_array_get (cur_nm, j); if (it_val && json_is_object (it_val)) { - if (json_object_get (it_val, "name") && json_object_get (it_val, "value")) { - cur_symbol = g_slice_alloc0 (sizeof (struct dynamic_cfg_symbol)); - cur_symbol->name = g_strdup (json_string_value (json_object_get (it_val, "name"))); - cur_symbol->value = json_number_value (json_object_get (it_val, "value")); + if (json_object_get (it_val, + "name") && json_object_get (it_val, "value")) { + cur_symbol = + g_slice_alloc0 (sizeof (struct dynamic_cfg_symbol)); + cur_symbol->name = + g_strdup (json_string_value (json_object_get (it_val, + "name"))); + cur_symbol->value = + json_number_value (json_object_get (it_val, + "value")); /* Insert symbol */ - cur_metric->symbols = g_list_prepend (cur_metric->symbols, cur_symbol); + cur_metric->symbols = g_list_prepend ( + cur_metric->symbols, + cur_symbol); } else { - msg_info ("json symbol object has no mandatory 'name' and 'value' attributes"); + msg_info ( + "json symbol object has no mandatory 'name' and 'value' attributes"); } } } @@ -274,21 +295,29 @@ json_config_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) /* Parse actions */ if (cur_nm && json_is_array (cur_nm)) { selts = json_array_size (cur_nm); - for (j = 0; j < selts; j ++) { + for (j = 0; j < selts; j++) { it_val = json_array_get (cur_nm, j); if (it_val && json_is_object (it_val)) { - if (json_object_get (it_val, "name") && json_object_get (it_val, "value")) { - if (!check_action_str (json_string_value (json_object_get (it_val, "name")), &test_act)) { - msg_err ("unknown action: %s", json_string_value (json_object_get (it_val, "name"))); - g_slice_free1 (sizeof (struct dynamic_cfg_action), cur_action); + if (json_object_get (it_val, + "name") && json_object_get (it_val, "value")) { + if (!check_action_str (json_string_value ( + json_object_get (it_val, "name")), &test_act)) { + msg_err ("unknown action: %s", + json_string_value (json_object_get (it_val, + "name"))); + g_slice_free1 (sizeof (struct dynamic_cfg_action), + cur_action); continue; } cur_action = &cur_metric->actions[test_act]; cur_action->action = test_act; - cur_action->value = json_number_value (json_object_get (it_val, "value")); + cur_action->value = + json_number_value (json_object_get (it_val, + "value")); } else { - msg_info ("json symbol object has no mandatory 'name' and 'value' attributes"); + msg_info ( + "json symbol object has no mandatory 'name' and 'value' attributes"); } } } @@ -314,7 +343,7 @@ json_config_fin_cb (rspamd_mempool_t * pool, struct map_cb_data *data) void init_dynamic_config (struct rspamd_config *cfg) { - struct config_json_buf *jb, **pjb; + struct config_json_buf *jb, **pjb; if (cfg->dynamic_conf == NULL) { /* No dynamic conf has been specified, so do not try to load it */ @@ -327,7 +356,8 @@ init_dynamic_config (struct rspamd_config *cfg) jb->buf = NULL; jb->cfg = cfg; *pjb = jb; - if (!add_map (cfg, cfg->dynamic_conf, "Dynamic configuration map", json_config_read_cb, json_config_fin_cb, (void **)pjb)) { + if (!add_map (cfg, cfg->dynamic_conf, "Dynamic configuration map", + json_config_read_cb, json_config_fin_cb, (void **)pjb)) { msg_err ("cannot add map for configuration %s", cfg->dynamic_conf); } } @@ -335,13 +365,13 @@ init_dynamic_config (struct rspamd_config *cfg) static gboolean dump_dynamic_list (gint fd, GList *rules) { - GList *cur, *cur_elt; - struct dynamic_cfg_metric *metric; - struct dynamic_cfg_symbol *sym; - struct dynamic_cfg_action *act; - FILE *f; - gint i; - gboolean start = TRUE; + GList *cur, *cur_elt; + struct dynamic_cfg_metric *metric; + struct dynamic_cfg_symbol *sym; + struct dynamic_cfg_action *act; + FILE *f; + gint i; + gboolean start = TRUE; /* Open buffered stream for the descriptor */ if ((f = fdopen (fd, "a+")) == NULL) { @@ -363,10 +393,16 @@ dump_dynamic_list (gint fd, GList *rules) sym = cur_elt->data; cur_elt = g_list_next (cur_elt); if (cur_elt) { - fprintf (f, " {\"name\": \"%s\",\"value\": %.2f},\n", sym->name, sym->value); + fprintf (f, + " {\"name\": \"%s\",\"value\": %.2f},\n", + sym->name, + sym->value); } else { - fprintf (f, " {\"name\": \"%s\",\"value\": %.2f}\n", sym->name, sym->value); + fprintf (f, + " {\"name\": \"%s\",\"value\": %.2f}\n", + sym->name, + sym->value); } } if (metric->actions) { @@ -379,13 +415,14 @@ dump_dynamic_list (gint fd, GList *rules) if (metric->actions) { fprintf (f, " \"actions\": [\n"); - for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i ++) { + for (i = METRIC_ACTION_REJECT; i < METRIC_ACTION_MAX; i++) { act = &metric->actions[i]; if (act->value < 0) { continue; } fprintf (f, " %s{\"name\": \"%s\",\"value\": %.2f}\n", - (start ? "" : ","), str_action_metric (act->action), act->value); + (start ? "" : ","), str_action_metric ( + act->action), act->value); if (start) { start = FALSE; } @@ -414,9 +451,9 @@ dump_dynamic_list (gint fd, GList *rules) gboolean dump_dynamic_config (struct rspamd_config *cfg) { - struct stat st; - gchar *dir, pathbuf[PATH_MAX]; - gint fd; + struct stat st; + gchar *dir, pathbuf[PATH_MAX]; + gint fd; if (cfg->dynamic_conf == NULL || cfg->current_dynamic_conf == NULL) { /* No dynamic conf has been specified, so do not try to dump it */ @@ -434,15 +471,20 @@ dump_dynamic_config (struct rspamd_config *cfg) } if (stat (cfg->dynamic_conf, &st) == -1) { - msg_debug ("%s is unavailable: %s", cfg->dynamic_conf, strerror (errno)); - st.st_mode = S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH; + msg_debug ("%s is unavailable: %s", cfg->dynamic_conf, + strerror (errno)); + st.st_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; } if (access (dir, W_OK | R_OK) == -1) { msg_warn ("%s is inaccessible: %s", dir, strerror (errno)); g_free (dir); return FALSE; } - rspamd_snprintf (pathbuf, sizeof (pathbuf), "%s%crconf-XXXXXX", dir, G_DIR_SEPARATOR); + rspamd_snprintf (pathbuf, + sizeof (pathbuf), + "%s%crconf-XXXXXX", + dir, + G_DIR_SEPARATOR); g_free (dir); #ifdef HAVE_MKSTEMP /* Umask is set before */ @@ -490,11 +532,14 @@ dump_dynamic_config (struct rspamd_config *cfg) * @return */ gboolean -add_dynamic_symbol (struct rspamd_config *cfg, const gchar *metric_name, const gchar *symbol, gdouble value) +add_dynamic_symbol (struct rspamd_config *cfg, + const gchar *metric_name, + const gchar *symbol, + gdouble value) { - GList *cur; - struct dynamic_cfg_metric *metric = NULL; - struct dynamic_cfg_symbol *sym = NULL; + GList *cur; + struct dynamic_cfg_metric *metric = NULL; + struct dynamic_cfg_symbol *sym = NULL; if (cfg->dynamic_conf == NULL) { msg_info ("dynamic conf is disabled"); @@ -541,7 +586,8 @@ add_dynamic_symbol (struct rspamd_config *cfg, const gchar *metric_name, const g sym->value = value; metric->symbols = g_list_prepend (metric->symbols, sym); metric->name = g_strdup (metric_name); - cfg->current_dynamic_conf = g_list_prepend (cfg->current_dynamic_conf, metric); + cfg->current_dynamic_conf = g_list_prepend (cfg->current_dynamic_conf, + metric); msg_debug ("create metric %s for symbol %s", metric_name, symbol); } @@ -560,10 +606,13 @@ add_dynamic_symbol (struct rspamd_config *cfg, const gchar *metric_name, const g * @return */ gboolean -add_dynamic_action (struct rspamd_config *cfg, const gchar *metric_name, guint action, gdouble value) +add_dynamic_action (struct rspamd_config *cfg, + const gchar *metric_name, + guint action, + gdouble value) { - GList *cur; - struct dynamic_cfg_metric *metric = NULL; + GList *cur; + struct dynamic_cfg_metric *metric = NULL; if (cfg->dynamic_conf == NULL) { msg_info ("dynamic conf is disabled"); @@ -589,7 +638,8 @@ add_dynamic_action (struct rspamd_config *cfg, const gchar *metric_name, guint a metric = g_slice_alloc0 (sizeof (struct dynamic_cfg_metric)); metric->actions[action].value = value; metric->name = g_strdup (metric_name); - cfg->current_dynamic_conf = g_list_prepend (cfg->current_dynamic_conf, metric); + cfg->current_dynamic_conf = g_list_prepend (cfg->current_dynamic_conf, + metric); msg_debug ("create metric %s for action %d", metric_name, action); } diff --git a/src/libserver/dynamic_cfg.h b/src/libserver/dynamic_cfg.h index a8cb02ae5..d69ee30f5 100644 --- a/src/libserver/dynamic_cfg.h +++ b/src/libserver/dynamic_cfg.h @@ -25,8 +25,8 @@ #ifndef DYNAMIC_CFG_H_ #define DYNAMIC_CFG_H_ -#include "config.h" #include "cfg_file.h" +#include "config.h" /** * Init dynamic configuration using map logic and specific configuration @@ -49,7 +49,10 @@ gboolean dump_dynamic_config (struct rspamd_config *cfg); * @param value value of symbol * @return */ -gboolean add_dynamic_symbol (struct rspamd_config *cfg, const gchar *metric, const gchar *symbol, gdouble value); +gboolean add_dynamic_symbol (struct rspamd_config *cfg, + const gchar *metric, + const gchar *symbol, + gdouble value); /** @@ -60,7 +63,10 @@ gboolean add_dynamic_symbol (struct rspamd_config *cfg, const gchar *metric, con * @param value value of symbol * @return */ -gboolean add_dynamic_action (struct rspamd_config *cfg, const gchar *metric, guint action, gdouble value); +gboolean add_dynamic_action (struct rspamd_config *cfg, + const gchar *metric, + guint action, + gdouble value); #endif /* DYNAMIC_CFG_H_ */ diff --git a/src/libserver/events.c b/src/libserver/events.c index 48dcb3444..f265de3fe 100644 --- a/src/libserver/events.c +++ b/src/libserver/events.c @@ -23,13 +23,13 @@ */ #include "config.h" -#include "main.h" #include "events.h" +#include "main.h" static gboolean rspamd_event_equal (gconstpointer a, gconstpointer b) { - const struct rspamd_async_event *ev1 = a, *ev2 = b; + const struct rspamd_async_event *ev1 = a, *ev2 = b; if (ev1->fin == ev2->fin) { return ev1->user_data == ev2->user_data; @@ -41,7 +41,7 @@ rspamd_event_equal (gconstpointer a, gconstpointer b) static guint rspamd_event_hash (gconstpointer a) { - const struct rspamd_async_event *ev = a; + const struct rspamd_async_event *ev = a; return GPOINTER_TO_UINT (ev->user_data); } @@ -50,7 +50,7 @@ rspamd_event_hash (gconstpointer a) static void event_mutex_free (gpointer data) { - GMutex *mtx = data; + GMutex *mtx = data; g_mutex_free (mtx); } @@ -58,17 +58,17 @@ event_mutex_free (gpointer data) static void event_cond_free (gpointer data) { - GCond *cond = data; + GCond *cond = data; g_cond_free (cond); } #endif -struct rspamd_async_session * +struct rspamd_async_session * new_async_session (rspamd_mempool_t * pool, session_finalizer_t fin, - event_finalizer_t restore, event_finalizer_t cleanup, void *user_data) + event_finalizer_t restore, event_finalizer_t cleanup, void *user_data) { - struct rspamd_async_session *new; + struct rspamd_async_session *new; new = rspamd_mempool_alloc (pool, sizeof (struct rspamd_async_session)); new->pool = pool; @@ -81,27 +81,40 @@ new_async_session (rspamd_mempool_t * pool, session_finalizer_t fin, #if ((GLIB_MAJOR_VERSION == 2) && (GLIB_MINOR_VERSION <= 30)) new->mtx = g_mutex_new (); new->cond = g_cond_new (); - rspamd_mempool_add_destructor (pool, (rspamd_mempool_destruct_t) event_mutex_free, new->mtx); - rspamd_mempool_add_destructor (pool, (rspamd_mempool_destruct_t) event_cond_free, new->cond); + rspamd_mempool_add_destructor (pool, + (rspamd_mempool_destruct_t) event_mutex_free, + new->mtx); + rspamd_mempool_add_destructor (pool, + (rspamd_mempool_destruct_t) event_cond_free, + new->cond); #else new->mtx = rspamd_mempool_alloc (pool, sizeof (GMutex)); g_mutex_init (new->mtx); new->cond = rspamd_mempool_alloc (pool, sizeof (GCond)); g_cond_init (new->cond); - rspamd_mempool_add_destructor (pool, (rspamd_mempool_destruct_t) g_mutex_clear, new->mtx); - rspamd_mempool_add_destructor (pool, (rspamd_mempool_destruct_t) g_cond_clear, new->cond); + rspamd_mempool_add_destructor (pool, + (rspamd_mempool_destruct_t) g_mutex_clear, + new->mtx); + rspamd_mempool_add_destructor (pool, + (rspamd_mempool_destruct_t) g_cond_clear, + new->cond); #endif new->threads = 0; - rspamd_mempool_add_destructor (pool, (rspamd_mempool_destruct_t) g_hash_table_destroy, new->events); + rspamd_mempool_add_destructor (pool, + (rspamd_mempool_destruct_t) g_hash_table_destroy, + new->events); return new; } void -register_async_event (struct rspamd_async_session *session, event_finalizer_t fin, void *user_data, GQuark subsystem) +register_async_event (struct rspamd_async_session *session, + event_finalizer_t fin, + void *user_data, + GQuark subsystem) { - struct rspamd_async_event *new; + struct rspamd_async_event *new; if (session == NULL) { msg_info ("session is NULL"); @@ -109,23 +122,28 @@ register_async_event (struct rspamd_async_session *session, event_finalizer_t fi } g_mutex_lock (session->mtx); - new = rspamd_mempool_alloc (session->pool, sizeof (struct rspamd_async_event)); + new = rspamd_mempool_alloc (session->pool, + sizeof (struct rspamd_async_event)); new->fin = fin; new->user_data = user_data; new->subsystem = subsystem; g_hash_table_insert (session->events, new, new); - msg_debug ("added event: %p, pending %d events, subsystem: %s", user_data, g_hash_table_size (session->events), - g_quark_to_string (subsystem)); + msg_debug ("added event: %p, pending %d events, subsystem: %s", + user_data, + g_hash_table_size (session->events), + g_quark_to_string (subsystem)); g_mutex_unlock (session->mtx); } void -remove_normal_event (struct rspamd_async_session *session, event_finalizer_t fin, void *ud) +remove_normal_event (struct rspamd_async_session *session, + event_finalizer_t fin, + void *ud) { - struct rspamd_async_event search_ev, *found_ev; + struct rspamd_async_event search_ev, *found_ev; if (session == NULL) { msg_info ("session is NULL"); @@ -136,10 +154,12 @@ remove_normal_event (struct rspamd_async_session *session, event_finalizer_t fin /* Search for event */ search_ev.fin = fin; search_ev.user_data = ud; - if ((found_ev = g_hash_table_lookup (session->events, &search_ev)) != NULL) { + if ((found_ev = + g_hash_table_lookup (session->events, &search_ev)) != NULL) { g_hash_table_remove (session->events, found_ev); msg_debug ("removed event: %p, subsystem: %s, pending %d events", ud, - g_quark_to_string (found_ev->subsystem), g_hash_table_size (session->events)); + g_quark_to_string (found_ev->subsystem), + g_hash_table_size (session->events)); /* Remove event */ fin (ud); } @@ -151,11 +171,11 @@ remove_normal_event (struct rspamd_async_session *session, event_finalizer_t fin static gboolean rspamd_session_destroy (gpointer k, gpointer v, gpointer unused) { - struct rspamd_async_event *ev = v; + struct rspamd_async_event *ev = v; /* Call event's finalizer */ msg_debug ("removed event on destroy: %p, subsystem: %s", ev->user_data, - g_quark_to_string (ev->subsystem)); + g_quark_to_string (ev->subsystem)); if (ev->fin != NULL) { ev->fin (ev->user_data); @@ -174,14 +194,16 @@ destroy_session (struct rspamd_async_session *session) g_mutex_lock (session->mtx); if (session->threads > 0) { - /* Wait for conditional variable to finish processing */ + /* Wait for conditional variable to finish processing */ g_mutex_unlock (session->mtx); g_cond_wait (session->cond, session->mtx); } session->wanna_die = TRUE; - g_hash_table_foreach_remove (session->events, rspamd_session_destroy, session); + g_hash_table_foreach_remove (session->events, + rspamd_session_destroy, + session); /* Mutex can be destroyed here */ g_mutex_unlock (session->mtx); @@ -204,7 +226,7 @@ check_session_pending (struct rspamd_async_session *session) } if (session->fin != NULL) { g_mutex_unlock (session->mtx); - if (! session->fin (session->user_data)) { + if (!session->fin (session->user_data)) { /* Session finished incompletely, perform restoration */ if (session->restore != NULL) { session->restore (session->user_data); diff --git a/src/libserver/events.h b/src/libserver/events.h index 6728288eb..166cd7138 100644 --- a/src/libserver/events.h +++ b/src/libserver/events.h @@ -38,9 +38,9 @@ struct rspamd_async_session { * @param user_data abstract user data * @return */ -struct rspamd_async_session *new_async_session (rspamd_mempool_t *pool, - session_finalizer_t fin, event_finalizer_t restore, - event_finalizer_t cleanup, void *user_data); +struct rspamd_async_session * new_async_session (rspamd_mempool_t *pool, + session_finalizer_t fin, event_finalizer_t restore, + event_finalizer_t cleanup, void *user_data); /** * Insert new event to the session @@ -50,7 +50,7 @@ struct rspamd_async_session *new_async_session (rspamd_mempool_t *pool, * @param forced unused */ void register_async_event (struct rspamd_async_session *session, - event_finalizer_t fin, void *user_data, GQuark subsystem); + event_finalizer_t fin, void *user_data, GQuark subsystem); /** * Remove normal event @@ -58,11 +58,13 @@ void register_async_event (struct rspamd_async_session *session, * @param fin final callback * @param ud user data object */ -void remove_normal_event (struct rspamd_async_session *session, event_finalizer_t fin, void *ud); +void remove_normal_event (struct rspamd_async_session *session, + event_finalizer_t fin, + void *ud); /** * Must be called at the end of session, it calls fin functions for all non-forced callbacks - * @return true if the whole session was destroyed and false if there are forced events + * @return true if the whole session was destroyed and false if there are forced events */ gboolean destroy_session (struct rspamd_async_session *session); diff --git a/src/libserver/html.c b/src/libserver/html.c index 028c54f6c..0b7b584f5 100644 --- a/src/libserver/html.c +++ b/src/libserver/html.c @@ -23,15 +23,15 @@ */ #include "config.h" -#include "util.h" +#include "html.h" #include "main.h" #include "message.h" -#include "html.h" #include "url.h" +#include "util.h" -static sig_atomic_t tags_sorted = 0; +static sig_atomic_t tags_sorted = 0; -static struct html_tag tag_defs[] = { +static struct html_tag tag_defs[] = { /* W3C defined elements */ {Tag_A, "a", (CM_INLINE)}, {Tag_ABBR, "abbr", (CM_INLINE)}, @@ -93,7 +93,8 @@ static struct html_tag tag_defs[] = { {Tag_META, "meta", (CM_HEAD | CM_EMPTY)}, {Tag_NOFRAMES, "noframes", (CM_BLOCK | CM_FRAMES)}, {Tag_NOSCRIPT, "noscript", (CM_BLOCK | CM_INLINE | CM_MIXED)}, - {Tag_OBJECT, "object", (CM_OBJECT | CM_HEAD | CM_IMG | CM_INLINE | CM_PARAM)}, + {Tag_OBJECT, "object", + (CM_OBJECT | CM_HEAD | CM_IMG | CM_INLINE | CM_PARAM)}, {Tag_OL, "ol", (CM_BLOCK)}, {Tag_OPTGROUP, "optgroup", (CM_FIELD | CM_OPT)}, {Tag_OPTION, "option", (CM_FIELD | CM_OPT)}, @@ -156,21 +157,21 @@ static struct html_tag tag_defs[] = { {Tag_WBR, "wbr", (CM_INLINE | CM_EMPTY)}, }; -static sig_atomic_t entities_sorted = 0; +static sig_atomic_t entities_sorted = 0; struct _entity; -typedef struct _entity entity; +typedef struct _entity entity; struct _entity { - gchar *name; - uint code; - gchar *replacement; + gchar *name; + uint code; + gchar *replacement; }; -static entity entities_defs[] = { +static entity entities_defs[] = { /* - ** Markup pre-defined character entities - */ + ** Markup pre-defined character entities + */ {"quot", 34, "\""}, {"amp", 38, "&"}, {"apos", 39, "'"}, @@ -178,8 +179,8 @@ static entity entities_defs[] = { {"gt", 62, ">"}, /* - ** Latin-1 character entities - */ + ** Latin-1 character entities + */ {"nbsp", 160, " "}, {"iexcl", 161, "!"}, {"cent", 162, "cent"}, @@ -278,8 +279,8 @@ static entity entities_defs[] = { {"yuml", 255, "y"}, /* - ** Extended Entities defined in HTML 4: Symbols - */ + ** Extended Entities defined in HTML 4: Symbols + */ {"fnof", 402, "f"}, {"Alpha", 913, "alpha"}, {"Beta", 914, "beta"}, @@ -406,8 +407,8 @@ static entity entities_defs[] = { {"diams", 9830, NULL}, /* - ** Extended Entities defined in HTML 4: Special (less Markup at top) - */ + ** Extended Entities defined in HTML 4: Special (less Markup at top) + */ {"OElig", 338, NULL}, {"oelig", 339, NULL}, {"Scaron", 352, NULL}, @@ -443,8 +444,8 @@ static entity entities_defs_num[ (G_N_ELEMENTS (entities_defs)) ]; static gint tag_cmp (const void *m1, const void *m2) { - const struct html_tag *p1 = m1; - const struct html_tag *p2 = m2; + const struct html_tag *p1 = m1; + const struct html_tag *p2 = m2; return g_ascii_strcasecmp (p1->name, p2->name); } @@ -452,8 +453,8 @@ tag_cmp (const void *m1, const void *m2) static gint entity_cmp (const void *m1, const void *m2) { - const entity *p1 = m1; - const entity *p2 = m2; + const entity *p1 = m1; + const entity *p2 = m2; return g_ascii_strcasecmp (p1->name, p2->name); } @@ -461,19 +462,19 @@ entity_cmp (const void *m1, const void *m2) static gint entity_cmp_num (const void *m1, const void *m2) { - const entity *p1 = m1; - const entity *p2 = m2; + const entity *p1 = m1; + const entity *p2 = m2; return p1->code - p2->code; } -static GNode * +static GNode * construct_html_node (rspamd_mempool_t * pool, gchar *text, gsize tag_len) { - struct html_node *html; - GNode *n = NULL; - struct html_tag key, *found; - gchar t; + struct html_node *html; + GNode *n = NULL; + struct html_tag key, *found; + gchar t; if (text == NULL || *text == '\0') { return NULL; @@ -487,7 +488,8 @@ construct_html_node (rspamd_mempool_t * pool, gchar *text, gsize tag_len) } /* Check xml tag */ - if (*text == '?' && g_ascii_strncasecmp (text + 1, "xml", sizeof ("xml") - 1) == 0) { + if (*text == '?' && + g_ascii_strncasecmp (text + 1, "xml", sizeof ("xml") - 1) == 0) { html->flags |= FL_XML; html->tag = NULL; } @@ -499,13 +501,15 @@ construct_html_node (rspamd_mempool_t * pool, gchar *text, gsize tag_len) /* Find end of tag name */ key.name = text; - while (*text && g_ascii_isalnum (*(++text))); + while (*text && g_ascii_isalnum (*(++text))) ; t = *text; *text = '\0'; /* Match tag id by tag name */ - if ((found = bsearch (&key, tag_defs, G_N_ELEMENTS (tag_defs), sizeof (struct html_tag), tag_cmp)) != NULL) { + if ((found = + bsearch (&key, tag_defs, G_N_ELEMENTS (tag_defs), + sizeof (struct html_tag), tag_cmp)) != NULL) { *text = t; html->tag = found; } @@ -520,18 +524,20 @@ construct_html_node (rspamd_mempool_t * pool, gchar *text, gsize tag_len) return n; } -static gboolean +static gboolean check_balance (GNode * node, GNode ** cur_level) { - struct html_node *arg = node->data, *tmp; - GNode *cur; + struct html_node *arg = node->data, *tmp; + GNode *cur; if (arg->flags & FL_CLOSING) { /* First of all check whether this tag is closing tag for parent node */ cur = node->parent; while (cur && cur->data) { tmp = cur->data; - if ((tmp->tag && arg->tag) && tmp->tag->id == arg->tag->id && (tmp->flags & FL_CLOSED) == 0) { + if ((tmp->tag && + arg->tag) && tmp->tag->id == arg->tag->id && + (tmp->flags & FL_CLOSED) == 0) { tmp->flags |= FL_CLOSED; /* Destroy current node as we find corresponding parent node */ g_node_destroy (node); @@ -549,27 +555,28 @@ check_balance (GNode * node, GNode ** cur_level) return FALSE; } -struct html_tag * +struct html_tag * get_tag_by_name (const gchar *name) { - struct html_tag key; + struct html_tag key; key.name = name; - return bsearch (&key, tag_defs, G_N_ELEMENTS (tag_defs), sizeof (struct html_tag), tag_cmp); + return bsearch (&key, tag_defs, G_N_ELEMENTS (tag_defs), + sizeof (struct html_tag), tag_cmp); } /* Decode HTML entitles in text */ void decode_entitles (gchar *s, guint * len) { - guint l, rep_len; - gchar *t = s; /* t - tortoise */ - gchar *h = s; /* h - hare */ - gchar *e = s; - gchar *end_ptr; - gint state = 0, val, base; - entity *found, key; + guint l, rep_len; + gchar *t = s; /* t - tortoise */ + gchar *h = s; /* h - hare */ + gchar *e = s; + gchar *end_ptr; + gint state = 0, val, base; + entity *found, key; if (len == NULL || *len == 0) { l = strlen (s); @@ -580,7 +587,7 @@ decode_entitles (gchar *s, guint * len) while (h - s < (gint)l) { switch (state) { - /* Out of entitle */ + /* Out of entitle */ case 0: if (*h == '&') { state = 1; @@ -601,7 +608,10 @@ decode_entitles (gchar *s, guint * len) key.name = e + 1; *h = '\0'; - if (*(e + 1) != '#' && (found = bsearch (&key, entities_defs, G_N_ELEMENTS (entities_defs), sizeof (entity), entity_cmp)) != NULL) { + if (*(e + 1) != '#' && + (found = + bsearch (&key, entities_defs, G_N_ELEMENTS (entities_defs), + sizeof (entity), entity_cmp)) != NULL) { if (found->replacement) { rep_len = strlen (found->replacement); memcpy (t, found->replacement, rep_len); @@ -631,7 +641,10 @@ decode_entitles (gchar *s, guint * len) else { /* Search for a replacement */ key.code = val; - found = bsearch (&key, entities_defs_num, G_N_ELEMENTS (entities_defs), sizeof (entity), entity_cmp_num); + found = + bsearch (&key, entities_defs_num, G_N_ELEMENTS ( + entities_defs), sizeof (entity), + entity_cmp_num); if (found) { if (found->replacement) { rep_len = strlen (found->replacement); @@ -656,15 +669,19 @@ decode_entitles (gchar *s, guint * len) } static void -check_phishing (struct rspamd_task *task, struct uri *href_url, const gchar *url_text, gsize remain, tag_id_t id) +check_phishing (struct rspamd_task *task, + struct uri *href_url, + const gchar *url_text, + gsize remain, + tag_id_t id) { - struct uri *new; - gchar *url_str; - const gchar *p, *c; - gchar tagbuf[128]; - struct html_tag *tag; - gsize len = 0; - gint rc; + struct uri *new; + gchar *url_str; + const gchar *p, *c; + gchar tagbuf[128]; + struct html_tag *tag; + gsize len = 0; + gint rc; p = url_text; while (len < remain) { @@ -678,8 +695,8 @@ check_phishing (struct rspamd_task *task, struct uri *href_url, const gchar *url } while (len < remain) { if (!g_ascii_isspace (*p) && *p != '>') { - p ++; - len ++; + p++; + len++; } else { break; @@ -693,11 +710,11 @@ check_phishing (struct rspamd_task *task, struct uri *href_url, const gchar *url else if (tag->id == Tag_IMG) { /* We should ignore IMG tag here */ while (len < remain && *p != '>' && *p != '<') { - p ++; - len ++; + p++; + len++; } if (*p == '>' && len < remain) { - p ++; + p++; } remain -= p - url_text; @@ -707,19 +724,21 @@ check_phishing (struct rspamd_task *task, struct uri *href_url, const gchar *url } } } - len ++; - p ++; + len++; + p++; } - if (url_try_text (task->task_pool, url_text, len, NULL, NULL, &url_str, TRUE) && url_str != NULL) { + if (url_try_text (task->task_pool, url_text, len, NULL, NULL, &url_str, + TRUE) && url_str != NULL) { new = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct uri)); if (new != NULL) { g_strstrip (url_str); rc = parse_uri (new, url_str, task->task_pool); - if (rc == URI_ERRNO_OK || rc == URI_ERRNO_NO_SLASHES || rc == URI_ERRNO_NO_HOST_SLASH) { + if (rc == URI_ERRNO_OK || rc == URI_ERRNO_NO_SLASHES || rc == + URI_ERRNO_NO_HOST_SLASH) { if (g_ascii_strncasecmp (href_url->host, new->host, - MAX (href_url->hostlen, new->hostlen)) != 0) { + MAX (href_url->hostlen, new->hostlen)) != 0) { /* Special check for urls beginning with 'www' */ if (new->hostlen > 4 && href_url->hostlen > 4) { p = new->host; @@ -762,7 +781,9 @@ check_phishing (struct rspamd_task *task, struct uri *href_url, const gchar *url } } else { - msg_info ("extract of url '%s' failed: %s", url_str, url_strerror (rc)); + msg_info ("extract of url '%s' failed: %s", + url_str, + url_strerror (rc)); } } } @@ -770,13 +791,17 @@ check_phishing (struct rspamd_task *task, struct uri *href_url, const gchar *url } static void -parse_tag_url (struct rspamd_task *task, struct mime_text_part *part, tag_id_t id, - gchar *tag_text, gsize tag_len, gsize remain) +parse_tag_url (struct rspamd_task *task, + struct mime_text_part *part, + tag_id_t id, + gchar *tag_text, + gsize tag_len, + gsize remain) { - gchar *c = NULL, *p, *url_text; - gint len, rc; - struct uri *url; - gboolean got_single_quote = FALSE, got_double_quote = FALSE; + gchar *c = NULL, *p, *url_text; + gint len, rc; + struct uri *url; + gboolean got_single_quote = FALSE, got_double_quote = FALSE; /* For A tags search for href= and for IMG tags search for src= */ if (id == Tag_A) { @@ -814,7 +839,8 @@ parse_tag_url (struct rspamd_task *task, struct mime_text_part *part, tag_id_t i len++; } } - else if (g_ascii_isspace (*p) || *p == '>' || (*p == '/' && *(p + 1) == '>') || *p == '\r' || *p == '\n') { + else if (g_ascii_isspace (*p) || *p == '>' || + (*p == '/' && *(p + 1) == '>') || *p == '\r' || *p == '\n') { break; } else { @@ -843,22 +869,27 @@ parse_tag_url (struct rspamd_task *task, struct mime_text_part *part, tag_id_t i rspamd_strlcpy (url_text, c, len + 1); decode_entitles (url_text, NULL); - if (g_ascii_strncasecmp (url_text, "http://", sizeof ("http://") - 1) != 0 && - g_ascii_strncasecmp (url_text, "www", sizeof ("www") - 1) != 0 && - g_ascii_strncasecmp (url_text, "ftp://", sizeof ("ftp://") - 1) != 0 && - g_ascii_strncasecmp (url_text, "mailto:", sizeof ("mailto:") - 1) != 0) { + if (g_ascii_strncasecmp (url_text, "http://", + sizeof ("http://") - 1) != 0 && + g_ascii_strncasecmp (url_text, "www", + sizeof ("www") - 1) != 0 && + g_ascii_strncasecmp (url_text, "ftp://", + sizeof ("ftp://") - 1) != 0 && + g_ascii_strncasecmp (url_text, "mailto:", + sizeof ("mailto:") - 1) != 0) { return; } url = rspamd_mempool_alloc (task->task_pool, sizeof (struct uri)); rc = parse_uri (url, url_text, task->task_pool); - if (rc != URI_ERRNO_EMPTY && rc != URI_ERRNO_NO_HOST && url->hostlen != 0) { + if (rc != URI_ERRNO_EMPTY && rc != URI_ERRNO_NO_HOST && url->hostlen != + 0) { /* * Check for phishing */ if ((p = strchr (c, '>')) != NULL && id == Tag_A) { - p ++; + p++; check_phishing (task, url, p, remain - (p - tag_text), id); } if (g_tree_lookup (task->urls, url) == NULL) { @@ -869,20 +900,28 @@ parse_tag_url (struct rspamd_task *task, struct mime_text_part *part, tag_id_t i } gboolean -add_html_node (struct rspamd_task *task, rspamd_mempool_t * pool, struct mime_text_part *part, - gchar *tag_text, gsize tag_len, gsize remain, GNode ** cur_level) +add_html_node (struct rspamd_task *task, + rspamd_mempool_t * pool, + struct mime_text_part *part, + gchar *tag_text, + gsize tag_len, + gsize remain, + GNode ** cur_level) { - GNode *new; - struct html_node *data; + GNode *new; + struct html_node *data; if (!tags_sorted) { - qsort (tag_defs, G_N_ELEMENTS (tag_defs), sizeof (struct html_tag), tag_cmp); + qsort (tag_defs, G_N_ELEMENTS ( + tag_defs), sizeof (struct html_tag), tag_cmp); tags_sorted = 1; } if (!entities_sorted) { - qsort (entities_defs, G_N_ELEMENTS (entities_defs), sizeof (entity), entity_cmp); + qsort (entities_defs, G_N_ELEMENTS ( + entities_defs), sizeof (entity), entity_cmp); memcpy (entities_defs_num, entities_defs, sizeof (entities_defs)); - qsort (entities_defs_num, G_N_ELEMENTS (entities_defs), sizeof (entity), entity_cmp_num); + qsort (entities_defs_num, G_N_ELEMENTS ( + entities_defs), sizeof (entity), entity_cmp_num); entities_sorted = 1; } @@ -892,19 +931,32 @@ add_html_node (struct rspamd_task *task, rspamd_mempool_t * pool, struct mime_te new = g_node_new (NULL); *cur_level = new; part->html_nodes = new; - rspamd_mempool_add_destructor (pool, (rspamd_mempool_destruct_t) g_node_destroy, part->html_nodes); + rspamd_mempool_add_destructor (pool, + (rspamd_mempool_destruct_t) g_node_destroy, + part->html_nodes); /* Call once again with root node */ - return add_html_node (task, pool, part, tag_text, tag_len, remain, cur_level); + return add_html_node (task, + pool, + part, + tag_text, + tag_len, + remain, + cur_level); } else { new = construct_html_node (pool, tag_text, tag_len); if (new == NULL) { - debug_task ("cannot construct HTML node for text '%*s'", tag_len, tag_text); + debug_task ("cannot construct HTML node for text '%*s'", + tag_len, + tag_text); return FALSE; } data = new->data; - if (data->tag && (data->tag->id == Tag_A || data->tag->id == Tag_IMG) && ((data->flags & FL_CLOSING) == 0)) { - parse_tag_url (task, part, data->tag->id, tag_text, tag_len, remain); + if (data->tag && + (data->tag->id == Tag_A || + data->tag->id == Tag_IMG) && ((data->flags & FL_CLOSING) == 0)) { + parse_tag_url (task, part, data->tag->id, tag_text, tag_len, + remain); } if (data->flags & FL_CLOSING) { @@ -914,7 +966,8 @@ add_html_node (struct rspamd_task *task, rspamd_mempool_t * pool, struct mime_te } g_node_append (*cur_level, new); if (!check_balance (new, cur_level)) { - debug_task ("mark part as unbalanced as it has not pairable closing tags"); + debug_task ( + "mark part as unbalanced as it has not pairable closing tags"); part->is_balanced = FALSE; } } @@ -926,9 +979,9 @@ add_html_node (struct rspamd_task *task, rspamd_mempool_t * pool, struct mime_te } /* Skip some tags */ if (data->tag && (data->tag->id == Tag_STYLE || - data->tag->id == Tag_SCRIPT || - data->tag->id == Tag_OBJECT || - data->tag->id == Tag_TITLE)) { + data->tag->id == Tag_SCRIPT || + data->tag->id == Tag_OBJECT || + data->tag->id == Tag_TITLE)) { return FALSE; } } diff --git a/src/libserver/html.h b/src/libserver/html.h index 3ea758e60..6709ed151 100644 --- a/src/libserver/html.h +++ b/src/libserver/html.h @@ -11,129 +11,129 @@ /* Known HTML tags */ typedef enum { - Tag_UNKNOWN, /**< Unknown tag! */ - Tag_A, /**< A */ - Tag_ABBR, /**< ABBR */ - Tag_ACRONYM, /**< ACRONYM */ - Tag_ADDRESS, /**< ADDRESS */ - Tag_ALIGN, /**< ALIGN */ - Tag_APPLET, /**< APPLET */ - Tag_AREA, /**< AREA */ - Tag_B, /**< B */ - Tag_BASE, /**< BASE */ - Tag_BASEFONT, /**< BASEFONT */ - Tag_BDO, /**< BDO */ - Tag_BGSOUND, /**< BGSOUND */ - Tag_BIG, /**< BIG */ - Tag_BLINK, /**< BLINK */ - Tag_BLOCKQUOTE, /**< BLOCKQUOTE */ - Tag_BODY, /**< BODY */ - Tag_BR, /**< BR */ - Tag_BUTTON, /**< BUTTON */ - Tag_CAPTION, /**< CAPTION */ - Tag_CENTER, /**< CENTER */ - Tag_CITE, /**< CITE */ - Tag_CODE, /**< CODE */ - Tag_COL, /**< COL */ - Tag_COLGROUP, /**< COLGROUP */ - Tag_COMMENT, /**< COMMENT */ - Tag_DD, /**< DD */ - Tag_DEL, /**< DEL */ - Tag_DFN, /**< DFN */ - Tag_DIR, /**< DIR */ - Tag_DIV, /**< DIF */ - Tag_DL, /**< DL */ - Tag_DT, /**< DT */ - Tag_EM, /**< EM */ - Tag_EMBED, /**< EMBED */ - Tag_FIELDSET, /**< FIELDSET */ - Tag_FONT, /**< FONT */ - Tag_FORM, /**< FORM */ - Tag_FRAME, /**< FRAME */ - Tag_FRAMESET, /**< FRAMESET */ - Tag_H1, /**< H1 */ - Tag_H2, /**< H2 */ - Tag_H3, /**< H3 */ - Tag_H4, /**< H4 */ - Tag_H5, /**< H5 */ - Tag_H6, /**< H6 */ - Tag_HEAD, /**< HEAD */ - Tag_HR, /**< HR */ - Tag_HTML, /**< HTML */ - Tag_I, /**< I */ - Tag_IFRAME, /**< IFRAME */ - Tag_ILAYER, /**< ILAYER */ - Tag_IMG, /**< IMG */ - Tag_INPUT, /**< INPUT */ - Tag_INS, /**< INS */ - Tag_ISINDEX, /**< ISINDEX */ - Tag_KBD, /**< KBD */ - Tag_KEYGEN, /**< KEYGEN */ - Tag_LABEL, /**< LABEL */ - Tag_LAYER, /**< LAYER */ - Tag_LEGEND, /**< LEGEND */ - Tag_LI, /**< LI */ - Tag_LINK, /**< LINK */ - Tag_LISTING, /**< LISTING */ - Tag_MAP, /**< MAP */ - Tag_MARQUEE, /**< MARQUEE */ - Tag_MENU, /**< MENU */ - Tag_META, /**< META */ - Tag_MULTICOL, /**< MULTICOL */ - Tag_NOBR, /**< NOBR */ - Tag_NOEMBED, /**< NOEMBED */ - Tag_NOFRAMES, /**< NOFRAMES */ - Tag_NOLAYER, /**< NOLAYER */ - Tag_NOSAVE, /**< NOSAVE */ - Tag_NOSCRIPT, /**< NOSCRIPT */ - Tag_OBJECT, /**< OBJECT */ - Tag_OL, /**< OL */ - Tag_OPTGROUP, /**< OPTGROUP */ - Tag_OPTION, /**< OPTION */ - Tag_P, /**< P */ - Tag_PARAM, /**< PARAM */ - Tag_PLAINTEXT,/**< PLAINTEXT */ - Tag_PRE, /**< PRE */ - Tag_Q, /**< Q */ - Tag_RB, /**< RB */ - Tag_RBC, /**< RBC */ - Tag_RP, /**< RP */ - Tag_RT, /**< RT */ - Tag_RTC, /**< RTC */ - Tag_RUBY, /**< RUBY */ - Tag_S, /**< S */ - Tag_SAMP, /**< SAMP */ - Tag_SCRIPT, /**< SCRIPT */ - Tag_SELECT, /**< SELECT */ - Tag_SERVER, /**< SERVER */ - Tag_SERVLET, /**< SERVLET */ - Tag_SMALL, /**< SMALL */ - Tag_SPACER, /**< SPACER */ - Tag_SPAN, /**< SPAN */ - Tag_STRIKE, /**< STRIKE */ - Tag_STRONG, /**< STRONG */ - Tag_STYLE, /**< STYLE */ - Tag_SUB, /**< SUB */ - Tag_SUP, /**< SUP */ - Tag_TABLE, /**< TABLE */ - Tag_TBODY, /**< TBODY */ - Tag_TD, /**< TD */ - Tag_TEXTAREA, /**< TEXTAREA */ - Tag_TFOOT, /**< TFOOT */ - Tag_TH, /**< TH */ - Tag_THEAD, /**< THEAD */ - Tag_TITLE, /**< TITLE */ - Tag_TR, /**< TR */ - Tag_TT, /**< TT */ - Tag_U, /**< U */ - Tag_UL, /**< UL */ - Tag_VAR, /**< VAR */ - Tag_WBR, /**< WBR */ - Tag_XMP, /**< XMP */ - Tag_XML, /**< XML */ - Tag_NEXTID, /**< NEXTID */ + Tag_UNKNOWN, /**< Unknown tag! */ + Tag_A, /**< A */ + Tag_ABBR, /**< ABBR */ + Tag_ACRONYM, /**< ACRONYM */ + Tag_ADDRESS, /**< ADDRESS */ + Tag_ALIGN, /**< ALIGN */ + Tag_APPLET, /**< APPLET */ + Tag_AREA, /**< AREA */ + Tag_B, /**< B */ + Tag_BASE, /**< BASE */ + Tag_BASEFONT, /**< BASEFONT */ + Tag_BDO, /**< BDO */ + Tag_BGSOUND, /**< BGSOUND */ + Tag_BIG, /**< BIG */ + Tag_BLINK, /**< BLINK */ + Tag_BLOCKQUOTE, /**< BLOCKQUOTE */ + Tag_BODY, /**< BODY */ + Tag_BR, /**< BR */ + Tag_BUTTON, /**< BUTTON */ + Tag_CAPTION, /**< CAPTION */ + Tag_CENTER, /**< CENTER */ + Tag_CITE, /**< CITE */ + Tag_CODE, /**< CODE */ + Tag_COL, /**< COL */ + Tag_COLGROUP, /**< COLGROUP */ + Tag_COMMENT, /**< COMMENT */ + Tag_DD, /**< DD */ + Tag_DEL, /**< DEL */ + Tag_DFN, /**< DFN */ + Tag_DIR, /**< DIR */ + Tag_DIV, /**< DIF */ + Tag_DL, /**< DL */ + Tag_DT, /**< DT */ + Tag_EM, /**< EM */ + Tag_EMBED, /**< EMBED */ + Tag_FIELDSET, /**< FIELDSET */ + Tag_FONT, /**< FONT */ + Tag_FORM, /**< FORM */ + Tag_FRAME, /**< FRAME */ + Tag_FRAMESET, /**< FRAMESET */ + Tag_H1, /**< H1 */ + Tag_H2, /**< H2 */ + Tag_H3, /**< H3 */ + Tag_H4, /**< H4 */ + Tag_H5, /**< H5 */ + Tag_H6, /**< H6 */ + Tag_HEAD, /**< HEAD */ + Tag_HR, /**< HR */ + Tag_HTML, /**< HTML */ + Tag_I, /**< I */ + Tag_IFRAME, /**< IFRAME */ + Tag_ILAYER, /**< ILAYER */ + Tag_IMG, /**< IMG */ + Tag_INPUT, /**< INPUT */ + Tag_INS, /**< INS */ + Tag_ISINDEX, /**< ISINDEX */ + Tag_KBD, /**< KBD */ + Tag_KEYGEN, /**< KEYGEN */ + Tag_LABEL, /**< LABEL */ + Tag_LAYER, /**< LAYER */ + Tag_LEGEND, /**< LEGEND */ + Tag_LI, /**< LI */ + Tag_LINK, /**< LINK */ + Tag_LISTING, /**< LISTING */ + Tag_MAP, /**< MAP */ + Tag_MARQUEE, /**< MARQUEE */ + Tag_MENU, /**< MENU */ + Tag_META, /**< META */ + Tag_MULTICOL, /**< MULTICOL */ + Tag_NOBR, /**< NOBR */ + Tag_NOEMBED, /**< NOEMBED */ + Tag_NOFRAMES, /**< NOFRAMES */ + Tag_NOLAYER, /**< NOLAYER */ + Tag_NOSAVE, /**< NOSAVE */ + Tag_NOSCRIPT, /**< NOSCRIPT */ + Tag_OBJECT, /**< OBJECT */ + Tag_OL, /**< OL */ + Tag_OPTGROUP, /**< OPTGROUP */ + Tag_OPTION, /**< OPTION */ + Tag_P, /**< P */ + Tag_PARAM, /**< PARAM */ + Tag_PLAINTEXT, /**< PLAINTEXT */ + Tag_PRE, /**< PRE */ + Tag_Q, /**< Q */ + Tag_RB, /**< RB */ + Tag_RBC, /**< RBC */ + Tag_RP, /**< RP */ + Tag_RT, /**< RT */ + Tag_RTC, /**< RTC */ + Tag_RUBY, /**< RUBY */ + Tag_S, /**< S */ + Tag_SAMP, /**< SAMP */ + Tag_SCRIPT, /**< SCRIPT */ + Tag_SELECT, /**< SELECT */ + Tag_SERVER, /**< SERVER */ + Tag_SERVLET, /**< SERVLET */ + Tag_SMALL, /**< SMALL */ + Tag_SPACER, /**< SPACER */ + Tag_SPAN, /**< SPAN */ + Tag_STRIKE, /**< STRIKE */ + Tag_STRONG, /**< STRONG */ + Tag_STYLE, /**< STYLE */ + Tag_SUB, /**< SUB */ + Tag_SUP, /**< SUP */ + Tag_TABLE, /**< TABLE */ + Tag_TBODY, /**< TBODY */ + Tag_TD, /**< TD */ + Tag_TEXTAREA, /**< TEXTAREA */ + Tag_TFOOT, /**< TFOOT */ + Tag_TH, /**< TH */ + Tag_THEAD, /**< THEAD */ + Tag_TITLE, /**< TITLE */ + Tag_TR, /**< TR */ + Tag_TT, /**< TT */ + Tag_U, /**< U */ + Tag_UL, /**< UL */ + Tag_VAR, /**< VAR */ + Tag_WBR, /**< WBR */ + Tag_XMP, /**< XMP */ + Tag_XML, /**< XML */ + Tag_NEXTID, /**< NEXTID */ - N_TAGS /**< Must be last */ + N_TAGS /**< Must be last */ } tag_id_t; #define CM_UNKNOWN 0 @@ -175,7 +175,7 @@ typedef enum #define CM_IMG (1 << 16) /* Elements with inline and block model. Used to avoid calling InlineDup. */ #define CM_MIXED (1 << 17) -/* Elements whose content needs to be indented only if containing one +/* Elements whose content needs to be indented only if containing one CM_BLOCK element. */ #define CM_NO_INDENT (1 << 18) /* Elements that are obsolete (such as "dir", "menu"). */ @@ -187,11 +187,11 @@ typedef enum #define CM_OMITST (1 << 21) /* XML tag */ -#define FL_XML (1 << 0) +#define FL_XML (1 << 0) /* Closing tag */ -#define FL_CLOSING (1 << 1) +#define FL_CLOSING (1 << 1) /* Fully closed tag (e.g. <a attrs />) */ -#define FL_CLOSED (1 << 2) +#define FL_CLOSED (1 << 2) struct html_tag { tag_id_t id; @@ -210,8 +210,13 @@ struct rspamd_task; /* * Add a single node to the tags tree */ -gboolean add_html_node (struct rspamd_task *task, rspamd_mempool_t *pool, - struct mime_text_part *part, gchar *tag_text, gsize tag_len, gsize remain, GNode **cur_level); +gboolean add_html_node (struct rspamd_task *task, + rspamd_mempool_t *pool, + struct mime_text_part *part, + gchar *tag_text, + gsize tag_len, + gsize remain, + GNode **cur_level); /* * Get tag structure by its name (binary search is used) diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c index fc4096f8c..30a155229 100644 --- a/src/libserver/protocol.c +++ b/src/libserver/protocol.c @@ -22,11 +22,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include "cfg_file.h" #include "config.h" #include "main.h" -#include "util.h" -#include "cfg_file.h" #include "message.h" +#include "util.h" /* Max line size */ #define OUTBUFSIZ BUFSIZ @@ -35,7 +35,7 @@ * described below */ #define MSG_CMD_CHECK "check" -/* +/* * Check if message is spam or not, and return score plus list * of symbols hit */ @@ -95,7 +95,7 @@ #define DELIVER_TO_HEADER "Deliver-To" #define NO_LOG_HEADER "Log" -static GList *custom_commands = NULL; +static GList *custom_commands = NULL; /* @@ -104,20 +104,22 @@ static GList *custom_commands = NULL; static gchar * rspamd_protocol_escape_braces (GString *in) { - gint len = 0; - gchar *orig, *p; + gint len = 0; + gchar *orig, *p; orig = in->str; - while ((g_ascii_isspace (*orig) || *orig == '<') && orig - in->str < (gint)in->len) { - orig ++; + while ((g_ascii_isspace (*orig) || *orig == + '<') && orig - in->str < (gint)in->len) { + orig++; } g_string_erase (in, 0, orig - in->str); p = in->str; - while ((!g_ascii_isspace (*p) && *p != '>') && p - in->str < (gint)in->len) { - p ++; - len ++; + while ((!g_ascii_isspace (*p) && *p != + '>') && p - in->str < (gint)in->len) { + p++; + len++; } g_string_truncate (in, len); @@ -126,10 +128,11 @@ rspamd_protocol_escape_braces (GString *in) } static gboolean -rspamd_protocol_handle_url (struct rspamd_task *task, struct rspamd_http_message *msg) +rspamd_protocol_handle_url (struct rspamd_task *task, + struct rspamd_http_message *msg) { - GList *cur; - struct custom_command *cmd; + GList *cur; + struct custom_command *cmd; const gchar *p; if (msg->url == NULL || msg->url->len == 0) { @@ -224,13 +227,14 @@ err: gboolean rspamd_protocol_handle_headers (struct rspamd_task *task, - struct rspamd_http_message *msg) + struct rspamd_http_message *msg) { - gchar *headern, *err, *tmp; - gboolean res = TRUE; - struct rspamd_http_header *h; + gchar *headern, *err, *tmp; + gboolean res = TRUE; + struct rspamd_http_header *h; - LL_FOREACH (msg->headers, h) { + LL_FOREACH (msg->headers, h) + { headern = h->name->str; switch (headern[0]) { @@ -238,7 +242,8 @@ rspamd_protocol_handle_headers (struct rspamd_task *task, case 'D': if (g_ascii_strcasecmp (headern, DELIVER_TO_HEADER) == 0) { task->deliver_to = rspamd_protocol_escape_braces (h->value); - debug_task ("read deliver-to header, value: %s", task->deliver_to); + debug_task ("read deliver-to header, value: %s", + task->deliver_to); } else { debug_task ("wrong header: %s", headern); @@ -327,7 +332,7 @@ rspamd_protocol_handle_headers (struct rspamd_task *task, case 'P': if (g_ascii_strcasecmp (headern, PASS_HEADER) == 0) { if (h->value->len == sizeof ("all") - 1 && - g_ascii_strcasecmp (h->value->str, "all") == 0) { + g_ascii_strcasecmp (h->value->str, "all") == 0) { task->pass_all_filters = TRUE; debug_task ("pass all filters"); } @@ -373,7 +378,8 @@ rspamd_protocol_handle_headers (struct rspamd_task *task, } if (!res && task->cfg->strict_protocol_headers) { - msg_err ("deny processing of a request with incorrect or unknown headers"); + msg_err ( + "deny processing of a request with incorrect or unknown headers"); task->last_error = "invalid header"; task->error_code = 400; return FALSE; @@ -384,7 +390,7 @@ rspamd_protocol_handle_headers (struct rspamd_task *task, gboolean rspamd_protocol_handle_request (struct rspamd_task *task, - struct rspamd_http_message *msg) + struct rspamd_http_message *msg) { gboolean ret = TRUE; @@ -407,19 +413,22 @@ rspamd_protocol_handle_request (struct rspamd_task *task, static void write_hashes_to_log (struct rspamd_task *task, GString *logbuf) { - GList *cur; - struct mime_text_part *text_part; - + GList *cur; + struct mime_text_part *text_part; + cur = task->text_parts; while (cur) { text_part = cur->data; if (text_part->fuzzy) { if (cur->next != NULL) { - rspamd_printf_gstring (logbuf, " part: %Xd,", text_part->fuzzy->h); + rspamd_printf_gstring (logbuf, + " part: %Xd,", + text_part->fuzzy->h); } else { - rspamd_printf_gstring (logbuf, " part: %Xd", text_part->fuzzy->h); + rspamd_printf_gstring (logbuf, " part: %Xd", + text_part->fuzzy->h); } } cur = g_list_next (cur); @@ -439,18 +448,20 @@ struct tree_cb_data { static gboolean urls_protocol_cb (gpointer key, gpointer value, gpointer ud) { - struct tree_cb_data *cb = ud; - struct uri *url = value; - ucl_object_t *obj; + struct tree_cb_data *cb = ud; + struct uri *url = value; + ucl_object_t *obj; obj = ucl_object_fromlstring (url->host, url->hostlen); DL_APPEND (cb->top->value.av, obj); if (cb->task->cfg->log_urls) { - msg_info ("<%s> URL: %s - %s: %s", cb->task->message_id, cb->task->user ? - cb->task->user : (cb->task->from ? cb->task->from : "unknown"), - rspamd_inet_address_to_string (&cb->task->from_addr), - struri (url)); + msg_info ("<%s> URL: %s - %s: %s", + cb->task->message_id, + cb->task->user ? + cb->task->user : (cb->task->from ? cb->task->from : "unknown"), + rspamd_inet_address_to_string (&cb->task->from_addr), + struri (url)); } return FALSE; @@ -459,8 +470,8 @@ urls_protocol_cb (gpointer key, gpointer value, gpointer ud) static ucl_object_t * rspamd_urls_tree_ucl (GTree *input, struct rspamd_task *task) { - struct tree_cb_data cb; - ucl_object_t *obj; + struct tree_cb_data cb; + ucl_object_t *obj; obj = ucl_object_typed_new (UCL_ARRAY); cb.top = obj; @@ -474,9 +485,9 @@ rspamd_urls_tree_ucl (GTree *input, struct rspamd_task *task) static gboolean emails_protocol_cb (gpointer key, gpointer value, gpointer ud) { - struct tree_cb_data *cb = ud; - struct uri *url = value; - ucl_object_t *obj; + struct tree_cb_data *cb = ud; + struct uri *url = value; + ucl_object_t *obj; obj = ucl_object_fromlstring (url->user, url->userlen + url->hostlen + 1); DL_APPEND (cb->top->value.av, obj); @@ -487,8 +498,8 @@ emails_protocol_cb (gpointer key, gpointer value, gpointer ud) static ucl_object_t * rspamd_emails_tree_ucl (GTree *input, struct rspamd_task *task) { - struct tree_cb_data cb; - ucl_object_t *obj; + struct tree_cb_data cb; + ucl_object_t *obj; obj = ucl_object_typed_new (UCL_ARRAY); cb.top = obj; @@ -504,9 +515,9 @@ rspamd_emails_tree_ucl (GTree *input, struct rspamd_task *task) static const gchar * make_rewritten_subject (struct metric *metric, struct rspamd_task *task) { - static gchar subj_buf[1024]; - gchar *p = subj_buf, *end, *c, *res; - const gchar *s; + static gchar subj_buf[1024]; + gchar *p = subj_buf, *end, *c, *res; + const gchar *s; end = p + sizeof(subj_buf); c = metric->subject; @@ -522,13 +533,15 @@ make_rewritten_subject (struct metric *metric, struct rspamd_task *task) c += 2; } else { - *p = *c ++; + *p = *c++; } - p ++; + p++; } res = g_mime_utils_header_encode_text (subj_buf); - rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)g_free, res); + rspamd_mempool_add_destructor (task->task_pool, + (rspamd_mempool_destruct_t)g_free, + res); return res; } @@ -536,8 +549,8 @@ make_rewritten_subject (struct metric *metric, struct rspamd_task *task) static ucl_object_t * rspamd_str_list_ucl (GList *str_list) { - ucl_object_t *top = NULL, *obj; - GList *cur; + ucl_object_t *top = NULL, *obj; + GList *cur; top = ucl_object_typed_new (UCL_ARRAY); cur = str_list; @@ -552,40 +565,46 @@ rspamd_str_list_ucl (GList *str_list) static ucl_object_t * rspamd_metric_symbol_ucl (struct rspamd_task *task, struct metric *m, - struct symbol *sym, GString *logbuf) + struct symbol *sym, GString *logbuf) { - ucl_object_t *obj = NULL; - const gchar *description = NULL; + ucl_object_t *obj = NULL; + const gchar *description = NULL; rspamd_printf_gstring (logbuf, "%s,", sym->name); description = g_hash_table_lookup (m->descriptions, sym->name); obj = ucl_object_typed_new (UCL_OBJECT); - ucl_object_insert_key (obj, ucl_object_fromstring (sym->name), "name", 0, false); - ucl_object_insert_key (obj, ucl_object_fromdouble (sym->score), "score", 0, false); + ucl_object_insert_key (obj, ucl_object_fromstring ( + sym->name), "name", 0, false); + ucl_object_insert_key (obj, ucl_object_fromdouble ( + sym->score), "score", 0, false); if (description) { - ucl_object_insert_key (obj, ucl_object_fromstring (description), "description", 0, false); + ucl_object_insert_key (obj, ucl_object_fromstring ( + description), "description", 0, false); } if (sym->options != NULL) { - ucl_object_insert_key (obj, rspamd_str_list_ucl (sym->options), "options", 0, false); + ucl_object_insert_key (obj, rspamd_str_list_ucl ( + sym->options), "options", 0, false); } return obj; } static ucl_object_t * -rspamd_metric_result_ucl (struct rspamd_task *task, struct metric_result *mres, GString *logbuf) +rspamd_metric_result_ucl (struct rspamd_task *task, + struct metric_result *mres, + GString *logbuf) { - GHashTableIter hiter; - struct symbol *sym; - struct metric *m; - gboolean is_spam; - enum rspamd_metric_action action = METRIC_ACTION_NOACTION; - ucl_object_t *obj = NULL, *sobj; - gdouble required_score; - gpointer h, v; - const gchar *subject; - gchar action_char; + GHashTableIter hiter; + struct symbol *sym; + struct metric *m; + gboolean is_spam; + enum rspamd_metric_action action = METRIC_ACTION_NOACTION; + ucl_object_t *obj = NULL, *sobj; + gdouble required_score; + gpointer h, v; + const gchar *subject; + gchar action_char; m = mres->metric; @@ -603,26 +622,27 @@ rspamd_metric_result_ucl (struct rspamd_task *task, struct metric_result *mres, action_char = 'F'; } rspamd_printf_gstring (logbuf, "(%s: %c (%s): [%.2f/%.2f] [", - m->name, action_char, - str_action_metric (action), - mres->score, required_score); + m->name, action_char, + str_action_metric (action), + mres->score, required_score); obj = ucl_object_typed_new (UCL_OBJECT); - ucl_object_insert_key (obj, ucl_object_frombool (is_spam), - "is_spam", 0, false); - ucl_object_insert_key (obj, ucl_object_frombool (task->is_skipped), - "is_skipped", 0, false); + ucl_object_insert_key (obj, ucl_object_frombool (is_spam), + "is_spam", 0, false); + ucl_object_insert_key (obj, ucl_object_frombool (task->is_skipped), + "is_skipped", 0, false); ucl_object_insert_key (obj, ucl_object_fromdouble (mres->score), - "score", 0, false); + "score", 0, false); ucl_object_insert_key (obj, ucl_object_fromdouble (required_score), - "required_score", 0, false); - ucl_object_insert_key (obj, ucl_object_fromstring (str_action_metric (action)), - "action", 0, false); + "required_score", 0, false); + ucl_object_insert_key (obj, + ucl_object_fromstring (str_action_metric (action)), + "action", 0, false); if (action == METRIC_ACTION_REWRITE_SUBJECT) { subject = make_rewritten_subject (m, task); ucl_object_insert_key (obj, ucl_object_fromstring (subject), - "subject", 0, false); + "subject", 0, false); } /* Now handle symbols */ g_hash_table_iter_init (&hiter, mres->symbols); @@ -634,28 +654,31 @@ rspamd_metric_result_ucl (struct rspamd_task *task, struct metric_result *mres, /* Cut the trailing comma if needed */ if (logbuf->str[logbuf->len - 1] == ',') { - logbuf->len --; + logbuf->len--; } #ifdef HAVE_CLOCK_GETTIME rspamd_printf_gstring (logbuf, "]), len: %z, time: %s, dns req: %d,", - task->msg->len, calculate_check_time (&task->tv, &task->ts, - task->cfg->clock_res, &task->scan_milliseconds), task->dns_requests); + task->msg->len, calculate_check_time (&task->tv, &task->ts, + task->cfg->clock_res, &task->scan_milliseconds), task->dns_requests); #else rspamd_printf_gstring (logbuf, "]), len: %z, time: %s, dns req: %d,", - task->msg->len, - calculate_check_time (&task->tv, task->cfg->clock_res, &task->scan_milliseconds), - task->dns_requests); + task->msg->len, + calculate_check_time (&task->tv, task->cfg->clock_res, + &task->scan_milliseconds), + task->dns_requests); #endif return obj; } static void -rspamd_ucl_tolegacy_output (struct rspamd_task *task, ucl_object_t *top, GString *out) +rspamd_ucl_tolegacy_output (struct rspamd_task *task, + ucl_object_t *top, + GString *out) { const ucl_object_t *metric, *score, - *required_score, *is_spam, *elt; + *required_score, *is_spam, *elt; ucl_object_iter_t iter = NULL; metric = ucl_object_find_key (top, DEFAULT_METRIC); @@ -663,10 +686,11 @@ rspamd_ucl_tolegacy_output (struct rspamd_task *task, ucl_object_t *top, GString score = ucl_object_find_key (metric, "score"); required_score = ucl_object_find_key (metric, "required_score"); is_spam = ucl_object_find_key (metric, "is_spam"); - g_string_append_printf (out, "Metric: default; %s; %.2f / %.2f / 0.0\r\n", - ucl_object_toboolean (is_spam) ? "True" : "False", - ucl_object_todouble (score), - ucl_object_todouble (required_score)); + g_string_append_printf (out, + "Metric: default; %s; %.2f / %.2f / 0.0\r\n", + ucl_object_toboolean (is_spam) ? "True" : "False", + ucl_object_todouble (score), + ucl_object_todouble (required_score)); elt = ucl_object_find_key (metric, "action"); if (elt != NULL) { g_string_append_printf (out, "Action: %s\r\n", @@ -679,34 +703,38 @@ rspamd_ucl_tolegacy_output (struct rspamd_task *task, ucl_object_t *top, GString const ucl_object_t *sym_score; sym_score = ucl_object_find_key (elt, "score"); g_string_append_printf (out, "Symbol: %s(%.2f)\r\n", - ucl_object_key (elt), - ucl_object_todouble (sym_score)); + ucl_object_key (elt), + ucl_object_todouble (sym_score)); } } elt = ucl_object_find_key (metric, "subject"); if (elt != NULL) { g_string_append_printf (out, "Subject: %s\r\n", - ucl_object_tostring (elt)); + ucl_object_tostring (elt)); } } g_string_append_printf (out, "Message-ID: %s\r\n", task->message_id); } void -rspamd_protocol_http_reply (struct rspamd_http_message *msg, struct rspamd_task *task) +rspamd_protocol_http_reply (struct rspamd_http_message *msg, + struct rspamd_task *task) { - GString *logbuf; - struct metric_result *metric_res; - GHashTableIter hiter; - gpointer h, v; - ucl_object_t *top = NULL, *obj; - gdouble required_score; - gint action; + GString *logbuf; + struct metric_result *metric_res; + GHashTableIter hiter; + gpointer h, v; + ucl_object_t *top = NULL, *obj; + gdouble required_score; + gint action; /* Output the first line - check status */ logbuf = g_string_sized_new (BUFSIZ); - rspamd_printf_gstring (logbuf, "id: <%s>, qid: <%s>, ", task->message_id, task->queue_id); + rspamd_printf_gstring (logbuf, + "id: <%s>, qid: <%s>, ", + task->message_id, + task->queue_id); if (task->user) { rspamd_printf_gstring (logbuf, "user: %s, ", task->user); @@ -726,18 +754,20 @@ rspamd_protocol_http_reply (struct rspamd_http_message *msg, struct rspamd_task } if (task->messages != NULL) { - ucl_object_insert_key (top, rspamd_str_list_ucl (task->messages), "messages", 0, false); + ucl_object_insert_key (top, rspamd_str_list_ucl ( + task->messages), "messages", 0, false); } if (g_tree_nnodes (task->urls) > 0) { - ucl_object_insert_key (top, rspamd_urls_tree_ucl (task->urls, task), "urls", 0, false); + ucl_object_insert_key (top, rspamd_urls_tree_ucl (task->urls, + task), "urls", 0, false); } if (g_tree_nnodes (task->emails) > 0) { ucl_object_insert_key (top, rspamd_emails_tree_ucl (task->emails, task), - "emails", 0, false); + "emails", 0, false); } - + ucl_object_insert_key (top, ucl_object_fromstring (task->message_id), - "message-id", 0, false); + "message-id", 0, false); write_hashes_to_log (task, logbuf); if (!task->no_log) { @@ -758,11 +788,12 @@ rspamd_protocol_http_reply (struct rspamd_http_message *msg, struct rspamd_task /* Update stat for default metric */ metric_res = g_hash_table_lookup (task->results, DEFAULT_METRIC); if (metric_res != NULL) { - required_score = metric_res->metric->actions[METRIC_ACTION_REJECT].score; + required_score = + metric_res->metric->actions[METRIC_ACTION_REJECT].score; action = check_metric_action (metric_res->score, required_score, metric_res->metric); if (action <= METRIC_ACTION_NOACTION) { - task->worker->srv->stat->actions_stat[action] ++; + task->worker->srv->stat->actions_stat[action]++; } } @@ -773,9 +804,9 @@ rspamd_protocol_http_reply (struct rspamd_http_message *msg, struct rspamd_task void rspamd_protocol_write_reply (struct rspamd_task *task) { - struct rspamd_http_message *msg; - const gchar *ctype = "application/json"; - ucl_object_t *top = NULL; + struct rspamd_http_message *msg; + const gchar *ctype = "application/json"; + ucl_object_t *top = NULL; msg = rspamd_http_new_message (HTTP_RESPONSE); if (!task->is_json) { @@ -792,7 +823,7 @@ rspamd_protocol_write_reply (struct rspamd_task *task) msg->code = 500 + task->error_code % 100; msg->status = g_string_new (task->last_error); ucl_object_insert_key (top, ucl_object_fromstring (task->last_error), - "error", 0, false); + "error", 0, false); msg->body = g_string_sized_new (256); rspamd_ucl_emit_gstring (top, UCL_EMIT_JSON_COMPACT, msg->body); ucl_object_unref (top); @@ -818,13 +849,13 @@ rspamd_protocol_write_reply (struct rspamd_task *task) rspamd_http_connection_reset (task->http_conn); rspamd_http_connection_write_message (task->http_conn, msg, NULL, - ctype, task, task->sock, &task->tv, task->ev_base); + ctype, task, task->sock, &task->tv, task->ev_base); } void register_protocol_command (const gchar *name, protocol_reply_func func) { - struct custom_command *cmd; + struct custom_command *cmd; cmd = g_malloc (sizeof (struct custom_command)); cmd->name = name; diff --git a/src/libserver/protocol.h b/src/libserver/protocol.h index aed45752c..8fdd7e56f 100644 --- a/src/libserver/protocol.h +++ b/src/libserver/protocol.h @@ -27,7 +27,7 @@ struct metric; * @return */ gboolean rspamd_protocol_handle_headers (struct rspamd_task *task, - struct rspamd_http_message *msg); + struct rspamd_http_message *msg); /** * Process HTTP request to the task structure @@ -36,7 +36,7 @@ gboolean rspamd_protocol_handle_headers (struct rspamd_task *task, * @return */ gboolean rspamd_protocol_handle_request (struct rspamd_task *task, - struct rspamd_http_message *msg); + struct rspamd_http_message *msg); /** * Write task results to http message @@ -44,7 +44,7 @@ gboolean rspamd_protocol_handle_request (struct rspamd_task *task, * @param task */ void rspamd_protocol_http_reply (struct rspamd_http_message *msg, - struct rspamd_task *task); + struct rspamd_task *task); /** * Write reply for specified task command diff --git a/src/libserver/proxy.c b/src/libserver/proxy.c index 67c7665b8..a4076954b 100644 --- a/src/libserver/proxy.c +++ b/src/libserver/proxy.c @@ -28,7 +28,7 @@ static void rspamd_proxy_backend_handler (gint fd, gshort what, gpointer data); static void rspamd_proxy_client_handler (gint fd, gshort what, gpointer data); -static inline GQuark +static inline GQuark proxy_error_quark (void) { return g_quark_from_static_string ("proxy-error"); @@ -50,9 +50,9 @@ rspamd_proxy_close (rspamd_proxy_t *proxy) static void rspamd_proxy_client_handler (gint fd, gshort what, gpointer data) { - rspamd_proxy_t *proxy = data; - gint r; - GError *err = NULL; + rspamd_proxy_t *proxy = data; + gint r; + GError *err = NULL; if (what == EV_READ) { /* Got data from client */ @@ -63,14 +63,20 @@ rspamd_proxy_client_handler (gint fd, gshort what, gpointer data) proxy->read_len = r; proxy->buf_offset = 0; event_del (&proxy->backend_ev); - event_set (&proxy->backend_ev, proxy->bfd, EV_WRITE, rspamd_proxy_backend_handler, proxy); + event_set (&proxy->backend_ev, + proxy->bfd, + EV_WRITE, + rspamd_proxy_backend_handler, + proxy); event_add (&proxy->backend_ev, proxy->tv); } else { /* Error case or zero reply */ if (r < 0) { /* Error case */ - g_set_error (&err, proxy_error_quark(), r, "Client read error: %s", strerror (errno)); + g_set_error (&err, + proxy_error_quark (), r, "Client read error: %s", + strerror (errno)); rspamd_proxy_close (proxy); proxy->err_cb (err, proxy->user_data); } @@ -83,17 +89,27 @@ rspamd_proxy_client_handler (gint fd, gshort what, gpointer data) } else if (what == EV_WRITE) { /* Can write to client */ - r = write (proxy->cfd, proxy->buf + proxy->buf_offset, proxy->read_len - proxy->buf_offset); + r = write (proxy->cfd, + proxy->buf + proxy->buf_offset, + proxy->read_len - proxy->buf_offset); if (r > 0) { /* We wrote something */ - proxy->buf_offset +=r; + proxy->buf_offset += r; if (proxy->buf_offset == proxy->read_len) { /* We wrote everything */ event_del (&proxy->client_ev); - event_set (&proxy->client_ev, proxy->cfd, EV_READ, rspamd_proxy_client_handler, proxy); + event_set (&proxy->client_ev, + proxy->cfd, + EV_READ, + rspamd_proxy_client_handler, + proxy); event_add (&proxy->client_ev, proxy->tv); event_del (&proxy->backend_ev); - event_set (&proxy->backend_ev, proxy->bfd, EV_READ, rspamd_proxy_backend_handler, proxy); + event_set (&proxy->backend_ev, + proxy->bfd, + EV_READ, + rspamd_proxy_backend_handler, + proxy); event_add (&proxy->backend_ev, proxy->tv); } else { @@ -105,7 +121,9 @@ rspamd_proxy_client_handler (gint fd, gshort what, gpointer data) /* Error case or zero reply */ if (r < 0) { /* Error case */ - g_set_error (&err, proxy_error_quark(), r, "Client write error: %s", strerror (errno)); + g_set_error (&err, + proxy_error_quark (), r, "Client write error: %s", + strerror (errno)); rspamd_proxy_close (proxy); proxy->err_cb (err, proxy->user_data); } @@ -118,7 +136,7 @@ rspamd_proxy_client_handler (gint fd, gshort what, gpointer data) } else { /* Got timeout */ - g_set_error (&err, proxy_error_quark(), ETIMEDOUT, "Client timeout"); + g_set_error (&err, proxy_error_quark (), ETIMEDOUT, "Client timeout"); rspamd_proxy_close (proxy); proxy->err_cb (err, proxy->user_data); } @@ -127,9 +145,9 @@ rspamd_proxy_client_handler (gint fd, gshort what, gpointer data) static void rspamd_proxy_backend_handler (gint fd, gshort what, gpointer data) { - rspamd_proxy_t *proxy = data; - gint r; - GError *err = NULL; + rspamd_proxy_t *proxy = data; + gint r; + GError *err = NULL; if (what == EV_READ) { /* Got data from backend */ @@ -140,14 +158,20 @@ rspamd_proxy_backend_handler (gint fd, gshort what, gpointer data) proxy->read_len = r; proxy->buf_offset = 0; event_del (&proxy->client_ev); - event_set (&proxy->client_ev, proxy->bfd, EV_WRITE, rspamd_proxy_client_handler, proxy); + event_set (&proxy->client_ev, + proxy->bfd, + EV_WRITE, + rspamd_proxy_client_handler, + proxy); event_add (&proxy->client_ev, proxy->tv); } else { /* Error case or zero reply */ if (r < 0) { /* Error case */ - g_set_error (&err, proxy_error_quark(), r, "Backend read error: %s", strerror (errno)); + g_set_error (&err, + proxy_error_quark (), r, "Backend read error: %s", + strerror (errno)); rspamd_proxy_close (proxy); proxy->err_cb (err, proxy->user_data); } @@ -160,17 +184,27 @@ rspamd_proxy_backend_handler (gint fd, gshort what, gpointer data) } else if (what == EV_WRITE) { /* Can write to backend */ - r = write (proxy->bfd, proxy->buf + proxy->buf_offset, proxy->read_len - proxy->buf_offset); + r = write (proxy->bfd, + proxy->buf + proxy->buf_offset, + proxy->read_len - proxy->buf_offset); if (r > 0) { /* We wrote something */ - proxy->buf_offset +=r; + proxy->buf_offset += r; if (proxy->buf_offset == proxy->read_len) { /* We wrote everything */ event_del (&proxy->backend_ev); - event_set (&proxy->backend_ev, proxy->bfd, EV_READ, rspamd_proxy_backend_handler, proxy); + event_set (&proxy->backend_ev, + proxy->bfd, + EV_READ, + rspamd_proxy_backend_handler, + proxy); event_add (&proxy->backend_ev, proxy->tv); event_del (&proxy->client_ev); - event_set (&proxy->client_ev, proxy->cfd, EV_READ, rspamd_proxy_client_handler, proxy); + event_set (&proxy->client_ev, + proxy->cfd, + EV_READ, + rspamd_proxy_client_handler, + proxy); event_add (&proxy->client_ev, proxy->tv); } else { @@ -182,7 +216,9 @@ rspamd_proxy_backend_handler (gint fd, gshort what, gpointer data) /* Error case or zero reply */ if (r < 0) { /* Error case */ - g_set_error (&err, proxy_error_quark(), r, "Backend write error: %s", strerror (errno)); + g_set_error (&err, + proxy_error_quark (), r, "Backend write error: %s", + strerror (errno)); rspamd_proxy_close (proxy); proxy->err_cb (err, proxy->user_data); } @@ -195,7 +231,7 @@ rspamd_proxy_backend_handler (gint fd, gshort what, gpointer data) } else { /* Got timeout */ - g_set_error (&err, proxy_error_quark(), ETIMEDOUT, "Client timeout"); + g_set_error (&err, proxy_error_quark (), ETIMEDOUT, "Client timeout"); rspamd_proxy_close (proxy); proxy->err_cb (err, proxy->user_data); } @@ -210,11 +246,17 @@ rspamd_proxy_backend_handler (gint fd, gshort what, gpointer data) * @param ud user data for callback * @return new proxy object */ -rspamd_proxy_t* -rspamd_create_proxy (gint cfd, gint bfd, rspamd_mempool_t *pool, struct event_base *base, - gsize bufsize, struct timeval *tv, dispatcher_err_callback_t err_cb, gpointer ud) +rspamd_proxy_t * +rspamd_create_proxy (gint cfd, + gint bfd, + rspamd_mempool_t *pool, + struct event_base *base, + gsize bufsize, + struct timeval *tv, + dispatcher_err_callback_t err_cb, + gpointer ud) { - rspamd_proxy_t *new; + rspamd_proxy_t *new; new = rspamd_mempool_alloc0 (pool, sizeof (rspamd_proxy_t)); @@ -229,11 +271,19 @@ rspamd_create_proxy (gint cfd, gint bfd, rspamd_mempool_t *pool, struct event_ba new->tv = tv; /* Set client's and backend's interfaces to read events */ - event_set (&new->client_ev, new->cfd, EV_READ, rspamd_proxy_client_handler, new); + event_set (&new->client_ev, + new->cfd, + EV_READ, + rspamd_proxy_client_handler, + new); event_base_set (new->base, &new->client_ev); event_add (&new->client_ev, new->tv); - event_set (&new->backend_ev, new->bfd, EV_READ, rspamd_proxy_backend_handler, new); + event_set (&new->backend_ev, + new->bfd, + EV_READ, + rspamd_proxy_backend_handler, + new); event_base_set (new->base, &new->backend_ev); event_add (&new->backend_ev, new->tv); diff --git a/src/libserver/proxy.h b/src/libserver/proxy.h index c505fe83d..199da765f 100644 --- a/src/libserver/proxy.h +++ b/src/libserver/proxy.h @@ -25,8 +25,8 @@ #ifndef PROXY_H_ #define PROXY_H_ -#include "config.h" #include "buffer.h" +#include "config.h" /** * @file proxy.h @@ -34,21 +34,21 @@ */ typedef struct rspamd_proxy_s { - struct event client_ev; /**< event for client's communication */ - struct event backend_ev; /**< event for backend communication */ - struct event_base *base; /**< base for event operations */ - rspamd_mempool_t *pool; /**< memory pool */ - dispatcher_err_callback_t err_cb; /**< error callback */ - struct event_base *ev_base; /**< event base */ - gint cfd; /**< client's socket */ - gint bfd; /**< backend's socket */ - guint8 *buf; /**< exchange buffer */ - gsize bufsize; /**< buffer size */ - gint read_len; /**< read length */ - gint buf_offset; /**< offset to write */ - gpointer user_data; /**< user's data for callbacks */ - struct timeval *tv; /**< timeout for communications */ - gboolean closed; /**< whether descriptors are closed */ + struct event client_ev; /**< event for client's communication */ + struct event backend_ev; /**< event for backend communication */ + struct event_base *base; /**< base for event operations */ + rspamd_mempool_t *pool; /**< memory pool */ + dispatcher_err_callback_t err_cb; /**< error callback */ + struct event_base *ev_base; /**< event base */ + gint cfd; /**< client's socket */ + gint bfd; /**< backend's socket */ + guint8 *buf; /**< exchange buffer */ + gsize bufsize; /**< buffer size */ + gint read_len; /**< read length */ + gint buf_offset; /**< offset to write */ + gpointer user_data; /**< user's data for callbacks */ + struct timeval *tv; /**< timeout for communications */ + gboolean closed; /**< whether descriptors are closed */ } rspamd_proxy_t; /** @@ -60,9 +60,14 @@ typedef struct rspamd_proxy_s { * @param ud user data for callback * @return new proxy object */ -rspamd_proxy_t* rspamd_create_proxy (gint cfd, gint bfd, rspamd_mempool_t *pool, - struct event_base *base, gsize bufsize, struct timeval *tv, - dispatcher_err_callback_t err_cb, gpointer ud); +rspamd_proxy_t * rspamd_create_proxy (gint cfd, + gint bfd, + rspamd_mempool_t *pool, + struct event_base *base, + gsize bufsize, + struct timeval *tv, + dispatcher_err_callback_t err_cb, + gpointer ud); void rspamd_proxy_close (rspamd_proxy_t *proxy); diff --git a/src/libserver/roll_history.c b/src/libserver/roll_history.c index 504f8ae3b..d9553fa93 100644 --- a/src/libserver/roll_history.c +++ b/src/libserver/roll_history.c @@ -33,10 +33,10 @@ * @param pool pool for shared memory * @return new structure */ -struct roll_history* +struct roll_history * rspamd_roll_history_new (rspamd_mempool_t *pool) { - struct roll_history *new; + struct roll_history *new; if (pool == NULL) { return NULL; @@ -57,9 +57,9 @@ struct history_metric_callback_data { static void roll_history_symbols_callback (gpointer key, gpointer value, void *user_data) { - struct history_metric_callback_data *cb = user_data; - struct symbol *s = value; - guint wr; + struct history_metric_callback_data *cb = user_data; + struct symbol *s = value; + guint wr; if (cb->remain > 0) { wr = rspamd_snprintf (cb->pos, cb->remain, "%s, ", s->name); @@ -74,12 +74,13 @@ roll_history_symbols_callback (gpointer key, gpointer value, void *user_data) * @param task task object */ void -rspamd_roll_history_update (struct roll_history *history, struct rspamd_task *task) +rspamd_roll_history_update (struct roll_history *history, + struct rspamd_task *task) { - gint row_num; - struct roll_history_row *row; - struct metric_result *metric_res; - struct history_metric_callback_data cbdata; + gint row_num; + struct roll_history_row *row; + struct metric_result *metric_res; + struct history_metric_callback_data cbdata; if (history->need_lock) { /* Some process is getting history, so wait on a mutex */ @@ -108,10 +109,11 @@ rspamd_roll_history_update (struct roll_history *history, struct rspamd_task *ta /* Add information from task to roll history */ memcpy (&row->from_addr, &task->from_addr, sizeof (row->from_addr)); - memcpy (&row->tv, &task->tv, sizeof (row->tv)); + memcpy (&row->tv, &task->tv, sizeof (row->tv)); /* Strings */ - rspamd_strlcpy (row->message_id, task->message_id, sizeof (row->message_id)); + rspamd_strlcpy (row->message_id, task->message_id, + sizeof (row->message_id)); if (task->user) { rspamd_strlcpy (row->user, task->user, sizeof (row->message_id)); } @@ -120,19 +122,23 @@ rspamd_roll_history_update (struct roll_history *history, struct rspamd_task *ta } /* Get default metric */ - metric_res = g_hash_table_lookup (task->results, DEFAULT_METRIC); + metric_res = g_hash_table_lookup (task->results, DEFAULT_METRIC); if (metric_res == NULL) { row->symbols[0] = '\0'; row->action = METRIC_ACTION_NOACTION; } else { row->score = metric_res->score; - row->required_score = metric_res->metric->actions[METRIC_ACTION_REJECT].score; + row->required_score = + metric_res->metric->actions[METRIC_ACTION_REJECT].score; row->action = check_metric_action (metric_res->score, - metric_res->metric->actions[METRIC_ACTION_REJECT].score, metric_res->metric); + metric_res->metric->actions[METRIC_ACTION_REJECT].score, + metric_res->metric); cbdata.pos = row->symbols; cbdata.remain = sizeof (row->symbols); - g_hash_table_foreach (metric_res->symbols, roll_history_symbols_callback, &cbdata); + g_hash_table_foreach (metric_res->symbols, + roll_history_symbols_callback, + &cbdata); if (cbdata.remain > 0) { /* Remove last whitespace and comma */ *cbdata.pos-- = '\0'; @@ -155,11 +161,12 @@ rspamd_roll_history_update (struct roll_history *history, struct rspamd_task *ta gboolean rspamd_roll_history_load (struct roll_history *history, const gchar *filename) { - gint fd; - struct stat st; + gint fd; + struct stat st; if (stat (filename, &st) == -1) { - msg_info ("cannot load history from %s: %s", filename, strerror (errno)); + msg_info ("cannot load history from %s: %s", filename, + strerror (errno)); return FALSE; } @@ -169,13 +176,15 @@ rspamd_roll_history_load (struct roll_history *history, const gchar *filename) } if ((fd = open (filename, O_RDONLY)) == -1) { - msg_info ("cannot load history from %s: %s", filename, strerror (errno)); + msg_info ("cannot load history from %s: %s", filename, + strerror (errno)); return FALSE; } if (read (fd, history->rows, sizeof (history->rows)) == -1) { close (fd); - msg_info ("cannot read history from %s: %s", filename, strerror (errno)); + msg_info ("cannot read history from %s: %s", filename, + strerror (errno)); return FALSE; } @@ -193,7 +202,7 @@ rspamd_roll_history_load (struct roll_history *history, const gchar *filename) gboolean rspamd_roll_history_save (struct roll_history *history, const gchar *filename) { - gint fd; + gint fd; if ((fd = open (filename, O_WRONLY | O_CREAT | O_TRUNC, 00600)) == -1) { msg_info ("cannot save history to %s: %s", filename, strerror (errno)); diff --git a/src/libserver/roll_history.h b/src/libserver/roll_history.h index 1dff93a4f..85a8f2ddc 100644 --- a/src/libserver/roll_history.h +++ b/src/libserver/roll_history.h @@ -78,14 +78,15 @@ struct roll_history { * @param pool pool for shared memory * @return new structure */ -struct roll_history* rspamd_roll_history_new (rspamd_mempool_t *pool); +struct roll_history * rspamd_roll_history_new (rspamd_mempool_t *pool); /** * Update roll history with data from task * @param history roll history object * @param task task object */ -void rspamd_roll_history_update (struct roll_history *history, struct rspamd_task *task); +void rspamd_roll_history_update (struct roll_history *history, + struct rspamd_task *task); /** * Load previously saved history from file @@ -93,7 +94,8 @@ void rspamd_roll_history_update (struct roll_history *history, struct rspamd_tas * @param filename filename to load from * @return TRUE if history has been loaded */ -gboolean rspamd_roll_history_load (struct roll_history *history, const gchar *filename); +gboolean rspamd_roll_history_load (struct roll_history *history, + const gchar *filename); /** * Save history to file @@ -101,6 +103,7 @@ gboolean rspamd_roll_history_load (struct roll_history *history, const gchar *fi * @param filename filename to load from * @return TRUE if history has been saved */ -gboolean rspamd_roll_history_save (struct roll_history *history, const gchar *filename); +gboolean rspamd_roll_history_save (struct roll_history *history, + const gchar *filename); #endif /* ROLL_HISTORY_H_ */ diff --git a/src/libserver/spf.c b/src/libserver/spf.c index d7811b4b6..908b10ac6 100644 --- a/src/libserver/spf.c +++ b/src/libserver/spf.c @@ -24,10 +24,10 @@ #include "config.h" #include "dns.h" -#include "spf.h" +#include "filter.h" #include "main.h" #include "message.h" -#include "filter.h" +#include "spf.h" #define SPF_VER1_STR "v=spf1" #define SPF_VER2_STR "spf2." @@ -79,40 +79,42 @@ struct spf_dns_cb { gboolean in_include; }; -#define CHECK_REC(rec) \ -do { \ - if ((rec)->nested > SPF_MAX_NESTING || \ - (rec)->dns_requests > SPF_MAX_DNS_REQUESTS) { \ - msg_info ("<%s> spf recursion limit %d is reached, domain: %s", \ - (rec)->task->message_id, (rec)->dns_requests, \ - (rec)->sender_domain); \ - return FALSE; \ - } \ -} while (0) \ - -static gboolean parse_spf_record (struct rspamd_task *task, struct spf_record *rec); -static gboolean start_spf_parse (struct spf_record *rec, gchar *begin, guint ttl); +#define CHECK_REC(rec) \ + do { \ + if ((rec)->nested > SPF_MAX_NESTING || \ + (rec)->dns_requests > SPF_MAX_DNS_REQUESTS) { \ + msg_info ("<%s> spf recursion limit %d is reached, domain: %s", \ + (rec)->task->message_id, (rec)->dns_requests, \ + (rec)->sender_domain); \ + return FALSE; \ + } \ + } while (0) \ + +static gboolean parse_spf_record (struct rspamd_task *task, + struct spf_record *rec); +static gboolean start_spf_parse (struct spf_record *rec, gchar *begin, + guint ttl); /* Determine spf mech */ static spf_mech_t check_spf_mech (const gchar *elt, gboolean *need_shift) { g_assert (elt != NULL); - + *need_shift = TRUE; switch (*elt) { - case '-': - return SPF_FAIL; - case '~': - return SPF_SOFT_FAIL; - case '+': - return SPF_PASS; - case '?': - return SPF_NEUTRAL; - default: - *need_shift = FALSE; - return SPF_PASS; + case '-': + return SPF_FAIL; + case '~': + return SPF_SOFT_FAIL; + case '+': + return SPF_PASS; + case '?': + return SPF_NEUTRAL; + default: + *need_shift = FALSE; + return SPF_PASS; } } @@ -120,14 +122,14 @@ check_spf_mech (const gchar *elt, gboolean *need_shift) static void dump_spf_record (GList *addrs) { - struct spf_addr *addr; - GList *cur; - gint r = 0; - gchar logbuf[BUFSIZ], c; + struct spf_addr *addr; + GList *cur; + gint r = 0; + gchar logbuf[BUFSIZ], c; #ifdef HAVE_INET_PTON - gchar ipbuf[INET6_ADDRSTRLEN]; + gchar ipbuf[INET6_ADDRSTRLEN]; #else - struct in_addr ina; + struct in_addr ina; #endif cur = addrs; @@ -149,20 +151,35 @@ dump_spf_record (GList *addrs) } #ifdef HAVE_INET_PTON if (addr->data.normal.ipv6) { - inet_ntop (AF_INET6, &addr->data.normal.d.in6, ipbuf, sizeof (ipbuf)); + inet_ntop (AF_INET6, &addr->data.normal.d.in6, ipbuf, + sizeof (ipbuf)); } else { - inet_ntop (AF_INET, &addr->data.normal.d.in4, ipbuf, sizeof (ipbuf)); + inet_ntop (AF_INET, &addr->data.normal.d.in4, ipbuf, + sizeof (ipbuf)); } - r += snprintf (logbuf + r, sizeof (logbuf) - r, "%c%s/%d; ", c, ipbuf, addr->data.normal.mask); + r += snprintf (logbuf + r, + sizeof (logbuf) - r, + "%c%s/%d; ", + c, + ipbuf, + addr->data.normal.mask); #else ina.s_addr = addr->data.normal.d.in4.s_addr; - r += snprintf (logbuf + r, sizeof (logbuf) - r, "%c%s/%d; ", c, inet_ntoa (ina), addr->data.normal.mask); + r += snprintf (logbuf + r, + sizeof (logbuf) - r, + "%c%s/%d; ", + c, + inet_ntoa (ina), + addr->data.normal.mask); #endif } else { - r += snprintf (logbuf + r, sizeof (logbuf) - r, "%s; ", addr->spf_string); + r += snprintf (logbuf + r, + sizeof (logbuf) - r, + "%s; ", + addr->spf_string); dump_spf_record (addr->data.list); } cur = g_list_next (cur); @@ -174,8 +191,8 @@ dump_spf_record (GList *addrs) static GList * spf_addr_find (GList *addrs, gpointer to_find) { - struct spf_addr *addr; - GList *cur, *res = NULL; + struct spf_addr *addr; + GList *cur, *res = NULL; cur = addrs; while (cur) { @@ -202,9 +219,9 @@ spf_addr_find (GList *addrs, gpointer to_find) static void spf_record_destructor (gpointer r) { - struct spf_record *rec = r; - GList *cur; - struct spf_addr *addr; + struct spf_record *rec = r; + GList *cur; + struct spf_addr *addr; if (rec->addrs) { cur = rec->addrs; @@ -220,87 +237,89 @@ spf_record_destructor (gpointer r) } static gboolean -parse_spf_ipmask (const gchar *begin, struct spf_addr *addr, struct spf_record *rec) +parse_spf_ipmask (const gchar *begin, + struct spf_addr *addr, + struct spf_record *rec) { - const gchar *pos; - gchar mask_buf[5] = {'\0'}, *p; - gint state = 0, dots = 0; + const gchar *pos; + gchar mask_buf[5] = {'\0'}, *p; + gint state = 0, dots = 0; #ifdef HAVE_INET_PTON - gchar ip_buf[INET6_ADDRSTRLEN]; + gchar ip_buf[INET6_ADDRSTRLEN]; #else - gchar ip_buf[INET_ADDRSTRLEN]; + gchar ip_buf[INET_ADDRSTRLEN]; #endif - - bzero (ip_buf, sizeof (ip_buf)); + + bzero (ip_buf, sizeof (ip_buf)); bzero (mask_buf, sizeof (mask_buf)); pos = begin; p = ip_buf; while (*pos) { switch (state) { - case 0: - /* Require ':' */ - if (*pos != ':') { - msg_info ("<%s>: spf error for domain %s: semicolon missing", - rec->task->message_id, rec->sender_domain); - return FALSE; - } - state = 1; - pos ++; - p = ip_buf; - dots = 0; - break; - case 1: + case 0: + /* Require ':' */ + if (*pos != ':') { + msg_info ("<%s>: spf error for domain %s: semicolon missing", + rec->task->message_id, rec->sender_domain); + return FALSE; + } + state = 1; + pos++; + p = ip_buf; + dots = 0; + break; + case 1: #ifdef HAVE_INET_PTON - if (p - ip_buf >= (gint)sizeof (ip_buf)) { - return FALSE; - } - if (g_ascii_isxdigit (*pos)) { - *p ++ = *pos ++; - } - else if (*pos == '.' || *pos == ':') { - *p ++ = *pos ++; - dots ++; - } + if (p - ip_buf >= (gint)sizeof (ip_buf)) { + return FALSE; + } + if (g_ascii_isxdigit (*pos)) { + *p++ = *pos++; + } + else if (*pos == '.' || *pos == ':') { + *p++ = *pos++; + dots++; + } #else - /* Begin parse ip */ - if (p - ip_buf >= (gint)sizeof (ip_buf) || dots > 3) { - return FALSE; - } - if (g_ascii_isdigit (*pos)) { - *p ++ = *pos ++; - } - else if (*pos == '.') { - *p ++ = *pos ++; - dots ++; - } + /* Begin parse ip */ + if (p - ip_buf >= (gint)sizeof (ip_buf) || dots > 3) { + return FALSE; + } + if (g_ascii_isdigit (*pos)) { + *p++ = *pos++; + } + else if (*pos == '.') { + *p++ = *pos++; + dots++; + } #endif - else if (*pos == '/') { - pos ++; - p = mask_buf; - state = 2; - } - else { - /* Invalid character */ - msg_info ("<%s>: spf error for domain %s: invalid ip address", - rec->task->message_id, rec->sender_domain); - return FALSE; - } - break; - case 2: - /* Parse mask */ - if (p - mask_buf >= (gint)sizeof (mask_buf)) { - msg_info ("<%s>: spf error for domain %s: too long mask", - rec->task->message_id, rec->sender_domain); - return FALSE; - } - if (g_ascii_isdigit (*pos)) { - *p ++ = *pos ++; - } - else { - return FALSE; - } - break; + else if (*pos == '/') { + pos++; + p = mask_buf; + state = 2; + } + else { + /* Invalid character */ + msg_info ("<%s>: spf error for domain %s: invalid ip address", + rec->task->message_id, rec->sender_domain); + return FALSE; + } + break; + case 2: + /* Parse mask */ + if (p - mask_buf >= (gint)sizeof (mask_buf)) { + msg_info ("<%s>: spf error for domain %s: too long mask", + rec->task->message_id, rec->sender_domain); + return FALSE; + } + if (g_ascii_isdigit (*pos)) { + *p++ = *pos++; + } + else { + return FALSE; + } + break; } } @@ -311,7 +330,7 @@ parse_spf_ipmask (const gchar *begin, struct spf_addr *addr, struct spf_record * } else { msg_info ("<%s>: spf error for domain %s: invalid ip address", - rec->task->message_id, rec->sender_domain); + rec->task->message_id, rec->sender_domain); return FALSE; } } @@ -328,16 +347,22 @@ parse_spf_ipmask (const gchar *begin, struct spf_addr *addr, struct spf_record * if (!addr->data.normal.ipv6) { addr->data.normal.mask = strtoul (mask_buf, NULL, 10); if (addr->data.normal.mask > 32) { - msg_info ("<%s>: spf error for domain %s: bad ipmask value: '%s'", - rec->task->message_id, rec->sender_domain, begin); + msg_info ( + "<%s>: spf error for domain %s: bad ipmask value: '%s'", + rec->task->message_id, + rec->sender_domain, + begin); return FALSE; } } else { addr->data.normal.mask = strtoul (mask_buf, NULL, 10); if (addr->data.normal.mask > 128) { - msg_info ("<%s>: spf error for domain %s: bad ipmask value: '%s'", - rec->task->message_id, rec->sender_domain, begin); + msg_info ( + "<%s>: spf error for domain %s: bad ipmask value: '%s'", + rec->task->message_id, + rec->sender_domain, + begin); return FALSE; } } @@ -351,10 +376,13 @@ parse_spf_ipmask (const gchar *begin, struct spf_addr *addr, struct spf_record * } static gchar * -parse_spf_hostmask (struct rspamd_task *task, const gchar *begin, struct spf_addr *addr, struct spf_record *rec) +parse_spf_hostmask (struct rspamd_task *task, + const gchar *begin, + struct spf_addr *addr, + struct spf_record *rec) { - gchar *host = NULL, *p, mask_buf[3]; - gint hostlen; + gchar *host = NULL, *p, mask_buf[3]; + gint hostlen; bzero (mask_buf, sizeof (mask_buf)); if (*begin == '\0' || *begin == '/') { @@ -368,7 +396,7 @@ parse_spf_hostmask (struct rspamd_task *task, const gchar *begin, struct spf_add addr->data.normal.mask = strtoul (mask_buf, NULL, 10); if (addr->data.normal.mask > 32) { msg_info ("<%s>: spf error for domain %s: too long mask", - rec->task->message_id, rec->sender_domain); + rec->task->message_id, rec->sender_domain); return FALSE; } if (host == NULL) { @@ -389,7 +417,7 @@ parse_spf_hostmask (struct rspamd_task *task, const gchar *begin, struct spf_add static void spf_record_process_addr (struct rdns_reply_entry *elt, - struct spf_dns_cb *cb, struct rspamd_task *task) + struct spf_dns_cb *cb, struct rspamd_task *task) { struct spf_addr *addr = cb->addr, *new_addr; GList *tmp = NULL; @@ -408,11 +436,13 @@ spf_record_process_addr (struct rdns_reply_entry *elt, memcpy (new_addr, addr, sizeof (struct spf_addr)); new_addr->data.normal.d.in4.s_addr = elt->content.a.addr.s_addr; new_addr->data.normal.parsed = TRUE; - cb->rec->addrs = g_list_insert_before (cb->rec->addrs, tmp, new_addr); + cb->rec->addrs = g_list_insert_before (cb->rec->addrs, + tmp, + new_addr); } else { msg_info ("<%s>: spf error for domain %s: addresses mismatch", - task->message_id, cb->rec->sender_domain); + task->message_id, cb->rec->sender_domain); } } @@ -420,7 +450,7 @@ spf_record_process_addr (struct rdns_reply_entry *elt, else if (elt->type == RDNS_REQUEST_AAAA) { if (!addr->data.normal.parsed) { memcpy (&addr->data.normal.d.in6, - &elt->content.aaa.addr, sizeof (struct in6_addr)); + &elt->content.aaa.addr, sizeof (struct in6_addr)); addr->data.normal.mask = 32; addr->data.normal.parsed = TRUE; addr->data.normal.ipv6 = TRUE; @@ -429,17 +459,21 @@ spf_record_process_addr (struct rdns_reply_entry *elt, /* Insert one more address */ tmp = spf_addr_find (cb->rec->addrs, addr); if (tmp) { - new_addr = rspamd_mempool_alloc (task->task_pool, sizeof (struct spf_addr)); + new_addr = + rspamd_mempool_alloc (task->task_pool, + sizeof (struct spf_addr)); memcpy (new_addr, addr, sizeof (struct spf_addr)); memcpy (&new_addr->data.normal.d.in6, - &elt->content.aaa.addr, sizeof (struct in6_addr)); + &elt->content.aaa.addr, sizeof (struct in6_addr)); new_addr->data.normal.parsed = TRUE; new_addr->data.normal.ipv6 = TRUE; - cb->rec->addrs = g_list_insert_before (cb->rec->addrs, tmp, new_addr); + cb->rec->addrs = g_list_insert_before (cb->rec->addrs, + tmp, + new_addr); } else { msg_info ("<%s>: spf error for domain %s: addresses mismatch", - task->message_id, cb->rec->sender_domain); + task->message_id, cb->rec->sender_domain); } } } @@ -448,34 +482,37 @@ spf_record_process_addr (struct rdns_reply_entry *elt, static void spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) { - struct spf_dns_cb *cb = arg; - gchar *begin; - struct rdns_reply_entry *elt_data; - GList *tmp = NULL; - struct rspamd_task *task; + struct spf_dns_cb *cb = arg; + gchar *begin; + struct rdns_reply_entry *elt_data; + GList *tmp = NULL; + struct rspamd_task *task; task = cb->rec->task; - cb->rec->requests_inflight --; + cb->rec->requests_inflight--; if (reply->code == RDNS_RC_NOERROR) { /* Add all logic for all DNS states here */ - LL_FOREACH (reply->entries, elt_data) { + LL_FOREACH (reply->entries, elt_data) + { switch (cb->cur_action) { case SPF_RESOLVE_MX: if (elt_data->type == RDNS_REQUEST_MX) { /* Now resolve A record for this MX */ - if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_record_dns_callback, (void *)cb, RDNS_REQUEST_A, - elt_data->content.mx.name)) { - task->dns_requests ++; - cb->rec->requests_inflight ++; + if (make_dns_request (task->resolver, task->s, + task->task_pool, + spf_record_dns_callback, (void *)cb, RDNS_REQUEST_A, + elt_data->content.mx.name)) { + task->dns_requests++; + cb->rec->requests_inflight++; } - if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_record_dns_callback, (void *)cb, RDNS_REQUEST_AAAA, - elt_data->content.mx.name)) { - task->dns_requests ++; - cb->rec->requests_inflight ++; + if (make_dns_request (task->resolver, task->s, + task->task_pool, + spf_record_dns_callback, (void *)cb, RDNS_REQUEST_AAAA, + elt_data->content.mx.name)) { + task->dns_requests++; + cb->rec->requests_inflight++; } } else { @@ -488,17 +525,19 @@ spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) break; case SPF_RESOLVE_PTR: if (elt_data->type == RDNS_REQUEST_PTR) { - if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_record_dns_callback, (void *)cb, RDNS_REQUEST_A, - elt_data->content.ptr.name)) { - task->dns_requests ++; - cb->rec->requests_inflight ++; + if (make_dns_request (task->resolver, task->s, + task->task_pool, + spf_record_dns_callback, (void *)cb, RDNS_REQUEST_A, + elt_data->content.ptr.name)) { + task->dns_requests++; + cb->rec->requests_inflight++; } - if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_record_dns_callback, (void *)cb, RDNS_REQUEST_AAAA, - elt_data->content.ptr.name)) { - task->dns_requests ++; - cb->rec->requests_inflight ++; + if (make_dns_request (task->resolver, task->s, + task->task_pool, + spf_record_dns_callback, (void *)cb, RDNS_REQUEST_AAAA, + elt_data->content.ptr.name)) { + task->dns_requests++; + cb->rec->requests_inflight++; } } else { @@ -543,7 +582,8 @@ spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) case SPF_RESOLVE_EXP: break; case SPF_RESOLVE_EXISTS: - if (elt_data->type == RDNS_REQUEST_A || elt_data->type == RDNS_REQUEST_AAAA) { + if (elt_data->type == RDNS_REQUEST_A || elt_data->type == + RDNS_REQUEST_AAAA) { /* If specified address resolves, we can accept connection from every IP */ cb->addr->data.normal.d.in4.s_addr = INADDR_NONE; cb->addr->data.normal.mask = 0; @@ -554,50 +594,63 @@ spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) } else if (reply->code == RDNS_RC_NXDOMAIN) { switch (cb->cur_action) { - case SPF_RESOLVE_MX: - if (rdns_request_has_type (reply->request, RDNS_REQUEST_MX)) { - msg_info ("<%s>: spf error for domain %s: cannot find MX record for %s", - task->message_id, cb->rec->sender_domain, cb->rec->cur_domain); - cb->addr->data.normal.d.in4.s_addr = INADDR_NONE; - cb->addr->data.normal.mask = 32; - } - else { - msg_info ("<%s>: spf error for domain %s: cannot resolve MX record for %s", - task->message_id, cb->rec->sender_domain, cb->rec->cur_domain); - cb->addr->data.normal.d.in4.s_addr = INADDR_NONE; - cb->addr->data.normal.mask = 32; - } - break; - case SPF_RESOLVE_A: - if (rdns_request_has_type (reply->request, RDNS_REQUEST_A)) { - cb->addr->data.normal.d.in4.s_addr = INADDR_NONE; - cb->addr->data.normal.mask = 32; - } - break; + case SPF_RESOLVE_MX: + if (rdns_request_has_type (reply->request, RDNS_REQUEST_MX)) { + msg_info ( + "<%s>: spf error for domain %s: cannot find MX record for %s", + task->message_id, + cb->rec->sender_domain, + cb->rec->cur_domain); + cb->addr->data.normal.d.in4.s_addr = INADDR_NONE; + cb->addr->data.normal.mask = 32; + } + else { + msg_info ( + "<%s>: spf error for domain %s: cannot resolve MX record for %s", + task->message_id, + cb->rec->sender_domain, + cb->rec->cur_domain); + cb->addr->data.normal.d.in4.s_addr = INADDR_NONE; + cb->addr->data.normal.mask = 32; + } + break; + case SPF_RESOLVE_A: + if (rdns_request_has_type (reply->request, RDNS_REQUEST_A)) { + cb->addr->data.normal.d.in4.s_addr = INADDR_NONE; + cb->addr->data.normal.mask = 32; + } + break; #ifdef HAVE_INET_PTON - case SPF_RESOLVE_AAA: - if (rdns_request_has_type (reply->request, RDNS_REQUEST_AAAA)) { - memset (&cb->addr->data.normal.d.in6, 0xff, sizeof (struct in6_addr)); - cb->addr->data.normal.mask = 32; - } - break; + case SPF_RESOLVE_AAA: + if (rdns_request_has_type (reply->request, RDNS_REQUEST_AAAA)) { + memset (&cb->addr->data.normal.d.in6, 0xff, + sizeof (struct in6_addr)); + cb->addr->data.normal.mask = 32; + } + break; #endif - case SPF_RESOLVE_PTR: - break; - case SPF_RESOLVE_REDIRECT: - msg_info ("<%s>: spf error for domain %s: cannot resolve TXT record for %s", - task->message_id, cb->rec->sender_domain, cb->rec->cur_domain); - break; - case SPF_RESOLVE_INCLUDE: - msg_info ("<%s>: spf error for domain %s: cannot resolve TXT record for %s", - task->message_id, cb->rec->sender_domain, cb->rec->cur_domain); - break; - case SPF_RESOLVE_EXP: - break; - case SPF_RESOLVE_EXISTS: - cb->addr->data.normal.d.in4.s_addr = INADDR_NONE; - cb->addr->data.normal.mask = 32; - break; + case SPF_RESOLVE_PTR: + break; + case SPF_RESOLVE_REDIRECT: + msg_info ( + "<%s>: spf error for domain %s: cannot resolve TXT record for %s", + task->message_id, + cb->rec->sender_domain, + cb->rec->cur_domain); + break; + case SPF_RESOLVE_INCLUDE: + msg_info ( + "<%s>: spf error for domain %s: cannot resolve TXT record for %s", + task->message_id, + cb->rec->sender_domain, + cb->rec->cur_domain); + break; + case SPF_RESOLVE_EXP: + break; + case SPF_RESOLVE_EXISTS: + cb->addr->data.normal.d.in4.s_addr = INADDR_NONE; + cb->addr->data.normal.mask = 32; + break; } } @@ -607,13 +660,16 @@ spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) } static gboolean -parse_spf_a (struct rspamd_task *task, const gchar *begin, struct spf_record *rec, struct spf_addr *addr) +parse_spf_a (struct rspamd_task *task, + const gchar *begin, + struct spf_record *rec, + struct spf_addr *addr) { struct spf_dns_cb *cb; - gchar *host = NULL; - + gchar *host = NULL; + CHECK_REC (rec); - + /* * a * a/<prefix-length> @@ -629,31 +685,31 @@ parse_spf_a (struct rspamd_task *task, const gchar *begin, struct spf_record *re addr->data.normal.mask = 32; } else if (*begin == ':') { - begin ++; + begin++; } else if (*begin != '/') { /* Invalid A record */ return FALSE; } - + if (host == NULL) { host = parse_spf_hostmask (task, begin, addr, rec); } - + if (host == NULL) { return FALSE; } - rec->dns_requests ++; + rec->dns_requests++; cb = rspamd_mempool_alloc (task->task_pool, sizeof (struct spf_dns_cb)); cb->rec = rec; cb->addr = addr; cb->cur_action = SPF_RESOLVE_A; cb->in_include = rec->in_include; if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_record_dns_callback, (void *)cb, RDNS_REQUEST_A, host)) { - task->dns_requests ++; - rec->requests_inflight ++; + spf_record_dns_callback, (void *)cb, RDNS_REQUEST_A, host)) { + task->dns_requests++; + rec->requests_inflight++; return TRUE; } @@ -662,10 +718,13 @@ parse_spf_a (struct rspamd_task *task, const gchar *begin, struct spf_record *re } static gboolean -parse_spf_ptr (struct rspamd_task *task, const gchar *begin, struct spf_record *rec, struct spf_addr *addr) +parse_spf_ptr (struct rspamd_task *task, + const gchar *begin, + struct spf_record *rec, + struct spf_addr *addr) { struct spf_dns_cb *cb; - gchar *host, *ptr; + gchar *host, *ptr; CHECK_REC (rec); @@ -673,7 +732,7 @@ parse_spf_ptr (struct rspamd_task *task, const gchar *begin, struct spf_record * return FALSE; } if (*begin == ':') { - begin ++; + begin++; host = rspamd_mempool_strdup (task->task_pool, begin); } else if (*begin == '\0') { @@ -683,7 +742,7 @@ parse_spf_ptr (struct rspamd_task *task, const gchar *begin, struct spf_record * return FALSE; } - rec->dns_requests ++; + rec->dns_requests++; cb = rspamd_mempool_alloc (task->task_pool, sizeof (struct spf_dns_cb)); cb->rec = rec; cb->addr = addr; @@ -691,15 +750,17 @@ parse_spf_ptr (struct rspamd_task *task, const gchar *begin, struct spf_record * cb->cur_action = SPF_RESOLVE_PTR; cb->in_include = rec->in_include; cb->ptr_host = host; - ptr = rdns_generate_ptr_from_str (rspamd_inet_address_to_string (&task->from_addr)); + ptr = + rdns_generate_ptr_from_str (rspamd_inet_address_to_string ( + &task->from_addr)); if (ptr == NULL) { return FALSE; } rspamd_mempool_add_destructor (task->task_pool, free, ptr); if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_record_dns_callback, (void *)cb, RDNS_REQUEST_PTR, ptr)) { - task->dns_requests ++; - rec->requests_inflight ++; + spf_record_dns_callback, (void *)cb, RDNS_REQUEST_PTR, ptr)) { + task->dns_requests++; + rec->requests_inflight++; return TRUE; } @@ -709,26 +770,29 @@ parse_spf_ptr (struct rspamd_task *task, const gchar *begin, struct spf_record * } static gboolean -parse_spf_mx (struct rspamd_task *task, const gchar *begin, struct spf_record *rec, struct spf_addr *addr) +parse_spf_mx (struct rspamd_task *task, + const gchar *begin, + struct spf_record *rec, + struct spf_addr *addr) { struct spf_dns_cb *cb; - gchar *host; - + gchar *host; + CHECK_REC (rec); - + if (begin == NULL) { return FALSE; } if (*begin == ':') { - begin ++; + begin++; } - + host = parse_spf_hostmask (task, begin, addr, rec); - + if (host == NULL) { return FALSE; } - rec->dns_requests ++; + rec->dns_requests++; cb = rspamd_mempool_alloc (task->task_pool, sizeof (struct spf_dns_cb)); cb->rec = rec; cb->addr = addr; @@ -736,10 +800,10 @@ parse_spf_mx (struct rspamd_task *task, const gchar *begin, struct spf_record *r cb->cur_action = SPF_RESOLVE_MX; cb->in_include = rec->in_include; if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_record_dns_callback, (void *)cb, RDNS_REQUEST_MX, host)) { - task->dns_requests ++; - rec->requests_inflight ++; - + spf_record_dns_callback, (void *)cb, RDNS_REQUEST_MX, host)) { + task->dns_requests++; + rec->requests_inflight++; + return TRUE; } @@ -747,7 +811,10 @@ parse_spf_mx (struct rspamd_task *task, const gchar *begin, struct spf_record *r } static gboolean -parse_spf_all (struct rspamd_task *task, const gchar *begin, struct spf_record *rec, struct spf_addr *addr) +parse_spf_all (struct rspamd_task *task, + const gchar *begin, + struct spf_record *rec, + struct spf_addr *addr) { /* All is 0/0 */ memset (&addr->data.normal.d, 0, sizeof (addr->data.normal.d)); @@ -764,7 +831,10 @@ parse_spf_all (struct rspamd_task *task, const gchar *begin, struct spf_record * } static gboolean -parse_spf_ip4 (struct rspamd_task *task, const gchar *begin, struct spf_record *rec, struct spf_addr *addr) +parse_spf_ip4 (struct rspamd_task *task, + const gchar *begin, + struct spf_record *rec, + struct spf_addr *addr) { /* ip4:addr[/mask] */ @@ -774,7 +844,10 @@ parse_spf_ip4 (struct rspamd_task *task, const gchar *begin, struct spf_record * #ifdef HAVE_INET_PTON static gboolean -parse_spf_ip6 (struct rspamd_task *task, const gchar *begin, struct spf_record *rec, struct spf_addr *addr) +parse_spf_ip6 (struct rspamd_task *task, + const gchar *begin, + struct spf_record *rec, + struct spf_addr *addr) { /* ip6:addr[/mask] */ @@ -784,18 +857,21 @@ parse_spf_ip6 (struct rspamd_task *task, const gchar *begin, struct spf_record * #endif static gboolean -parse_spf_include (struct rspamd_task *task, const gchar *begin, struct spf_record *rec, struct spf_addr *addr) +parse_spf_include (struct rspamd_task *task, + const gchar *begin, + struct spf_record *rec, + struct spf_addr *addr) { struct spf_dns_cb *cb; - gchar *domain; + gchar *domain; CHECK_REC (rec); if (begin == NULL || *begin != ':') { return FALSE; } - begin ++; - rec->dns_requests ++; + begin++; + rec->dns_requests++; cb = rspamd_mempool_alloc (task->task_pool, sizeof (struct spf_dns_cb)); cb->rec = rec; @@ -806,9 +882,9 @@ parse_spf_include (struct rspamd_task *task, const gchar *begin, struct spf_reco addr->data.list = NULL; domain = rspamd_mempool_strdup (task->task_pool, begin); if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_record_dns_callback, (void *)cb, RDNS_REQUEST_TXT, domain)) { - task->dns_requests ++; - rec->requests_inflight ++; + spf_record_dns_callback, (void *)cb, RDNS_REQUEST_TXT, domain)) { + task->dns_requests++; + rec->requests_inflight++; return TRUE; } @@ -818,7 +894,10 @@ parse_spf_include (struct rspamd_task *task, const gchar *begin, struct spf_reco } static gboolean -parse_spf_exp (struct rspamd_task *task, const gchar *begin, struct spf_record *rec, struct spf_addr *addr) +parse_spf_exp (struct rspamd_task *task, + const gchar *begin, + struct spf_record *rec, + struct spf_addr *addr) { CHECK_REC (rec); @@ -827,18 +906,21 @@ parse_spf_exp (struct rspamd_task *task, const gchar *begin, struct spf_record * } static gboolean -parse_spf_redirect (struct rspamd_task *task, const gchar *begin, struct spf_record *rec, struct spf_addr *addr) +parse_spf_redirect (struct rspamd_task *task, + const gchar *begin, + struct spf_record *rec, + struct spf_addr *addr) { struct spf_dns_cb *cb; - gchar *domain; + gchar *domain; CHECK_REC (rec); if (begin == NULL || *begin != '=') { return FALSE; } - begin ++; - rec->dns_requests ++; + begin++; + rec->dns_requests++; cb = rspamd_mempool_alloc (task->task_pool, sizeof (struct spf_dns_cb)); cb->rec = rec; @@ -847,10 +929,10 @@ parse_spf_redirect (struct rspamd_task *task, const gchar *begin, struct spf_rec cb->in_include = rec->in_include; domain = rspamd_mempool_strdup (task->task_pool, begin); if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_record_dns_callback, (void *)cb, RDNS_REQUEST_TXT, domain)) { - task->dns_requests ++; - rec->requests_inflight ++; - + spf_record_dns_callback, (void *)cb, RDNS_REQUEST_TXT, domain)) { + task->dns_requests++; + rec->requests_inflight++; + return TRUE; } @@ -858,18 +940,21 @@ parse_spf_redirect (struct rspamd_task *task, const gchar *begin, struct spf_rec } static gboolean -parse_spf_exists (struct rspamd_task *task, const gchar *begin, struct spf_record *rec, struct spf_addr *addr) +parse_spf_exists (struct rspamd_task *task, + const gchar *begin, + struct spf_record *rec, + struct spf_addr *addr) { struct spf_dns_cb *cb; - gchar *host; + gchar *host; CHECK_REC (rec); - + if (begin == NULL || *begin != ':') { return FALSE; } - begin ++; - rec->dns_requests ++; + begin++; + rec->dns_requests++; addr->data.normal.mask = 32; cb = rspamd_mempool_alloc (task->task_pool, sizeof (struct spf_dns_cb)); @@ -880,10 +965,10 @@ parse_spf_exists (struct rspamd_task *task, const gchar *begin, struct spf_recor host = rspamd_mempool_strdup (task->task_pool, begin); if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_record_dns_callback, (void *)cb, RDNS_REQUEST_A, host)) { - task->dns_requests ++; - rec->requests_inflight ++; - + spf_record_dns_callback, (void *)cb, RDNS_REQUEST_A, host)) { + task->dns_requests++; + rec->requests_inflight++; + return TRUE; } @@ -893,8 +978,8 @@ parse_spf_exists (struct rspamd_task *task, const gchar *begin, struct spf_recor static void reverse_spf_ip (gchar *ip, gint len) { - gchar ipbuf[sizeof("255.255.255.255") - 1], *p, *c; - gint t = 0, l = len; + gchar ipbuf[sizeof("255.255.255.255") - 1], *p, *c; + gint t = 0, l = len; if (len > (gint)sizeof (ipbuf)) { msg_info ("cannot reverse string of length %d", len); @@ -902,127 +987,136 @@ reverse_spf_ip (gchar *ip, gint len) } p = ipbuf + len; - c = ip; - while (-- l) { + c = ip; + while (--l) { if (*c == '.') { memcpy (p, c - t, t); *--p = '.'; - c ++; + c++; t = 0; continue; } - t ++; - c ++; - p --; + t++; + c++; + p--; } memcpy (p - 1, c - t, t + 1); - memcpy (ip, ipbuf, len); + memcpy (ip, ipbuf, len); } static gchar * -expand_spf_macro (struct rspamd_task *task, struct spf_record *rec, gchar *begin) +expand_spf_macro (struct rspamd_task *task, struct spf_record *rec, + gchar *begin) { - gchar *p, *c, *new, *tmp; - gint len = 0, slen = 0, state = 0; + gchar *p, *c, *new, *tmp; + gint len = 0, slen = 0, state = 0; #ifdef HAVE_INET_PTON - gchar ip_buf[INET6_ADDRSTRLEN]; + gchar ip_buf[INET6_ADDRSTRLEN]; #endif - gboolean need_expand = FALSE; + gboolean need_expand = FALSE; p = begin; /* Calculate length */ while (*p) { switch (state) { - case 0: - /* Skip any character and wait for % in input */ - if (*p == '%') { - state = 1; - } - else { - len ++; - } + case 0: + /* Skip any character and wait for % in input */ + if (*p == '%') { + state = 1; + } + else { + len++; + } - slen ++; - p ++; - break; - case 1: - /* We got % sign, so we should whether wait for { or for - or for _ or for % */ - if (*p == '%' || *p == '-') { - /* Just a single % sign or space */ - len ++; - } - else if (*p == '_') { - /* %20 */ - len += sizeof ("%20") - 1; - } - else if (*p == '{') { - state = 2; - } - else { - /* Something unknown */ - msg_info ("<%s>: spf error for domain %s: unknown spf element", - task->message_id, rec->sender_domain); - return begin; - } - p ++; - slen ++; - break; - case 2: - /* Read macro name */ - switch (g_ascii_tolower (*p)) { - case 'i': + slen++; + p++; + break; + case 1: + /* We got % sign, so we should whether wait for { or for - or for _ or for % */ + if (*p == '%' || *p == '-') { + /* Just a single % sign or space */ + len++; + } + else if (*p == '_') { + /* %20 */ + len += sizeof ("%20") - 1; + } + else if (*p == '{') { + state = 2; + } + else { + /* Something unknown */ + msg_info ("<%s>: spf error for domain %s: unknown spf element", + task->message_id, rec->sender_domain); + return begin; + } + p++; + slen++; + break; + case 2: + /* Read macro name */ + switch (g_ascii_tolower (*p)) { + case 'i': #ifdef HAVE_INET_PTON - len += sizeof (INET6_ADDRSTRLEN) - 1; + len += sizeof (INET6_ADDRSTRLEN) - 1; #else - len += sizeof (INET_ADDRSTRLEN) - 1; + len += sizeof (INET_ADDRSTRLEN) - 1; #endif - break; - case 's': - len += strlen (rec->sender); - break; - case 'l': - len += strlen (rec->local_part); - break; - case 'o': - len += strlen (rec->sender_domain); - break; - case 'd': - len += strlen (rec->cur_domain); - break; - case 'v': - len += sizeof ("in-addr") - 1; - break; - case 'h': - if (task->helo) { - len += strlen (task->helo); - } - break; - default: - msg_info ("<%s>: spf error for domain %s: unknown or unsupported spf macro %c in %s", - task->message_id, rec->sender_domain, *p, begin); - return begin; - } - p ++; - slen ++; - state = 3; break; - case 3: - /* Read modifier */ - if (*p == '}') { - state = 0; - need_expand = TRUE; + case 's': + len += strlen (rec->sender); + break; + case 'l': + len += strlen (rec->local_part); + break; + case 'o': + len += strlen (rec->sender_domain); + break; + case 'd': + len += strlen (rec->cur_domain); + break; + case 'v': + len += sizeof ("in-addr") - 1; + break; + case 'h': + if (task->helo) { + len += strlen (task->helo); } - else if (*p != 'r' && !g_ascii_isdigit (*p)) { - msg_info ("<%s>: spf error for domain %s: unknown or unsupported spf modifier %c in %s", - task->message_id, rec->sender_domain, *p, begin); - return begin; - } - p ++; - slen ++; break; + default: + msg_info ( + "<%s>: spf error for domain %s: unknown or unsupported spf macro %c in %s", + task->message_id, + rec->sender_domain, + *p, + begin); + return begin; + } + p++; + slen++; + state = 3; + break; + case 3: + /* Read modifier */ + if (*p == '}') { + state = 0; + need_expand = TRUE; + } + else if (*p != 'r' && !g_ascii_isdigit (*p)) { + msg_info ( + "<%s>: spf error for domain %s: unknown or unsupported spf modifier %c in %s", + task->message_id, + rec->sender_domain, + *p, + begin); + return begin; + } + p++; + slen++; + break; } } @@ -1030,7 +1124,7 @@ expand_spf_macro (struct rspamd_task *task, struct spf_record *rec, gchar *begin /* No expansion needed */ return begin; } - + new = rspamd_mempool_alloc (task->task_pool, len + 1); c = new; @@ -1040,138 +1134,146 @@ expand_spf_macro (struct rspamd_task *task, struct spf_record *rec, gchar *begin while (*p) { switch (state) { - case 0: - /* Skip any character and wait for % in input */ - if (*p == '%') { - state = 1; - } - else { - *c = *p; - c ++; - } + case 0: + /* Skip any character and wait for % in input */ + if (*p == '%') { + state = 1; + } + else { + *c = *p; + c++; + } - p ++; - break; - case 1: - /* We got % sign, so we should whether wait for { or for - or for _ or for % */ - if (*p == '%') { - /* Just a single % sign or space */ - *c++ = '%'; - } - else if (*p == '-') { - *c++ = ' '; - } - else if (*p == '_') { - /* %20 */ - *c++ = '%'; - *c++ = '2'; - *c++ = '0'; - } - else if (*p == '{') { - state = 2; - } - else { - /* Something unknown */ - msg_info ("<%s>: spf error for domain %s: unknown spf element", - task->message_id, rec->sender_domain); - return begin; - } - p ++; - break; - case 2: - /* Read macro name */ - switch (g_ascii_tolower (*p)) { - case 'i': + p++; + break; + case 1: + /* We got % sign, so we should whether wait for { or for - or for _ or for % */ + if (*p == '%') { + /* Just a single % sign or space */ + *c++ = '%'; + } + else if (*p == '-') { + *c++ = ' '; + } + else if (*p == '_') { + /* %20 */ + *c++ = '%'; + *c++ = '2'; + *c++ = '0'; + } + else if (*p == '{') { + state = 2; + } + else { + /* Something unknown */ + msg_info ("<%s>: spf error for domain %s: unknown spf element", + task->message_id, rec->sender_domain); + return begin; + } + p++; + break; + case 2: + /* Read macro name */ + switch (g_ascii_tolower (*p)) { + case 'i': #ifdef HAVE_INET_PTON - len = rspamd_strlcpy (ip_buf, - rspamd_inet_address_to_string (&task->from_addr), - sizeof (ip_buf)); - memcpy (c, ip_buf, len); + len = rspamd_strlcpy (ip_buf, + rspamd_inet_address_to_string (&task->from_addr), + sizeof (ip_buf)); + memcpy (c, ip_buf, len); #else - tmp = inet_ntoa (task->from_addr); - len = strlen (tmp); - memcpy (c, tmp, len); + tmp = inet_ntoa (task->from_addr); + len = strlen (tmp); + memcpy (c, tmp, len); #endif - c += len; - break; - case 's': - len = strlen (rec->sender); - memcpy (c, rec->sender, len); - c += len; - break; - case 'l': - len = strlen (rec->local_part); - memcpy (c, rec->local_part, len); - c += len; - break; - case 'o': - len = strlen (rec->sender_domain); - memcpy (c, rec->sender_domain, len); - c += len; - break; - case 'd': - len = strlen (rec->cur_domain); - memcpy (c, rec->cur_domain, len); - c += len; - break; - case 'v': - len = sizeof ("in-addr") - 1; - memcpy (c, "in-addr", len); - c += len; - break; - case 'h': - if (task->helo) { - tmp = strchr (task->helo, '@'); - if (tmp) { - len = strlen (tmp + 1); - memcpy (c, tmp + 1, len); - c += len; - } - } - break; - default: - msg_info ("<%s>: spf error for domain %s: unknown or unsupported spf macro %c in %s", - task->message_id, rec->sender_domain, *p, begin); - return begin; - } - p ++; - state = 3; + c += len; break; - case 3: - /* Read modifier */ - if (*p == '}') { - state = 0; - } - else if (*p == 'r' && len != 0) { - reverse_spf_ip (c - len, len); - len = 0; - } - else if (g_ascii_isdigit (*p)) { - /*XXX: try to implement domain strimming */ + case 's': + len = strlen (rec->sender); + memcpy (c, rec->sender, len); + c += len; + break; + case 'l': + len = strlen (rec->local_part); + memcpy (c, rec->local_part, len); + c += len; + break; + case 'o': + len = strlen (rec->sender_domain); + memcpy (c, rec->sender_domain, len); + c += len; + break; + case 'd': + len = strlen (rec->cur_domain); + memcpy (c, rec->cur_domain, len); + c += len; + break; + case 'v': + len = sizeof ("in-addr") - 1; + memcpy (c, "in-addr", len); + c += len; + break; + case 'h': + if (task->helo) { + tmp = strchr (task->helo, '@'); + if (tmp) { + len = strlen (tmp + 1); + memcpy (c, tmp + 1, len); + c += len; + } } - else { - msg_info ("<%s>: spf error for domain %s: unknown or unsupported spf macro %c in %s", - task->message_id, rec->sender_domain, *p, begin); - return begin; - } - p ++; break; + default: + msg_info ( + "<%s>: spf error for domain %s: unknown or unsupported spf macro %c in %s", + task->message_id, + rec->sender_domain, + *p, + begin); + return begin; + } + p++; + state = 3; + break; + case 3: + /* Read modifier */ + if (*p == '}') { + state = 0; + } + else if (*p == 'r' && len != 0) { + reverse_spf_ip (c - len, len); + len = 0; + } + else if (g_ascii_isdigit (*p)) { + /*XXX: try to implement domain strimming */ + } + else { + msg_info ( + "<%s>: spf error for domain %s: unknown or unsupported spf macro %c in %s", + task->message_id, + rec->sender_domain, + *p, + begin); + return begin; + } + p++; + break; } } /* Null terminate */ *c = '\0'; return new; - + } -#define NEW_ADDR(x) do { \ - (x) = rspamd_mempool_alloc (task->task_pool, sizeof (struct spf_addr)); \ - (x)->mech = check_spf_mech (rec->cur_elt, &need_shift); \ - (x)->spf_string = rspamd_mempool_strdup (task->task_pool, begin); \ - memset (&(x)->data.normal, 0, sizeof ((x)->data.normal)); \ - (x)->data.normal.mask = 32; \ - (x)->is_list = FALSE; \ +#define NEW_ADDR(x) do { \ + (x) = rspamd_mempool_alloc (task->task_pool, sizeof (struct spf_addr)); \ + (x)->mech = check_spf_mech (rec->cur_elt, &need_shift); \ + (x)->spf_string = rspamd_mempool_strdup (task->task_pool, begin); \ + memset (&(x)->data.normal, 0, sizeof ((x)->data.normal)); \ + (x)->data.normal.mask = 32; \ + (x)->is_list = FALSE; \ } while (0); /* Read current element and try to parse record */ @@ -1180,141 +1282,153 @@ parse_spf_record (struct rspamd_task *task, struct spf_record *rec) { struct spf_addr *new = NULL; gboolean need_shift, res = FALSE; - gchar *begin; - + gchar *begin; + rec->cur_elt = rec->elts[rec->elt_num]; if (rec->cur_elt == NULL) { return FALSE; } else if (*rec->cur_elt == '\0') { /* Silently skip empty elements */ - rec->elt_num ++; + rec->elt_num++; return TRUE; } else { begin = expand_spf_macro (task, rec, rec->cur_elt); if (*begin == '?' || *begin == '+' || *begin == '-' || *begin == '~') { - begin ++; + begin++; } /* Now check what we have */ switch (g_ascii_tolower (*begin)) { - case 'a': - /* all or a */ - if (g_ascii_strncasecmp (begin, SPF_ALL, sizeof (SPF_ALL) - 1) == 0) { - NEW_ADDR (new); - begin += sizeof (SPF_ALL) - 1; - res = parse_spf_all (task, begin, rec, new); - } - else if (g_ascii_strncasecmp (begin, SPF_A, sizeof (SPF_A) - 1) == 0) { - NEW_ADDR (new); - begin += sizeof (SPF_A) - 1; - res = parse_spf_a (task, begin, rec, new); - } - else { - msg_info ("<%s>: spf error for domain %s: bad spf command %s", - task->message_id, rec->sender_domain, begin); - } - break; - case 'i': - /* include or ip4 */ - if (g_ascii_strncasecmp (begin, SPF_IP4, sizeof (SPF_IP4) - 1) == 0) { - NEW_ADDR (new); - begin += sizeof (SPF_IP4) - 1; - res = parse_spf_ip4 (task, begin, rec, new); - } - else if (g_ascii_strncasecmp (begin, SPF_INCLUDE, sizeof (SPF_INCLUDE) - 1) == 0) { - NEW_ADDR (new); - begin += sizeof (SPF_INCLUDE) - 1; - res = parse_spf_include (task, begin, rec, new); - } - else if (g_ascii_strncasecmp (begin, SPF_IP6, sizeof (SPF_IP6) - 1) == 0) { + case 'a': + /* all or a */ + if (g_ascii_strncasecmp (begin, SPF_ALL, + sizeof (SPF_ALL) - 1) == 0) { + NEW_ADDR (new); + begin += sizeof (SPF_ALL) - 1; + res = parse_spf_all (task, begin, rec, new); + } + else if (g_ascii_strncasecmp (begin, SPF_A, + sizeof (SPF_A) - 1) == 0) { + NEW_ADDR (new); + begin += sizeof (SPF_A) - 1; + res = parse_spf_a (task, begin, rec, new); + } + else { + msg_info ("<%s>: spf error for domain %s: bad spf command %s", + task->message_id, rec->sender_domain, begin); + } + break; + case 'i': + /* include or ip4 */ + if (g_ascii_strncasecmp (begin, SPF_IP4, + sizeof (SPF_IP4) - 1) == 0) { + NEW_ADDR (new); + begin += sizeof (SPF_IP4) - 1; + res = parse_spf_ip4 (task, begin, rec, new); + } + else if (g_ascii_strncasecmp (begin, SPF_INCLUDE, + sizeof (SPF_INCLUDE) - 1) == 0) { + NEW_ADDR (new); + begin += sizeof (SPF_INCLUDE) - 1; + res = parse_spf_include (task, begin, rec, new); + } + else if (g_ascii_strncasecmp (begin, SPF_IP6, sizeof (SPF_IP6) - + 1) == 0) { #ifdef HAVE_INET_PTON - NEW_ADDR (new); - begin += sizeof (SPF_IP6) - 1; - res = parse_spf_ip6 (task, begin, rec, new); + NEW_ADDR (new); + begin += sizeof (SPF_IP6) - 1; + res = parse_spf_ip6 (task, begin, rec, new); #else - msg_info ("ignoring ip6 spf command as IPv6 is not supported: %s", begin); - new = NULL; - res = TRUE; - begin += sizeof (SPF_IP6) - 1; + msg_info ( + "ignoring ip6 spf command as IPv6 is not supported: %s", + begin); + new = NULL; + res = TRUE; + begin += sizeof (SPF_IP6) - 1; #endif - } - else { - msg_info ("<%s>: spf error for domain %s: bad spf command %s", - task->message_id, rec->sender_domain, begin); - } - break; - case 'm': - /* mx */ - if (g_ascii_strncasecmp (begin, SPF_MX, sizeof (SPF_MX) - 1) == 0) { - NEW_ADDR (new); - begin += sizeof (SPF_MX) - 1; - res = parse_spf_mx (task, begin, rec, new); - } - else { - msg_info ("<%s>: spf error for domain %s: bad spf command %s", - task->message_id, rec->sender_domain, begin); - } - break; - case 'p': - /* ptr */ - if (g_ascii_strncasecmp (begin, SPF_PTR, sizeof (SPF_PTR) - 1) == 0) { - NEW_ADDR (new); - begin += sizeof (SPF_PTR) - 1; - res = parse_spf_ptr (task, begin, rec, new); - } - else { - msg_info ("<%s>: spf error for domain %s: bad spf command %s", - task->message_id, rec->sender_domain, begin); - } - break; - case 'e': - /* exp or exists */ - if (g_ascii_strncasecmp (begin, SPF_EXP, sizeof (SPF_EXP) - 1) == 0) { - begin += sizeof (SPF_EXP) - 1; - res = parse_spf_exp (task, begin, rec, NULL); - } - else if (g_ascii_strncasecmp (begin, SPF_EXISTS, sizeof (SPF_EXISTS) - 1) == 0) { - NEW_ADDR (new); - begin += sizeof (SPF_EXISTS) - 1; - res = parse_spf_exists (task, begin, rec, new); - } - else { - msg_info ("<%s>: spf error for domain %s: bad spf command %s", - task->message_id, rec->sender_domain, begin); - } - break; - case 'r': - /* redirect */ - if (g_ascii_strncasecmp (begin, SPF_REDIRECT, sizeof (SPF_REDIRECT) - 1) == 0) { - begin += sizeof (SPF_REDIRECT) - 1; - res = parse_spf_redirect (task, begin, rec, NULL); - } - else { - msg_info ("<%s>: spf error for domain %s: bad spf command %s", - task->message_id, rec->sender_domain, begin); - } - break; - case 'v': - if (g_ascii_strncasecmp (begin, "v=spf", sizeof ("v=spf") - 1) == 0) { - /* Skip this element till the end of record */ - while (*begin && !g_ascii_isspace (*begin)) { - begin ++; - } - } - break; - default: + } + else { msg_info ("<%s>: spf error for domain %s: bad spf command %s", - task->message_id, rec->sender_domain, begin); - break; + task->message_id, rec->sender_domain, begin); + } + break; + case 'm': + /* mx */ + if (g_ascii_strncasecmp (begin, SPF_MX, sizeof (SPF_MX) - 1) == 0) { + NEW_ADDR (new); + begin += sizeof (SPF_MX) - 1; + res = parse_spf_mx (task, begin, rec, new); + } + else { + msg_info ("<%s>: spf error for domain %s: bad spf command %s", + task->message_id, rec->sender_domain, begin); + } + break; + case 'p': + /* ptr */ + if (g_ascii_strncasecmp (begin, SPF_PTR, + sizeof (SPF_PTR) - 1) == 0) { + NEW_ADDR (new); + begin += sizeof (SPF_PTR) - 1; + res = parse_spf_ptr (task, begin, rec, new); + } + else { + msg_info ("<%s>: spf error for domain %s: bad spf command %s", + task->message_id, rec->sender_domain, begin); + } + break; + case 'e': + /* exp or exists */ + if (g_ascii_strncasecmp (begin, SPF_EXP, + sizeof (SPF_EXP) - 1) == 0) { + begin += sizeof (SPF_EXP) - 1; + res = parse_spf_exp (task, begin, rec, NULL); + } + else if (g_ascii_strncasecmp (begin, SPF_EXISTS, + sizeof (SPF_EXISTS) - 1) == 0) { + NEW_ADDR (new); + begin += sizeof (SPF_EXISTS) - 1; + res = parse_spf_exists (task, begin, rec, new); + } + else { + msg_info ("<%s>: spf error for domain %s: bad spf command %s", + task->message_id, rec->sender_domain, begin); + } + break; + case 'r': + /* redirect */ + if (g_ascii_strncasecmp (begin, SPF_REDIRECT, + sizeof (SPF_REDIRECT) - 1) == 0) { + begin += sizeof (SPF_REDIRECT) - 1; + res = parse_spf_redirect (task, begin, rec, NULL); + } + else { + msg_info ("<%s>: spf error for domain %s: bad spf command %s", + task->message_id, rec->sender_domain, begin); + } + break; + case 'v': + if (g_ascii_strncasecmp (begin, "v=spf", + sizeof ("v=spf") - 1) == 0) { + /* Skip this element till the end of record */ + while (*begin && !g_ascii_isspace (*begin)) { + begin++; + } + } + break; + default: + msg_info ("<%s>: spf error for domain %s: bad spf command %s", + task->message_id, rec->sender_domain, begin); + break; } if (res) { if (new != NULL) { rec->addrs = g_list_prepend (rec->addrs, new); } - rec->elt_num ++; + rec->elt_num++; } } @@ -1325,14 +1439,16 @@ parse_spf_record (struct rspamd_task *task, struct spf_record *rec) static void parse_spf_scopes (struct spf_record *rec, gchar **begin) { - for (;;) { - if (g_ascii_strncasecmp (*begin, SPF_SCOPE_PRA, sizeof (SPF_SCOPE_PRA) - 1) == 0) { + for (;; ) { + if (g_ascii_strncasecmp (*begin, SPF_SCOPE_PRA, sizeof (SPF_SCOPE_PRA) - + 1) == 0) { *begin += sizeof (SPF_SCOPE_PRA) - 1; /* XXX: Implement actual PRA check */ /* extract_pra_info (rec); */ continue; } - else if (g_ascii_strncasecmp (*begin, SPF_SCOPE_MFROM, sizeof (SPF_SCOPE_MFROM) - 1) == 0) { + else if (g_ascii_strncasecmp (*begin, SPF_SCOPE_MFROM, + sizeof (SPF_SCOPE_MFROM) - 1) == 0) { /* mfrom is standart spf1 check */ *begin += sizeof (SPF_SCOPE_MFROM) - 1; continue; @@ -1340,7 +1456,7 @@ parse_spf_scopes (struct spf_record *rec, gchar **begin) else if (**begin != ',') { break; } - (*begin) ++; + (*begin)++; } } @@ -1349,47 +1465,51 @@ start_spf_parse (struct spf_record *rec, gchar *begin, guint ttl) { /* Skip spaces */ while (g_ascii_isspace (*begin)) { - begin ++; + begin++; } - if (g_ascii_strncasecmp (begin, SPF_VER1_STR, sizeof (SPF_VER1_STR) - 1) == 0) { + if (g_ascii_strncasecmp (begin, SPF_VER1_STR, + sizeof (SPF_VER1_STR) - 1) == 0) { begin += sizeof (SPF_VER1_STR) - 1; while (g_ascii_isspace (*begin) && *begin) { - begin ++; + begin++; } rec->elts = g_strsplit_set (begin, " ", 0); rec->elt_num = 0; if (rec->elts) { - rspamd_mempool_add_destructor (rec->task->task_pool, (rspamd_mempool_destruct_t)g_strfreev, rec->elts); + rspamd_mempool_add_destructor (rec->task->task_pool, + (rspamd_mempool_destruct_t)g_strfreev, rec->elts); rec->cur_elt = rec->elts[0]; - while (parse_spf_record (rec->task, rec)); + while (parse_spf_record (rec->task, rec)) ; if (ttl != 0) { rec->ttl = ttl; } return TRUE; } } - else if (g_ascii_strncasecmp (begin, SPF_VER2_STR, sizeof (SPF_VER2_STR) - 1) == 0) { + else if (g_ascii_strncasecmp (begin, SPF_VER2_STR, sizeof (SPF_VER2_STR) - + 1) == 0) { /* Skip one number of record, so no we are here spf2.0/ */ begin += sizeof (SPF_VER2_STR); if (*begin != '/') { msg_info ("<%s>: spf error for domain %s: sender id is invalid", - rec->task->message_id, rec->sender_domain); + rec->task->message_id, rec->sender_domain); } else { - begin ++; + begin++; parse_spf_scopes (rec, &begin); } /* Now common spf record */ while (g_ascii_isspace (*begin) && *begin) { - begin ++; + begin++; } rec->elts = g_strsplit_set (begin, " ", 0); rec->elt_num = 0; if (rec->elts) { - rspamd_mempool_add_destructor (rec->task->task_pool, (rspamd_mempool_destruct_t)g_strfreev, rec->elts); + rspamd_mempool_add_destructor (rec->task->task_pool, + (rspamd_mempool_destruct_t)g_strfreev, rec->elts); rec->cur_elt = rec->elts[0]; - while (parse_spf_record (rec->task, rec)); + while (parse_spf_record (rec->task, rec)) ; if (ttl != 0) { rec->ttl = ttl; } @@ -1398,7 +1518,10 @@ start_spf_parse (struct spf_record *rec, gchar *begin, guint ttl) } else { msg_debug ("<%s>: spf error for domain %s: bad spf record version: %*s", - rec->task->message_id, rec->sender_domain, sizeof (SPF_VER1_STR) - 1, begin); + rec->task->message_id, + rec->sender_domain, + sizeof (SPF_VER1_STR) - 1, + begin); } return FALSE; } @@ -1409,9 +1532,10 @@ spf_dns_callback (struct rdns_reply *reply, gpointer arg) struct spf_record *rec = arg; struct rdns_reply_entry *elt; - rec->requests_inflight --; + rec->requests_inflight--; if (reply->code == RDNS_RC_NOERROR) { - LL_FOREACH (reply->entries, elt) { + LL_FOREACH (reply->entries, elt) + { if (start_spf_parse (rec, elt->content.txt.data, elt->ttl)) { break; } @@ -1426,10 +1550,11 @@ spf_dns_callback (struct rdns_reply *reply, gpointer arg) gchar * get_spf_domain (struct rspamd_task *task) { - gchar *domain, *res = NULL; - GList *domains; + gchar *domain, *res = NULL; + GList *domains; - if (task->from && (domain = strchr (task->from, '@')) != NULL && *domain == '@') { + if (task->from && + (domain = strchr (task->from, '@')) != NULL && *domain == '@') { res = rspamd_mempool_strdup (task->task_pool, domain + 1); if ((domain = strchr (res, '>')) != NULL) { *domain = '\0'; @@ -1437,7 +1562,10 @@ get_spf_domain (struct rspamd_task *task) } else { /* Extract from header */ - domains = message_get_header (task->task_pool, task->message, "From", FALSE); + domains = message_get_header (task->task_pool, + task->message, + "From", + FALSE); if (domains != NULL) { res = rspamd_mempool_strdup (task->task_pool, domains->data); @@ -1461,24 +1589,28 @@ get_spf_domain (struct rspamd_task *task) gboolean resolve_spf (struct rspamd_task *task, spf_cb_t callback) { - struct spf_record *rec; - gchar *domain; - GList *domains; + struct spf_record *rec; + gchar *domain; + GList *domains; rec = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct spf_record)); rec->task = task; rec->callback = callback; /* Add destructor */ - rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)spf_record_destructor, rec); + rspamd_mempool_add_destructor (task->task_pool, + (rspamd_mempool_destruct_t)spf_record_destructor, + rec); /* Extract from data */ - if (task->from && (domain = strchr (task->from, '@')) != NULL && *domain == '@') { + if (task->from && + (domain = strchr (task->from, '@')) != NULL && *domain == '@') { rec->sender = task->from; rec->local_part = rspamd_mempool_strdup (task->task_pool, task->from); *(rec->local_part + (domain - task->from)) = '\0'; if (*rec->local_part == '<') { - memmove (rec->local_part, rec->local_part + 1, strlen (rec->local_part)); + memmove (rec->local_part, rec->local_part + 1, + strlen (rec->local_part)); } rec->cur_domain = rspamd_mempool_strdup (task->task_pool, domain + 1); if ((domain = strchr (rec->cur_domain, '>')) != NULL) { @@ -1486,25 +1618,31 @@ resolve_spf (struct rspamd_task *task, spf_cb_t callback) } rec->sender_domain = rec->cur_domain; - if (make_dns_request (task->resolver, task->s, task->task_pool, spf_dns_callback, - (void *)rec, RDNS_REQUEST_TXT, rec->cur_domain)) { - task->dns_requests ++; - rec->requests_inflight ++; + if (make_dns_request (task->resolver, task->s, task->task_pool, + spf_dns_callback, + (void *)rec, RDNS_REQUEST_TXT, rec->cur_domain)) { + task->dns_requests++; + rec->requests_inflight++; return TRUE; } } else { /* Extract from header */ - domains = message_get_header (task->task_pool, task->message, "From", FALSE); + domains = message_get_header (task->task_pool, + task->message, + "From", + FALSE); if (domains != NULL) { - rec->cur_domain = rspamd_mempool_strdup (task->task_pool, domains->data); + rec->cur_domain = rspamd_mempool_strdup (task->task_pool, + domains->data); g_list_free (domains); if ((domain = strrchr (rec->cur_domain, '@')) == NULL) { return FALSE; } - rec->sender = rspamd_mempool_strdup (task->task_pool, rec->cur_domain); + rec->sender = rspamd_mempool_strdup (task->task_pool, + rec->cur_domain); rec->local_part = rec->cur_domain; *domain = '\0'; rec->cur_domain = domain + 1; @@ -1518,9 +1656,10 @@ resolve_spf (struct rspamd_task *task, spf_cb_t callback) } rec->sender_domain = rec->cur_domain; if (make_dns_request (task->resolver, task->s, task->task_pool, - spf_dns_callback, (void *)rec, RDNS_REQUEST_TXT, rec->cur_domain)) { - task->dns_requests ++; - rec->requests_inflight ++; + spf_dns_callback, (void *)rec, RDNS_REQUEST_TXT, + rec->cur_domain)) { + task->dns_requests++; + rec->requests_inflight++; return TRUE; } } @@ -1529,6 +1668,6 @@ resolve_spf (struct rspamd_task *task, spf_cb_t callback) return FALSE; } -/* - * vi:ts=4 +/* + * vi:ts=4 */ diff --git a/src/libserver/spf.h b/src/libserver/spf.h index 94c613e42..18d8229f8 100644 --- a/src/libserver/spf.h +++ b/src/libserver/spf.h @@ -78,7 +78,7 @@ gboolean resolve_spf (struct rspamd_task *task, spf_cb_t callback); /* * Get a domain for spf for specified task */ -gchar *get_spf_domain (struct rspamd_task *task); +gchar * get_spf_domain (struct rspamd_task *task); #endif diff --git a/src/libserver/statfile.c b/src/libserver/statfile.c index a993fd116..b0db2bc4c 100644 --- a/src/libserver/statfile.c +++ b/src/libserver/statfile.c @@ -24,8 +24,8 @@ #include "config.h" -#include "statfile.h" #include "main.h" +#include "statfile.h" #define RSPAMD_STATFILE_VERSION {'1', '2'} #define BACKUP_SUFFIX ".old" @@ -33,33 +33,33 @@ /* Maximum number of statistics files */ #define STATFILES_MAX 255 static void statfile_pool_set_block_common ( - statfile_pool_t * pool, stat_file_t * file, - guint32 h1, guint32 h2, - time_t t, double value, - gboolean from_now); + statfile_pool_t * pool, stat_file_t * file, + guint32 h1, guint32 h2, + time_t t, double value, + gboolean from_now); static gint cmpstatfile (const void *a, const void *b) { - const stat_file_t *s1 = a, *s2 = b; + const stat_file_t *s1 = a, *s2 = b; return g_ascii_strcasecmp (s1->filename, s2->filename); } /* Convert statfile version 1.0 to statfile version 1.2, saving backup */ struct stat_file_header_10 { - u_char magic[3]; /**< magic signature ('r' 's' 'd') */ - u_char version[2]; /**< version of statfile */ - u_char padding[3]; /**< padding */ - guint64 create_time; /**< create time (time_t->guint64) */ + u_char magic[3]; /**< magic signature ('r' 's' 'd') */ + u_char version[2]; /**< version of statfile */ + u_char padding[3]; /**< padding */ + guint64 create_time; /**< create time (time_t->guint64) */ }; static gboolean convert_statfile_10 (stat_file_t * file) { - gchar *backup_name; + gchar *backup_name; struct stat st; - struct stat_file_header header = { + struct stat_file_header header = { .magic = {'r', 's', 'd'}, .version = RSPAMD_STATFILE_VERSION, .padding = {0, 0, 0}, @@ -70,9 +70,12 @@ convert_statfile_10 (stat_file_t * file) /* Format backup name */ backup_name = g_strdup_printf ("%s.%s", file->filename, BACKUP_SUFFIX); - - msg_info ("convert old statfile %s to version %c.%c, backup in %s", file->filename, - header.version[0], header.version[1], backup_name); + + msg_info ("convert old statfile %s to version %c.%c, backup in %s", + file->filename, + header.version[0], + header.version[1], + backup_name); if (stat (backup_name, &st) != -1) { msg_info ("replace old %s", backup_name); @@ -85,31 +88,51 @@ convert_statfile_10 (stat_file_t * file) /* XXX: maybe race condition here */ unlock_file (file->fd, FALSE); close (file->fd); - if ((file->fd = open (file->filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { - msg_info ("cannot create file %s, error %d, %s", file->filename, errno, strerror (errno)); + if ((file->fd = + open (file->filename, O_RDWR | O_TRUNC | O_CREAT, + S_IWUSR | S_IRUSR)) == -1) { + msg_info ("cannot create file %s, error %d, %s", + file->filename, + errno, + strerror (errno)); return FALSE; } lock_file (file->fd, FALSE); /* Now make new header and copy it to new file */ if (write (file->fd, &header, sizeof (header)) == -1) { - msg_info ("cannot write to file %s, error %d, %s", file->filename, errno, strerror (errno)); + msg_info ("cannot write to file %s, error %d, %s", + file->filename, + errno, + strerror (errno)); return FALSE; } /* Now write old map to new file */ - if (write (file->fd, ((u_char *)file->map + sizeof (struct stat_file_header_10)), - file->len - sizeof (struct stat_file_header_10)) == -1) { - msg_info ("cannot write to file %s, error %d, %s", file->filename, errno, strerror (errno)); + if (write (file->fd, + ((u_char *)file->map + sizeof (struct stat_file_header_10)), + file->len - sizeof (struct stat_file_header_10)) == -1) { + msg_info ("cannot write to file %s, error %d, %s", + file->filename, + errno, + strerror (errno)); return FALSE; } /* Unmap old memory and map new */ munmap (file->map, file->len); - file->len = file->len + sizeof (struct stat_file_header) - sizeof (struct stat_file_header_10); + file->len = file->len + sizeof (struct stat_file_header) - + sizeof (struct stat_file_header_10); #ifdef HAVE_MMAP_NOCORE - if ((file->map = mmap (NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_NOCORE, file->fd, 0)) == MAP_FAILED) { + if ((file->map = + mmap (NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_NOCORE, + file->fd, 0)) == MAP_FAILED) { #else - if ((file->map = mmap (NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0)) == MAP_FAILED) { + if ((file->map = + mmap (NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, + 0)) == MAP_FAILED) { #endif - msg_info ("cannot mmap file %s, error %d, %s", file->filename, errno, strerror (errno)); + msg_info ("cannot mmap file %s, error %d, %s", + file->filename, + errno, + strerror (errno)); return FALSE; } @@ -120,9 +143,9 @@ convert_statfile_10 (stat_file_t * file) static gint statfile_pool_check (stat_file_t * file) { - struct stat_file *f; - gchar *c; - static gchar valid_version[] = RSPAMD_STATFILE_VERSION; + struct stat_file *f; + gchar *c; + static gchar valid_version[] = RSPAMD_STATFILE_VERSION; if (!file || !file->map) { @@ -130,7 +153,9 @@ statfile_pool_check (stat_file_t * file) } if (file->len < sizeof (struct stat_file)) { - msg_info ("file %s is too short to be stat file: %z", file->filename, file->len); + msg_info ("file %s is too short to be stat file: %z", + file->filename, + file->len); return -1; } @@ -150,31 +175,40 @@ statfile_pool_check (stat_file_t * file) } else if (memcmp (c, valid_version, sizeof (valid_version)) != 0) { /* Unknown version */ - msg_info ("file %s has invalid version %c.%c", file->filename, '0' + *c, '0' + *(c + 1)); + msg_info ("file %s has invalid version %c.%c", + file->filename, + '0' + *c, + '0' + *(c + 1)); return -1; } /* Check first section and set new offset */ file->cur_section.code = f->section.code; file->cur_section.length = f->section.length; - if (file->cur_section.length * sizeof (struct stat_file_block) > file->len) { - msg_info ("file %s is truncated: %z, must be %z", file->filename, file->len, file->cur_section.length * sizeof (struct stat_file_block)); + if (file->cur_section.length * sizeof (struct stat_file_block) > + file->len) { + msg_info ("file %s is truncated: %z, must be %z", + file->filename, + file->len, + file->cur_section.length * sizeof (struct stat_file_block)); return -1; } - file->seek_pos = sizeof (struct stat_file) - sizeof (struct stat_file_block); + file->seek_pos = sizeof (struct stat_file) - + sizeof (struct stat_file_block); return 0; } -statfile_pool_t * +statfile_pool_t * statfile_pool_new (rspamd_mempool_t *pool, gboolean use_mlock) { - statfile_pool_t *new; + statfile_pool_t *new; new = rspamd_mempool_alloc0 (pool, sizeof (statfile_pool_t)); new->pool = rspamd_mempool_new (rspamd_mempool_suggest_size ()); - new->files = rspamd_mempool_alloc0 (new->pool, STATFILES_MAX * sizeof (stat_file_t)); + new->files = + rspamd_mempool_alloc0 (new->pool, STATFILES_MAX * sizeof (stat_file_t)); new->lock = rspamd_mempool_get_mutex (new->pool); new->mlock_ok = use_mlock; @@ -182,18 +216,24 @@ statfile_pool_new (rspamd_mempool_t *pool, gboolean use_mlock) } static stat_file_t * -statfile_pool_reindex (statfile_pool_t * pool, gchar *filename, size_t old_size, size_t size) +statfile_pool_reindex (statfile_pool_t * pool, + gchar *filename, + size_t old_size, + size_t size) { - gchar *backup; - gint fd; - stat_file_t *new; - u_char *map, *pos; - struct stat_file_block *block; - struct stat_file_header *header; + gchar *backup; + gint fd; + stat_file_t *new; + u_char *map, *pos; + struct stat_file_block *block; + struct stat_file_header *header; if (size < - sizeof (struct stat_file_header) + sizeof (struct stat_file_section) + sizeof (block)) { - msg_err ("file %s is too small to carry any statistic: %z", filename, size); + sizeof (struct stat_file_header) + sizeof (struct stat_file_section) + + sizeof (block)) { + msg_err ("file %s is too small to carry any statistic: %z", + filename, + size); return NULL; } @@ -202,7 +242,8 @@ statfile_pool_reindex (statfile_pool_t * pool, gchar *filename, size_t old_size, backup = g_strconcat (filename, ".old", NULL); if (rename (filename, backup) == -1) { - msg_err ("cannot rename %s to %s: %s", filename, backup, strerror (errno)); + msg_err ("cannot rename %s to %s: %s", filename, backup, strerror ( + errno)); g_free (backup); rspamd_mempool_unlock_mutex (pool->lock); return NULL; @@ -227,7 +268,8 @@ statfile_pool_reindex (statfile_pool_t * pool, gchar *filename, size_t old_size, } /* Now start reading blocks from old statfile */ - if ((map = mmap (NULL, old_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { + if ((map = + mmap (NULL, old_size, PROT_READ, MAP_SHARED, fd, 0)) == MAP_FAILED) { msg_err ("cannot mmap file: %s", strerror (errno)); close (fd); g_free (backup); @@ -238,14 +280,20 @@ statfile_pool_reindex (statfile_pool_t * pool, gchar *filename, size_t old_size, while (old_size - (pos - map) >= sizeof (struct stat_file_block)) { block = (struct stat_file_block *)pos; if (block->hash1 != 0 && block->value != 0) { - statfile_pool_set_block_common (pool, new, block->hash1, block->hash2, 0, block->value, FALSE); + statfile_pool_set_block_common (pool, + new, + block->hash1, + block->hash2, + 0, + block->value, + FALSE); } pos += sizeof (block); } header = (struct stat_file_header *)map; statfile_set_revision (new, header->revision, header->rev_time); - + munmap (map, old_size); close (fd); unlink (backup); @@ -261,9 +309,9 @@ statfile_pool_reindex (statfile_pool_t * pool, gchar *filename, size_t old_size, static void statfile_preload (stat_file_t *file) { - guint8 *pos, *end; - volatile guint8 t; - gsize size; + guint8 *pos, *end; + volatile guint8 t; + gsize size; pos = (guint8 *)file->map; end = (guint8 *)file->map + file->len; @@ -286,50 +334,66 @@ statfile_preload (stat_file_t *file) } } -stat_file_t * -statfile_pool_open (statfile_pool_t * pool, gchar *filename, size_t size, gboolean forced) +stat_file_t * +statfile_pool_open (statfile_pool_t * pool, + gchar *filename, + size_t size, + gboolean forced) { - struct stat st; - stat_file_t *new_file; + struct stat st; + stat_file_t *new_file; if ((new_file = statfile_pool_is_open (pool, filename)) != NULL) { return new_file; } if (pool->opened >= STATFILES_MAX - 1) { - msg_err ("reached hard coded limit of statfiles opened: %d", STATFILES_MAX); + msg_err ("reached hard coded limit of statfiles opened: %d", + STATFILES_MAX); return NULL; } if (stat (filename, &st) == -1) { - msg_info ("cannot stat file %s, error %s, %d", filename, strerror (errno), errno); + msg_info ("cannot stat file %s, error %s, %d", filename, strerror ( + errno), errno); return NULL; } rspamd_mempool_lock_mutex (pool->lock); - if (!forced && labs (size - st.st_size) > (long)sizeof (struct stat_file) * 2 - && size > sizeof (struct stat_file)) { + if (!forced && + labs (size - st.st_size) > (long)sizeof (struct stat_file) * 2 + && size > sizeof (struct stat_file)) { rspamd_mempool_unlock_mutex (pool->lock); - msg_warn ("need to reindex statfile old size: %Hz, new size: %Hz", (size_t)st.st_size, size); + msg_warn ("need to reindex statfile old size: %Hz, new size: %Hz", + (size_t)st.st_size, size); return statfile_pool_reindex (pool, filename, st.st_size, size); } else if (size < sizeof (struct stat_file)) { - msg_err ("requested to shrink statfile to %Hz but it is too small", size); + msg_err ("requested to shrink statfile to %Hz but it is too small", + size); } new_file = &pool->files[pool->opened++]; bzero (new_file, sizeof (stat_file_t)); if ((new_file->fd = open (filename, O_RDWR)) == -1) { - msg_info ("cannot open file %s, error %d, %s", filename, errno, strerror (errno)); + msg_info ("cannot open file %s, error %d, %s", + filename, + errno, + strerror (errno)); rspamd_mempool_unlock_mutex (pool->lock); pool->opened--; return NULL; } - if ((new_file->map = mmap (NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, new_file->fd, 0)) == MAP_FAILED) { + if ((new_file->map = + mmap (NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, + new_file->fd, 0)) == MAP_FAILED) { close (new_file->fd); rspamd_mempool_unlock_mutex (pool->lock); - msg_info ("cannot mmap file %s, error %d, %s", filename, errno, strerror (errno)); + msg_info ("cannot mmap file %s, error %d, %s", + filename, + errno, + strerror (errno)); pool->opened--; return NULL; @@ -340,7 +404,9 @@ statfile_pool_open (statfile_pool_t * pool, gchar *filename, size_t size, gboole /* Try to lock pages in RAM */ if (pool->mlock_ok) { if (mlock (new_file->map, new_file->len) == -1) { - msg_warn ("mlock of statfile failed, maybe you need to increase RLIMIT_MEMLOCK limit for a process: %s", strerror (errno)); + msg_warn ( + "mlock of statfile failed, maybe you need to increase RLIMIT_MEMLOCK limit for a process: %s", + strerror (errno)); pool->mlock_ok = FALSE; } } @@ -350,7 +416,7 @@ statfile_pool_open (statfile_pool_t * pool, gchar *filename, size_t size, gboole pool->opened--; rspamd_mempool_unlock_mutex (pool->lock); unlock_file (new_file->fd, FALSE); - munmap (new_file->map, st.st_size); + munmap (new_file->map, st.st_size); return NULL; } unlock_file (new_file->fd, FALSE); @@ -367,9 +433,11 @@ statfile_pool_open (statfile_pool_t * pool, gchar *filename, size_t size, gboole } gint -statfile_pool_close (statfile_pool_t * pool, stat_file_t * file, gboolean keep_sorted) +statfile_pool_close (statfile_pool_t * pool, + stat_file_t * file, + gboolean keep_sorted) { - stat_file_t *pos; + stat_file_t *pos; if ((pos = statfile_pool_is_open (pool, file->filename)) == NULL) { msg_info ("file %s is not opened", file->filename); @@ -388,7 +456,7 @@ statfile_pool_close (statfile_pool_t * pool, stat_file_t * file, gboolean keep_s } /* Move the remain statfiles */ memmove (pos, ((guint8 *)pos) + sizeof (stat_file_t), - (--pool->opened - (pos - pool->files)) * sizeof (stat_file_t)); + (--pool->opened - (pos - pool->files)) * sizeof (stat_file_t)); rspamd_mempool_unlock_mutex (pool->lock); @@ -398,7 +466,7 @@ statfile_pool_close (statfile_pool_t * pool, stat_file_t * file, gboolean keep_s gint statfile_pool_create (statfile_pool_t * pool, gchar *filename, size_t size) { - struct stat_file_header header = { + struct stat_file_header header = { .magic = {'r', 's', 'd'}, .version = RSPAMD_STATFILE_VERSION, .padding = {0, 0, 0}, @@ -406,13 +474,13 @@ statfile_pool_create (statfile_pool_t * pool, gchar *filename, size_t size) .rev_time = 0, .used_blocks = 0 }; - struct stat_file_section section = { + struct stat_file_section section = { .code = STATFILE_SECTION_COMMON, }; - struct stat_file_block block = { 0, 0, 0 }; - gint fd; - guint buflen = 0, nblocks; - gchar *buf = NULL; + struct stat_file_block block = { 0, 0, 0 }; + gint fd; + guint buflen = 0, nblocks; + gchar *buf = NULL; if (statfile_pool_is_open (pool, filename) != NULL) { msg_info ("file %s is already opened", filename); @@ -420,26 +488,40 @@ statfile_pool_create (statfile_pool_t * pool, gchar *filename, size_t size) } if (size < - sizeof (struct stat_file_header) + sizeof (struct stat_file_section) + sizeof (block)) { - msg_err ("file %s is too small to carry any statistic: %z", filename, size); + sizeof (struct stat_file_header) + sizeof (struct stat_file_section) + + sizeof (block)) { + msg_err ("file %s is too small to carry any statistic: %z", + filename, + size); return -1; } rspamd_mempool_lock_mutex (pool->lock); - nblocks = (size - sizeof (struct stat_file_header) - sizeof (struct stat_file_section)) / sizeof (struct stat_file_block); + nblocks = + (size - sizeof (struct stat_file_header) - + sizeof (struct stat_file_section)) / sizeof (struct stat_file_block); header.total_blocks = nblocks; - if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { - msg_info ("cannot create file %s, error %d, %s", filename, errno, strerror (errno)); + if ((fd = + open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { + msg_info ("cannot create file %s, error %d, %s", + filename, + errno, + strerror (errno)); rspamd_mempool_unlock_mutex (pool->lock); return -1; } - rspamd_fallocate (fd, 0, sizeof (header) + sizeof (section) + sizeof (block) * nblocks); + rspamd_fallocate (fd, + 0, + sizeof (header) + sizeof (section) + sizeof (block) * nblocks); header.create_time = (guint64) time (NULL); if (write (fd, &header, sizeof (header)) == -1) { - msg_info ("cannot write header to file %s, error %d, %s", filename, errno, strerror (errno)); + msg_info ("cannot write header to file %s, error %d, %s", + filename, + errno, + strerror (errno)); close (fd); rspamd_mempool_unlock_mutex (pool->lock); return -1; @@ -447,12 +529,15 @@ statfile_pool_create (statfile_pool_t * pool, gchar *filename, size_t size) section.length = (guint64) nblocks; if (write (fd, §ion, sizeof (section)) == -1) { - msg_info ("cannot write section header to file %s, error %d, %s", filename, errno, strerror (errno)); + msg_info ("cannot write section header to file %s, error %d, %s", + filename, + errno, + strerror (errno)); close (fd); rspamd_mempool_unlock_mutex (pool->lock); return -1; } - + /* Buffer for write 256 blocks at once */ if (nblocks > 256) { buflen = sizeof (block) * 256; @@ -463,7 +548,10 @@ statfile_pool_create (statfile_pool_t * pool, gchar *filename, size_t size) if (nblocks > 256) { /* Just write buffer */ if (write (fd, buf, buflen) == -1) { - msg_info ("cannot write blocks buffer to file %s, error %d, %s", filename, errno, strerror (errno)); + msg_info ("cannot write blocks buffer to file %s, error %d, %s", + filename, + errno, + strerror (errno)); close (fd); rspamd_mempool_unlock_mutex (pool->lock); g_free (buf); @@ -473,7 +561,10 @@ statfile_pool_create (statfile_pool_t * pool, gchar *filename, size_t size) } else { if (write (fd, &block, sizeof (block)) == -1) { - msg_info ("cannot write block to file %s, error %d, %s", filename, errno, strerror (errno)); + msg_info ("cannot write block to file %s, error %d, %s", + filename, + errno, + strerror (errno)); close (fd); if (buf) { g_free (buf); @@ -481,7 +572,7 @@ statfile_pool_create (statfile_pool_t * pool, gchar *filename, size_t size) rspamd_mempool_unlock_mutex (pool->lock); return -1; } - nblocks --; + nblocks--; } } @@ -498,7 +589,7 @@ statfile_pool_create (statfile_pool_t * pool, gchar *filename, size_t size) void statfile_pool_delete (statfile_pool_t * pool) { - gint i; + gint i; for (i = 0; i < pool->opened; i++) { statfile_pool_close (pool, &pool->files[i], FALSE); @@ -521,11 +612,15 @@ statfile_pool_unlock_file (statfile_pool_t * pool, stat_file_t * file) } double -statfile_pool_get_block (statfile_pool_t * pool, stat_file_t * file, guint32 h1, guint32 h2, time_t now) +statfile_pool_get_block (statfile_pool_t * pool, + stat_file_t * file, + guint32 h1, + guint32 h2, + time_t now) { - struct stat_file_block *block; - guint i, blocknum; - u_char *c; + struct stat_file_block *block; + guint i, blocknum; + u_char *c; file->access_time = now; @@ -534,7 +629,8 @@ statfile_pool_get_block (statfile_pool_t * pool, stat_file_t * file, guint32 h1, } blocknum = h1 % file->cur_section.length; - c = (u_char *) file->map + file->seek_pos + blocknum * sizeof (struct stat_file_block); + c = (u_char *) file->map + file->seek_pos + blocknum * + sizeof (struct stat_file_block); block = (struct stat_file_block *)c; for (i = 0; i < CHAIN_LENGTH; i++) { @@ -553,13 +649,19 @@ statfile_pool_get_block (statfile_pool_t * pool, stat_file_t * file, guint32 h1, } static void -statfile_pool_set_block_common (statfile_pool_t * pool, stat_file_t * file, guint32 h1, guint32 h2, time_t t, double value, gboolean from_now) +statfile_pool_set_block_common (statfile_pool_t * pool, + stat_file_t * file, + guint32 h1, + guint32 h2, + time_t t, + double value, + gboolean from_now) { - struct stat_file_block *block, *to_expire = NULL; - struct stat_file_header *header; - guint i, blocknum; - u_char *c; - double min = G_MAXDOUBLE; + struct stat_file_block *block, *to_expire = NULL; + struct stat_file_header *header; + guint i, blocknum; + u_char *c; + double min = G_MAXDOUBLE; if (from_now) { file->access_time = t; @@ -570,13 +672,16 @@ statfile_pool_set_block_common (statfile_pool_t * pool, stat_file_t * file, guin blocknum = h1 % file->cur_section.length; header = (struct stat_file_header *)file->map; - c = (u_char *) file->map + file->seek_pos + blocknum * sizeof (struct stat_file_block); + c = (u_char *) file->map + file->seek_pos + blocknum * + sizeof (struct stat_file_block); block = (struct stat_file_block *)c; for (i = 0; i < CHAIN_LENGTH; i++) { if (i + blocknum >= file->cur_section.length) { /* Need to expire some block in chain */ - msg_info ("chain %ud is full in statfile %s, starting expire", blocknum, file->filename); + msg_info ("chain %ud is full in statfile %s, starting expire", + blocknum, + file->filename); break; } /* First try to find block in chain */ @@ -587,15 +692,19 @@ statfile_pool_set_block_common (statfile_pool_t * pool, stat_file_t * file, guin /* Check whether we have a free block in chain */ if (block->hash1 == 0 && block->hash2 == 0) { /* Write new block here */ - msg_debug ("found free block %ud in chain %ud, set h1=%ud, h2=%ud", i, blocknum, h1, h2); + msg_debug ("found free block %ud in chain %ud, set h1=%ud, h2=%ud", + i, + blocknum, + h1, + h2); block->hash1 = h1; block->hash2 = h2; block->value = value; - header->used_blocks ++; + header->used_blocks++; return; } - + /* Expire block with minimum value otherwise */ if (block->value < min) { to_expire = block; @@ -611,7 +720,8 @@ statfile_pool_set_block_common (statfile_pool_t * pool, stat_file_t * file, guin } else { /* Expire first block in chain */ - c = (u_char *) file->map + file->seek_pos + blocknum * sizeof (struct stat_file_block); + c = (u_char *) file->map + file->seek_pos + blocknum * + sizeof (struct stat_file_block); block = (struct stat_file_block *)c; } @@ -621,17 +731,26 @@ statfile_pool_set_block_common (statfile_pool_t * pool, stat_file_t * file, guin } void -statfile_pool_set_block (statfile_pool_t * pool, stat_file_t * file, guint32 h1, guint32 h2, time_t now, double value) +statfile_pool_set_block (statfile_pool_t * pool, + stat_file_t * file, + guint32 h1, + guint32 h2, + time_t now, + double value) { statfile_pool_set_block_common (pool, file, h1, h2, now, value, TRUE); } -stat_file_t * +stat_file_t * statfile_pool_is_open (statfile_pool_t * pool, gchar *filename) { - static stat_file_t f, *ret; + static stat_file_t f, *ret; rspamd_strlcpy (f.filename, filename, sizeof (f.filename)); - ret = lfind (&f, pool->files, (size_t *)&pool->opened, sizeof (stat_file_t), cmpstatfile); + ret = lfind (&f, + pool->files, + (size_t *)&pool->opened, + sizeof (stat_file_t), + cmpstatfile); return ret; } @@ -643,10 +762,13 @@ statfile_pool_get_section (statfile_pool_t * pool, stat_file_t * file) } gboolean -statfile_pool_set_section (statfile_pool_t * pool, stat_file_t * file, guint32 code, gboolean from_begin) +statfile_pool_set_section (statfile_pool_t * pool, + stat_file_t * file, + guint32 code, + gboolean from_begin) { - struct stat_file_section *sec; - off_t cur_offset; + struct stat_file_section *sec; + off_t cur_offset; /* Try to find section */ @@ -671,13 +793,19 @@ statfile_pool_set_section (statfile_pool_t * pool, stat_file_t * file, guint32 c } gboolean -statfile_pool_add_section (statfile_pool_t * pool, stat_file_t * file, guint32 code, guint64 length) +statfile_pool_add_section (statfile_pool_t * pool, + stat_file_t * file, + guint32 code, + guint64 length) { - struct stat_file_section sect; - struct stat_file_block block = { 0, 0, 0 }; + struct stat_file_section sect; + struct stat_file_block block = { 0, 0, 0 }; if (lseek (file->fd, 0, SEEK_END) == -1) { - msg_info ("cannot lseek file %s, error %d, %s", file->filename, errno, strerror (errno)); + msg_info ("cannot lseek file %s, error %d, %s", + file->filename, + errno, + strerror (errno)); return FALSE; } @@ -685,13 +813,19 @@ statfile_pool_add_section (statfile_pool_t * pool, stat_file_t * file, guint32 c sect.length = length; if (write (file->fd, §, sizeof (sect)) == -1) { - msg_info ("cannot write block to file %s, error %d, %s", file->filename, errno, strerror (errno)); + msg_info ("cannot write block to file %s, error %d, %s", + file->filename, + errno, + strerror (errno)); return FALSE; } while (length--) { if (write (file->fd, &block, sizeof (block)) == -1) { - msg_info ("cannot write block to file %s, error %d, %s", file->filename, errno, strerror (errno)); + msg_info ("cannot write block to file %s, error %d, %s", + file->filename, + errno, + strerror (errno)); return FALSE; } } @@ -702,8 +836,13 @@ statfile_pool_add_section (statfile_pool_t * pool, stat_file_t * file, guint32 c fsync (file->fd); file->len += length; - if ((file->map = mmap (NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, 0)) == NULL) { - msg_info ("cannot mmap file %s, error %d, %s", file->filename, errno, strerror (errno)); + if ((file->map = + mmap (NULL, file->len, PROT_READ | PROT_WRITE, MAP_SHARED, file->fd, + 0)) == NULL) { + msg_info ("cannot mmap file %s, error %d, %s", + file->filename, + errno, + strerror (errno)); return FALSE; } statfile_pool_unlock_file (pool, file); @@ -731,15 +870,15 @@ statfile_get_section_by_name (const gchar *name) return 0; } -gboolean +gboolean statfile_set_revision (stat_file_t *file, guint64 rev, time_t time) { - struct stat_file_header *header; + struct stat_file_header *header; if (file == NULL || file->map == NULL) { return FALSE; } - + header = (struct stat_file_header *)file->map; header->revision = rev; @@ -748,10 +887,10 @@ statfile_set_revision (stat_file_t *file, guint64 rev, time_t time) return TRUE; } -gboolean +gboolean statfile_inc_revision (stat_file_t *file) { - struct stat_file_header *header; + struct stat_file_header *header; if (file == NULL || file->map == NULL) { return FALSE; @@ -759,7 +898,7 @@ statfile_inc_revision (stat_file_t *file) header = (struct stat_file_header *)file->map; - header->revision ++; + header->revision++; return TRUE; } @@ -767,12 +906,12 @@ statfile_inc_revision (stat_file_t *file) gboolean statfile_get_revision (stat_file_t *file, guint64 *rev, time_t *time) { - struct stat_file_header *header; + struct stat_file_header *header; if (file == NULL || file->map == NULL) { return FALSE; } - + header = (struct stat_file_header *)file->map; if (rev != NULL) { @@ -785,29 +924,29 @@ statfile_get_revision (stat_file_t *file, guint64 *rev, time_t *time) return TRUE; } -guint64 +guint64 statfile_get_used_blocks (stat_file_t *file) { - struct stat_file_header *header; + struct stat_file_header *header; if (file == NULL || file->map == NULL) { - return (guint64)-1; + return (guint64) - 1; } - + header = (struct stat_file_header *)file->map; return header->used_blocks; } -guint64 +guint64 statfile_get_total_blocks (stat_file_t *file) { - struct stat_file_header *header; + struct stat_file_header *header; if (file == NULL || file->map == NULL) { - return (guint64)-1; + return (guint64) - 1; } - + header = (struct stat_file_header *)file->map; /* If total blocks is 0 we have old version of header, so set total blocks correctly */ @@ -821,13 +960,13 @@ statfile_get_total_blocks (stat_file_t *file) static void statfile_pool_invalidate_callback (gint fd, short what, void *ud) { - statfile_pool_t *pool = ud; - stat_file_t *file; - gint i; + statfile_pool_t *pool = ud; + stat_file_t *file; + gint i; msg_info ("invalidating %d statfiles", pool->opened); - for (i = 0; i < pool->opened; i ++) { + for (i = 0; i < pool->opened; i++) { file = &pool->files[i]; msync (file->map, file->len, MS_ASYNC); } @@ -836,44 +975,54 @@ statfile_pool_invalidate_callback (gint fd, short what, void *ud) void -statfile_pool_plan_invalidate (statfile_pool_t *pool, time_t seconds, time_t jitter) +statfile_pool_plan_invalidate (statfile_pool_t *pool, + time_t seconds, + time_t jitter) { - gboolean pending; + gboolean pending; if (pool->invalidate_event != NULL) { pending = evtimer_pending (pool->invalidate_event, NULL); if (pending) { /* Replan event */ - pool->invalidate_tv.tv_sec = seconds + g_random_int_range (0, jitter); + pool->invalidate_tv.tv_sec = seconds + + g_random_int_range (0, jitter); pool->invalidate_tv.tv_usec = 0; evtimer_add (pool->invalidate_event, &pool->invalidate_tv); } } else { - pool->invalidate_event = rspamd_mempool_alloc (pool->pool, sizeof (struct event)); + pool->invalidate_event = + rspamd_mempool_alloc (pool->pool, sizeof (struct event)); pool->invalidate_tv.tv_sec = seconds + g_random_int_range (0, jitter); pool->invalidate_tv.tv_usec = 0; - evtimer_set (pool->invalidate_event, statfile_pool_invalidate_callback, pool); + evtimer_set (pool->invalidate_event, + statfile_pool_invalidate_callback, + pool); evtimer_add (pool->invalidate_event, &pool->invalidate_tv); - msg_info ("invalidate of statfile pool is planned in %d seconds", (gint)pool->invalidate_tv.tv_sec); + msg_info ("invalidate of statfile pool is planned in %d seconds", + (gint)pool->invalidate_tv.tv_sec); } } stat_file_t * -get_statfile_by_symbol (statfile_pool_t *pool, struct rspamd_classifier_config *ccf, - const gchar *symbol, struct rspamd_statfile_config **st, gboolean try_create) +get_statfile_by_symbol (statfile_pool_t *pool, + struct rspamd_classifier_config *ccf, + const gchar *symbol, + struct rspamd_statfile_config **st, + gboolean try_create) { - stat_file_t *res = NULL; - GList *cur; + stat_file_t *res = NULL; + GList *cur; - if (pool == NULL || ccf == NULL || symbol == NULL) { + if (pool == NULL || ccf == NULL || symbol == NULL) { msg_err ("invalid input arguments"); - return NULL; - } + return NULL; + } - cur = g_list_first (ccf->statfiles); + cur = g_list_first (ccf->statfiles); while (cur) { *st = cur->data; if (strcmp (symbol, (*st)->symbol) == 0) { @@ -882,28 +1031,33 @@ get_statfile_by_symbol (statfile_pool_t *pool, struct rspamd_classifier_config * *st = NULL; cur = g_list_next (cur); } - if (*st == NULL) { + if (*st == NULL) { msg_info ("cannot find statfile with symbol %s", symbol); - return NULL; - } + return NULL; + } - if ((res = statfile_pool_is_open (pool, (*st)->path)) == NULL) { - if ((res = statfile_pool_open (pool, (*st)->path, (*st)->size, FALSE)) == NULL) { + if ((res = statfile_pool_is_open (pool, (*st)->path)) == NULL) { + if ((res = + statfile_pool_open (pool, (*st)->path, (*st)->size, + FALSE)) == NULL) { msg_warn ("cannot open %s", (*st)->path); - if (try_create) { - if (statfile_pool_create (pool, (*st)->path, (*st)->size) == -1) { + if (try_create) { + if (statfile_pool_create (pool, (*st)->path, + (*st)->size) == -1) { msg_err ("cannot create statfile %s", (*st)->path); return NULL; } - res = statfile_pool_open (pool, (*st)->path, (*st)->size, FALSE); + res = + statfile_pool_open (pool, (*st)->path, (*st)->size, FALSE); if (res == NULL) { - msg_err ("cannot open statfile %s after creation", (*st)->path); + msg_err ("cannot open statfile %s after creation", + (*st)->path); } - } + } } } - return res; + return res; } void @@ -913,10 +1067,12 @@ statfile_pool_lockall (statfile_pool_t *pool) gint i; if (pool->mlock_ok) { - for (i = 0; i < pool->opened; i ++) { + for (i = 0; i < pool->opened; i++) { file = &pool->files[i]; if (mlock (file->map, file->len) == -1) { - msg_warn ("mlock of statfile failed, maybe you need to increase RLIMIT_MEMLOCK limit for a process: %s", strerror (errno)); + msg_warn ( + "mlock of statfile failed, maybe you need to increase RLIMIT_MEMLOCK limit for a process: %s", + strerror (errno)); pool->mlock_ok = FALSE; return; } diff --git a/src/libserver/statfile.h b/src/libserver/statfile.h index 3ad5f1321..b85fc87a5 100644 --- a/src/libserver/statfile.h +++ b/src/libserver/statfile.h @@ -7,8 +7,8 @@ #define RSPAMD_STATFILE_H #include "config.h" -#include "mem_pool.h" #include "hash.h" +#include "mem_pool.h" #define CHAIN_LENGTH 128 @@ -25,41 +25,41 @@ * Common statfile header */ struct stat_file_header { - u_char magic[3]; /**< magic signature ('r' 's' 'd') */ - u_char version[2]; /**< version of statfile */ - u_char padding[3]; /**< padding */ - guint64 create_time; /**< create time (time_t->guint64) */ - guint64 revision; /**< revision number */ - guint64 rev_time; /**< revision time */ - guint64 used_blocks; /**< used blocks number */ - guint64 total_blocks; /**< total number of blocks */ - u_char unused[239]; /**< some bytes that can be used in future */ + u_char magic[3]; /**< magic signature ('r' 's' 'd') */ + u_char version[2]; /**< version of statfile */ + u_char padding[3]; /**< padding */ + guint64 create_time; /**< create time (time_t->guint64) */ + guint64 revision; /**< revision number */ + guint64 rev_time; /**< revision time */ + guint64 used_blocks; /**< used blocks number */ + guint64 total_blocks; /**< total number of blocks */ + u_char unused[239]; /**< some bytes that can be used in future */ }; /** * Section header */ struct stat_file_section { - guint64 code; /**< section's code */ - guint64 length; /**< section's length in blocks */ + guint64 code; /**< section's code */ + guint64 length; /**< section's length in blocks */ }; /** * Block of data in statfile */ struct stat_file_block { - guint32 hash1; /**< hash1 (also acts as index) */ - guint32 hash2; /**< hash2 */ - double value; /**< double value */ + guint32 hash1; /**< hash1 (also acts as index) */ + guint32 hash2; /**< hash2 */ + double value; /**< double value */ }; /** * Statistic file */ struct stat_file { - struct stat_file_header header; /**< header */ - struct stat_file_section section; /**< first section */ - struct stat_file_block blocks[1]; /**< first block of data */ + struct stat_file_header header; /**< header */ + struct stat_file_section section; /**< first section */ + struct stat_file_block blocks[1]; /**< first block of data */ }; /** @@ -67,32 +67,32 @@ struct stat_file { */ typedef struct stat_file_s { #ifdef HAVE_PATH_MAX - gchar filename[PATH_MAX]; /**< name of file */ + gchar filename[PATH_MAX]; /**< name of file */ #else - gchar filename[MAXPATHLEN]; /**< name of file */ + gchar filename[MAXPATHLEN]; /**< name of file */ #endif - gint fd; /**< descriptor */ - void *map; /**< mmaped area */ - off_t seek_pos; /**< current seek position */ - struct stat_file_section cur_section; /**< current section */ - time_t open_time; /**< time when file was opened */ - time_t access_time; /**< last access time */ - size_t len; /**< length of file(in bytes) */ - rspamd_mempool_mutex_t *lock; /**< mutex */ + gint fd; /**< descriptor */ + void *map; /**< mmaped area */ + off_t seek_pos; /**< current seek position */ + struct stat_file_section cur_section; /**< current section */ + time_t open_time; /**< time when file was opened */ + time_t access_time; /**< last access time */ + size_t len; /**< length of file(in bytes) */ + rspamd_mempool_mutex_t *lock; /**< mutex */ } stat_file_t; /** * Statfiles pool */ typedef struct statfile_pool_s { - stat_file_t *files; /**< hash table of opened files indexed by name */ - void **maps; /**< shared hash table of mmaped areas indexed by name */ - gint opened; /**< number of opened files */ - rspamd_mempool_t *pool; /**< memory pool object */ - rspamd_mempool_mutex_t *lock; /**< mutex */ - struct event *invalidate_event; /**< event for pool invalidation */ + stat_file_t *files; /**< hash table of opened files indexed by name */ + void **maps; /**< shared hash table of mmaped areas indexed by name */ + gint opened; /**< number of opened files */ + rspamd_mempool_t *pool; /**< memory pool object */ + rspamd_mempool_mutex_t *lock; /**< mutex */ + struct event *invalidate_event; /**< event for pool invalidation */ struct timeval invalidate_tv; - gboolean mlock_ok; /**< whether it is possible to use mlock (2) to avoid statfiles unloading */ + gboolean mlock_ok; /**< whether it is possible to use mlock (2) to avoid statfiles unloading */ } statfile_pool_t; /* Forwarded declarations */ @@ -104,7 +104,8 @@ struct rspamd_statfile_config; * @param max_size maximum size * @return statfile pool object */ -statfile_pool_t* statfile_pool_new (rspamd_mempool_t *pool, gboolean use_mlock); +statfile_pool_t * statfile_pool_new (rspamd_mempool_t *pool, + gboolean use_mlock); /** * Open statfile and attach it to pool @@ -112,7 +113,10 @@ statfile_pool_t* statfile_pool_new (rspamd_mempool_t *pool, gboolean use_mlock); * @param filename name of statfile to open * @return 0 if specified statfile is attached and -1 in case of error */ -stat_file_t* statfile_pool_open (statfile_pool_t *pool, gchar *filename, size_t len, gboolean forced); +stat_file_t * statfile_pool_open (statfile_pool_t *pool, + gchar *filename, + size_t len, + gboolean forced); /** * Create new statfile but DOES NOT attach it to pool, use @see statfile_pool_open for attaching @@ -130,7 +134,9 @@ gint statfile_pool_create (statfile_pool_t *pool, gchar *filename, size_t len); * @param remove_hash remove filename from opened files hash also * @return 0 if file was closed and -1 if statfile was not opened */ -gint statfile_pool_close (statfile_pool_t *pool, stat_file_t *file, gboolean keep_sorted); +gint statfile_pool_close (statfile_pool_t *pool, + stat_file_t *file, + gboolean keep_sorted); /** * Delete statfile pool and close all attached statfiles @@ -167,7 +173,11 @@ void statfile_pool_unlock_file (statfile_pool_t *pool, stat_file_t *file); * @param now current time * @return block value or 0 if block is not found */ -double statfile_pool_get_block (statfile_pool_t *pool, stat_file_t *file, guint32 h1, guint32 h2, time_t now); +double statfile_pool_get_block (statfile_pool_t *pool, + stat_file_t *file, + guint32 h1, + guint32 h2, + time_t now); /** * Set specified block in statfile @@ -178,7 +188,12 @@ double statfile_pool_get_block (statfile_pool_t *pool, stat_file_t *file, guint3 * @param now current time * @param value value of block */ -void statfile_pool_set_block (statfile_pool_t *pool, stat_file_t *file, guint32 h1, guint32 h2, time_t now, double value); +void statfile_pool_set_block (statfile_pool_t *pool, + stat_file_t *file, + guint32 h1, + guint32 h2, + time_t now, + double value); /** * Check whether statfile is opened @@ -186,7 +201,7 @@ void statfile_pool_set_block (statfile_pool_t *pool, stat_file_t *file, guint32 * @param filename name of statfile * @return TRUE if specified statfile is opened and FALSE otherwise */ -stat_file_t* statfile_pool_is_open (statfile_pool_t *pool, gchar *filename); +stat_file_t * statfile_pool_is_open (statfile_pool_t *pool, gchar *filename); /** * Returns current statfile section @@ -204,7 +219,10 @@ guint32 statfile_pool_get_section (statfile_pool_t *pool, stat_file_t *file); * @param from_begin search for section from begin of file if true * @return TRUE if section was set and FALSE otherwise */ -gboolean statfile_pool_set_section (statfile_pool_t *pool, stat_file_t *file, guint32 code, gboolean from_begin); +gboolean statfile_pool_set_section (statfile_pool_t *pool, + stat_file_t *file, + guint32 code, + gboolean from_begin); /** * Add new section to statfile @@ -214,7 +232,10 @@ gboolean statfile_pool_set_section (statfile_pool_t *pool, stat_file_t *file, gu * @param length length in blocks of new section * @return TRUE if section was successfully added and FALSE in case of error */ -gboolean statfile_pool_add_section (statfile_pool_t *pool, stat_file_t *file, guint32 code, guint64 length); +gboolean statfile_pool_add_section (statfile_pool_t *pool, + stat_file_t *file, + guint32 code, + guint64 length); /** @@ -268,7 +289,9 @@ guint64 statfile_get_total_blocks (stat_file_t *file); /** * Plan statfile pool invalidation */ -void statfile_pool_plan_invalidate (statfile_pool_t *pool, time_t seconds, time_t jitter); +void statfile_pool_plan_invalidate (statfile_pool_t *pool, + time_t seconds, + time_t jitter); /** * Get a statfile by symbol @@ -278,7 +301,10 @@ void statfile_pool_plan_invalidate (statfile_pool_t *pool, time_t seconds, time_ * @param st statfile to get * @param try_create whether we need to create statfile if it is absent */ -stat_file_t* get_statfile_by_symbol (statfile_pool_t *pool, struct rspamd_classifier_config *ccf, - const gchar *symbol, struct rspamd_statfile_config **st, gboolean try_create); +stat_file_t * get_statfile_by_symbol (statfile_pool_t *pool, + struct rspamd_classifier_config *ccf, + const gchar *symbol, + struct rspamd_statfile_config **st, + gboolean try_create); #endif diff --git a/src/libserver/statfile_sync.c b/src/libserver/statfile_sync.c index 1121658fa..55c08f879 100644 --- a/src/libserver/statfile_sync.c +++ b/src/libserver/statfile_sync.c @@ -22,14 +22,14 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "config.h" +#include "binlog.h" +#include "buffer.h" #include "cfg_file.h" -#include "tokenizers/tokenizers.h" #include "classifiers/classifiers.h" +#include "config.h" #include "statfile.h" -#include "binlog.h" -#include "buffer.h" #include "statfile_sync.h" +#include "tokenizers/tokenizers.h" enum rspamd_sync_state { SYNC_STATE_GREETING, @@ -56,63 +56,69 @@ struct rspamd_sync_ctx { enum rspamd_sync_state state; gboolean is_busy; - guint64 new_rev; - guint64 new_time; - guint64 new_len; + guint64 new_rev; + guint64 new_time; + guint64 new_len; }; static void log_next_sync (const gchar *symbol, time_t delay) { - gchar outstr[200]; - time_t t; + gchar outstr[200]; + time_t t; struct tm *tmp; - gint r; + gint r; - t = time(NULL); + t = time (NULL); t += delay; - tmp = localtime(&t); + tmp = localtime (&t); if (tmp) { - r = rspamd_snprintf (outstr, sizeof (outstr), "statfile_sync: next sync of %s at ", symbol); - if ((r = strftime(outstr + r, sizeof(outstr) - r, "%T", tmp)) != 0) { + r = rspamd_snprintf (outstr, + sizeof (outstr), + "statfile_sync: next sync of %s at ", + symbol); + if ((r = strftime (outstr + r, sizeof(outstr) - r, "%T", tmp)) != 0) { msg_info (outstr); } } } -static gboolean +static gboolean parse_revision_line (struct rspamd_sync_ctx *ctx, f_str_t *in) { - guint i, state = 0; - gchar *p, *c, numbuf[sizeof("18446744073709551615")]; - guint64 *val; + guint i, state = 0; + gchar *p, *c, numbuf[sizeof("18446744073709551615")]; + guint64 *val; /* First of all try to find END line */ - if (in->len >= sizeof ("END") - 1 && memcmp (in->begin, "END", sizeof ("END") - 1) == 0) { + if (in->len >= sizeof ("END") - 1 && + memcmp (in->begin, "END", sizeof ("END") - 1) == 0) { ctx->state = SYNC_STATE_QUIT; ctx->is_busy = FALSE; return TRUE; } /* Next check for error line */ - if (in->len >= sizeof ("FAIL") - 1 && memcmp (in->begin, "FAIL", sizeof ("FAIL") - 1) == 0) { + if (in->len >= sizeof ("FAIL") - 1 && + memcmp (in->begin, "FAIL", sizeof ("FAIL") - 1) == 0) { ctx->state = SYNC_STATE_QUIT; ctx->is_busy = FALSE; return TRUE; } - + /* Now try to extract 3 numbers from string: revision, time and length */ p = in->begin; val = &ctx->new_rev; - for (i = 0; i < in->len; i ++, p ++) { + for (i = 0; i < in->len; i++, p++) { if (g_ascii_isspace (*p) || i == in->len - 1) { if (state == 1) { if (i == in->len - 1) { /* One more character */ - p ++; + p++; } - rspamd_strlcpy (numbuf, c, MIN (p - c + 1, (gint)sizeof (numbuf))); + rspamd_strlcpy (numbuf, c, MIN (p - c + 1, + (gint)sizeof (numbuf))); errno = 0; *val = strtoull (numbuf, NULL, 10); if (errno != 0) { @@ -144,94 +150,113 @@ parse_revision_line (struct rspamd_sync_ctx *ctx, f_str_t *in) return ((val == &ctx->new_len)); } -static gboolean +static gboolean read_blocks (struct rspamd_sync_ctx *ctx, f_str_t *in) { struct rspamd_binlog_element *elt; - guint i; - + guint i; + statfile_pool_lock_file (ctx->pool, ctx->real_statfile); elt = (struct rspamd_binlog_element *)in->begin; - for (i = 0; i < in->len / sizeof (struct rspamd_binlog_element); i ++, elt ++) { - statfile_pool_set_block (ctx->pool, ctx->real_statfile, elt->h1, elt->h2, ctx->new_time, elt->value); + for (i = 0; i < in->len / sizeof (struct rspamd_binlog_element); i++, + elt++) { + statfile_pool_set_block (ctx->pool, + ctx->real_statfile, + elt->h1, + elt->h2, + ctx->new_time, + elt->value); } statfile_pool_unlock_file (ctx->pool, ctx->real_statfile); return TRUE; } -static gboolean +static gboolean sync_read (f_str_t * in, void *arg) { struct rspamd_sync_ctx *ctx = arg; - gchar buf[256]; - guint64 rev = 0; - time_t ti = 0; + gchar buf[256]; + guint64 rev = 0; + time_t ti = 0; if (in->len == 0) { /* Skip empty lines */ return TRUE; } switch (ctx->state) { - case SYNC_STATE_GREETING: - /* Skip greeting line and write sync command */ - /* Write initial data */ - statfile_get_revision (ctx->real_statfile, &rev, &ti); - rev = rspamd_snprintf (buf, sizeof (buf), "sync %s %uL %T" CRLF, ctx->st->symbol, rev, ti); - ctx->state = SYNC_STATE_READ_LINE; - return rspamd_dispatcher_write (ctx->dispatcher, buf, rev, FALSE, FALSE); - break; - case SYNC_STATE_READ_LINE: - /* Try to parse line from server */ - if (!parse_revision_line (ctx, in)) { - msg_info ("cannot parse line of length %z: '%*s'", in->len, (gint)in->len, in->begin); - close (ctx->sock); - rspamd_remove_dispatcher (ctx->dispatcher); - ctx->is_busy = FALSE; - return FALSE; - } - else if (ctx->state != SYNC_STATE_QUIT) { - if (ctx->new_len > 0) { - ctx->state = SYNC_STATE_READ_REV; - rspamd_set_dispatcher_policy (ctx->dispatcher, BUFFER_CHARACTER, ctx->new_len); - } - } - else { - /* Quit this session */ - msg_info ("sync ended for: %s", ctx->st->symbol); - close (ctx->sock); - rspamd_remove_dispatcher (ctx->dispatcher); - ctx->is_busy = FALSE; - /* Immediately return from callback */ - return FALSE; - } - break; - case SYNC_STATE_READ_REV: - /* In now contains all blocks of specified revision, so we can read them directly */ - if (!read_blocks (ctx, in)) { - msg_info ("cannot read blocks"); - close (ctx->sock); - rspamd_remove_dispatcher (ctx->dispatcher); - ctx->is_busy = FALSE; - return FALSE; + case SYNC_STATE_GREETING: + /* Skip greeting line and write sync command */ + /* Write initial data */ + statfile_get_revision (ctx->real_statfile, &rev, &ti); + rev = rspamd_snprintf (buf, + sizeof (buf), + "sync %s %uL %T" CRLF, + ctx->st->symbol, + rev, + ti); + ctx->state = SYNC_STATE_READ_LINE; + return rspamd_dispatcher_write (ctx->dispatcher, buf, rev, FALSE, + FALSE); + break; + case SYNC_STATE_READ_LINE: + /* Try to parse line from server */ + if (!parse_revision_line (ctx, in)) { + msg_info ("cannot parse line of length %z: '%*s'", + in->len, + (gint)in->len, + in->begin); + close (ctx->sock); + rspamd_remove_dispatcher (ctx->dispatcher); + ctx->is_busy = FALSE; + return FALSE; + } + else if (ctx->state != SYNC_STATE_QUIT) { + if (ctx->new_len > 0) { + ctx->state = SYNC_STATE_READ_REV; + rspamd_set_dispatcher_policy (ctx->dispatcher, + BUFFER_CHARACTER, + ctx->new_len); } - statfile_set_revision (ctx->real_statfile, ctx->new_rev, ctx->new_time); - msg_info ("set new revision: %uL, readed %z bytes", ctx->new_rev, in->len); - /* Now try to read other revision or END line */ - ctx->state = SYNC_STATE_READ_LINE; - rspamd_set_dispatcher_policy (ctx->dispatcher, BUFFER_LINE, 0); - break; - case SYNC_STATE_QUIT: + } + else { + /* Quit this session */ + msg_info ("sync ended for: %s", ctx->st->symbol); + close (ctx->sock); + rspamd_remove_dispatcher (ctx->dispatcher); + ctx->is_busy = FALSE; + /* Immediately return from callback */ + return FALSE; + } + break; + case SYNC_STATE_READ_REV: + /* In now contains all blocks of specified revision, so we can read them directly */ + if (!read_blocks (ctx, in)) { + msg_info ("cannot read blocks"); close (ctx->sock); rspamd_remove_dispatcher (ctx->dispatcher); ctx->is_busy = FALSE; return FALSE; + } + statfile_set_revision (ctx->real_statfile, ctx->new_rev, ctx->new_time); + msg_info ("set new revision: %uL, readed %z bytes", + ctx->new_rev, + in->len); + /* Now try to read other revision or END line */ + ctx->state = SYNC_STATE_READ_LINE; + rspamd_set_dispatcher_policy (ctx->dispatcher, BUFFER_LINE, 0); + break; + case SYNC_STATE_QUIT: + close (ctx->sock); + rspamd_remove_dispatcher (ctx->dispatcher); + ctx->is_busy = FALSE; + return FALSE; } return TRUE; } -static void +static void sync_err (GError *err, void *arg) { struct rspamd_sync_ctx *ctx = arg; @@ -248,30 +273,40 @@ sync_timer_callback (gint fd, short what, void *ud) { struct rspamd_sync_ctx *ctx = ud; guint32 jittered_interval; - + /* Plan new event */ evtimer_del (&ctx->tm_ev); /* Add some jittering for synchronization */ - jittered_interval = g_random_int_range (ctx->sync_interval, ctx->sync_interval * 2); + jittered_interval = g_random_int_range (ctx->sync_interval, + ctx->sync_interval * 2); msec_to_tv (jittered_interval, &ctx->interval); evtimer_add (&ctx->tm_ev, &ctx->interval); log_next_sync (ctx->st->symbol, ctx->interval.tv_sec); - + if (ctx->is_busy) { /* Sync is in progress */ msg_info ("syncronization process is in progress, do not start new one"); return; } - if ((ctx->sock = make_universal_socket (ctx->st->binlog->master_addr, ctx->st->binlog->master_port, - SOCK_STREAM, TRUE, FALSE, TRUE)) == -1) { + if ((ctx->sock = + make_universal_socket (ctx->st->binlog->master_addr, + ctx->st->binlog->master_port, + SOCK_STREAM, TRUE, FALSE, TRUE)) == -1) { msg_info ("cannot connect to %s", ctx->st->binlog->master_addr); return; } /* Now create and activate dispatcher */ msec_to_tv (ctx->timeout, &ctx->io_tv); - ctx->dispatcher = rspamd_create_dispatcher (ctx->ev_base, ctx->sock, BUFFER_LINE, sync_read, NULL, sync_err, &ctx->io_tv, ctx); - + ctx->dispatcher = rspamd_create_dispatcher (ctx->ev_base, + ctx->sock, + BUFFER_LINE, + sync_read, + NULL, + sync_err, + &ctx->io_tv, + ctx); + ctx->state = SYNC_STATE_GREETING; ctx->is_busy = TRUE; @@ -280,29 +315,39 @@ sync_timer_callback (gint fd, short what, void *ud) } static gboolean -add_statfile_watch (statfile_pool_t *pool, struct rspamd_statfile_config *st, struct rspamd_config *cfg, struct event_base *ev_base) +add_statfile_watch (statfile_pool_t *pool, + struct rspamd_statfile_config *st, + struct rspamd_config *cfg, + struct event_base *ev_base) { struct rspamd_sync_ctx *ctx; guint32 jittered_interval; - + if (st->binlog->master_addr != NULL) { - ctx = rspamd_mempool_alloc (pool->pool, sizeof (struct rspamd_sync_ctx)); + ctx = + rspamd_mempool_alloc (pool->pool, sizeof (struct rspamd_sync_ctx)); ctx->st = st; ctx->timeout = cfg->statfile_sync_timeout; ctx->sync_interval = cfg->statfile_sync_interval; ctx->ev_base = ev_base; /* Add some jittering for synchronization */ - jittered_interval = g_random_int_range (ctx->sync_interval, ctx->sync_interval * 2); + jittered_interval = g_random_int_range (ctx->sync_interval, + ctx->sync_interval * 2); msec_to_tv (jittered_interval, &ctx->interval); /* Open statfile and attach it to pool */ - if ((ctx->real_statfile = statfile_pool_is_open (pool, st->path)) == NULL) { - if ((ctx->real_statfile = statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) { + if ((ctx->real_statfile = + statfile_pool_is_open (pool, st->path)) == NULL) { + if ((ctx->real_statfile = + statfile_pool_open (pool, st->path, st->size, FALSE)) == NULL) { msg_warn ("cannot open %s", st->path); if (statfile_pool_create (pool, st->path, st->size) == -1) { msg_err ("cannot create statfile %s", st->path); return FALSE; } - ctx->real_statfile = statfile_pool_open (pool, st->path, st->size, FALSE); + ctx->real_statfile = statfile_pool_open (pool, + st->path, + st->size, + FALSE); } } /* Now plan event for it's future executing */ @@ -312,21 +357,24 @@ add_statfile_watch (statfile_pool_t *pool, struct rspamd_statfile_config *st, st log_next_sync (st->symbol, ctx->interval.tv_sec); } else { - msg_err ("cannot add statfile watch for statfile %s: no master defined", st->symbol); + msg_err ("cannot add statfile watch for statfile %s: no master defined", + st->symbol); return FALSE; } return TRUE; } -gboolean -start_statfile_sync (statfile_pool_t *pool, struct rspamd_config *cfg, struct event_base *ev_base) +gboolean +start_statfile_sync (statfile_pool_t *pool, + struct rspamd_config *cfg, + struct event_base *ev_base) { GList *cur, *l; struct rspamd_classifier_config *cl; struct rspamd_statfile_config *st; - /* + /* * First of all walk through all classifiers and find those statfiles * for which we should do sync (slave affinity) */ diff --git a/src/libserver/statfile_sync.h b/src/libserver/statfile_sync.h index bcaeb3bb5..37a0ddb78 100644 --- a/src/libserver/statfile_sync.h +++ b/src/libserver/statfile_sync.h @@ -1,14 +1,16 @@ #ifndef RSPAMD_STATFILE_SYNC_H #define RSPAMD_STATFILE_SYNC_H +#include "cfg_file.h" #include "config.h" #include "main.h" #include "statfile.h" -#include "cfg_file.h" /* * Start synchronization of statfiles. Must be called after event_init as it adds events */ -gboolean start_statfile_sync (statfile_pool_t *pool, struct rspamd_config *cfg, struct event_base *ev_base); +gboolean start_statfile_sync (statfile_pool_t *pool, + struct rspamd_config *cfg, + struct event_base *ev_base); #endif diff --git a/src/libserver/symbols_cache.c b/src/libserver/symbols_cache.c index b0ce7d872..6ffd8b103 100644 --- a/src/libserver/symbols_cache.c +++ b/src/libserver/symbols_cache.c @@ -22,12 +22,12 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include "cfg_file.h" #include "config.h" -#include "util.h" #include "main.h" #include "message.h" #include "symbols_cache.h" -#include "cfg_file.h" +#include "util.h" #define WEIGHT_MULT 4.0 #define FREQUENCY_MULT 10.0 @@ -41,13 +41,13 @@ #define MIN_CACHE 17 -static guint64 total_frequency = 0; -static guint32 nsymbols = 0; +static guint64 total_frequency = 0; +static guint32 nsymbols = 0; gint cache_cmp (const void *p1, const void *p2) { - const struct cache_item *i1 = p1, *i2 = p2; + const struct cache_item *i1 = p1, *i2 = p2; return strcmp (i1->s->symbol, i2->s->symbol); } @@ -55,20 +55,24 @@ cache_cmp (const void *p1, const void *p2) gint cache_logic_cmp (const void *p1, const void *p2) { - const struct cache_item *i1 = p1, *i2 = p2; - double w1, w2; - double weight1, weight2; - double f1 = 0, f2 = 0; + const struct cache_item *i1 = p1, *i2 = p2; + double w1, w2; + double weight1, weight2; + double f1 = 0, f2 = 0; if (i1->priority == 0 && i2->priority == 0) { if (total_frequency > 0) { - f1 = ((double)i1->s->frequency * nsymbols) / (double)total_frequency; - f2 = ((double)i2->s->frequency * nsymbols) / (double)total_frequency; + f1 = + ((double)i1->s->frequency * nsymbols) / (double)total_frequency; + f2 = + ((double)i2->s->frequency * nsymbols) / (double)total_frequency; } weight1 = i1->metric_weight == 0 ? i1->s->weight : i1->metric_weight; weight2 = i2->metric_weight == 0 ? i2->s->weight : i2->metric_weight; - w1 = abs (weight1) * WEIGHT_MULT + f1 * FREQUENCY_MULT + i1->s->avg_time * TIME_MULT; - w2 = abs (weight2) * WEIGHT_MULT + f2 * FREQUENCY_MULT + i2->s->avg_time * TIME_MULT; + w1 = abs (weight1) * WEIGHT_MULT + f1 * FREQUENCY_MULT + + i1->s->avg_time * TIME_MULT; + w2 = abs (weight2) * WEIGHT_MULT + f2 * FREQUENCY_MULT + + i2->s->avg_time * TIME_MULT; } else { /* Strict sorting */ @@ -79,12 +83,12 @@ cache_logic_cmp (const void *p1, const void *p2) return (gint)w2 - w1; } -static GChecksum * +static GChecksum * get_mem_cksum (struct symbols_cache *cache) { - GChecksum *result; - GList *cur, *l; - struct cache_item *item; + GChecksum *result; + GList *cur, *l; + struct cache_item *item; result = g_checksum_new (G_CHECKSUM_SHA1); @@ -94,7 +98,8 @@ get_mem_cksum (struct symbols_cache *cache) while (cur) { item = cur->data; if (item->s->symbol[0] != '\0') { - g_checksum_update (result, item->s->symbol, strlen (item->s->symbol)); + g_checksum_update (result, item->s->symbol, + strlen (item->s->symbol)); } cur = g_list_next (cur); } @@ -107,7 +112,8 @@ get_mem_cksum (struct symbols_cache *cache) while (cur) { item = cur->data; if (item->s->symbol[0] != '\0') { - g_checksum_update (result, item->s->symbol, strlen (item->s->symbol)); + g_checksum_update (result, item->s->symbol, + strlen (item->s->symbol)); } total_frequency += item->s->frequency; cur = g_list_next (cur); @@ -121,8 +127,8 @@ get_mem_cksum (struct symbols_cache *cache) static void post_cache_init (struct symbols_cache *cache) { - GList *cur; - struct cache_item *item; + GList *cur; + struct cache_item *item; total_frequency = 0; nsymbols = cache->used_items; @@ -139,7 +145,8 @@ post_cache_init (struct symbols_cache *cache) cur = g_list_next (cur); } - cache->negative_items = g_list_sort (cache->negative_items, cache_logic_cmp); + cache->negative_items = + g_list_sort (cache->negative_items, cache_logic_cmp); cache->static_items = g_list_sort (cache->static_items, cache_logic_cmp); } @@ -147,22 +154,27 @@ post_cache_init (struct symbols_cache *cache) static void unmap_cache_file (gpointer arg) { - struct symbols_cache *cache = arg; - + struct symbols_cache *cache = arg; + /* A bit ugly usage */ munmap (cache->map, cache->used_items * sizeof (struct saved_cache_item)); } -static gboolean +static gboolean mmap_cache_file (struct symbols_cache *cache, gint fd, rspamd_mempool_t *pool) { - guint8 *map; - gint i; - GList *cur; - struct cache_item *item; + guint8 *map; + gint i; + GList *cur; + struct cache_item *item; if (cache->used_items > 0) { - map = mmap (NULL, cache->used_items * sizeof (struct saved_cache_item), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + map = mmap (NULL, + cache->used_items * sizeof (struct saved_cache_item), + PROT_READ | PROT_WRITE, + MAP_SHARED, + fd, + 0); if (map == MAP_FAILED) { msg_err ("cannot mmap cache file: %d, %s", errno, strerror (errno)); close (fd); @@ -176,16 +188,20 @@ mmap_cache_file (struct symbols_cache *cache, gint fd, rspamd_mempool_t *pool) cur = g_list_first (cache->negative_items); while (cur) { item = cur->data; - item->s = (struct saved_cache_item *)(map + i * sizeof (struct saved_cache_item)); + item->s = + (struct saved_cache_item *)(map + i * + sizeof (struct saved_cache_item)); cur = g_list_next (cur); - i ++; + i++; } cur = g_list_first (cache->static_items); while (cur) { item = cur->data; - item->s = (struct saved_cache_item *)(map + i * sizeof (struct saved_cache_item)); + item->s = + (struct saved_cache_item *)(map + i * + sizeof (struct saved_cache_item)); cur = g_list_next (cur); - i ++; + i++; } post_cache_init (cache); @@ -195,14 +211,17 @@ mmap_cache_file (struct symbols_cache *cache, gint fd, rspamd_mempool_t *pool) } /* Fd must be opened for writing, after creating file is mmapped */ -static gboolean -create_cache_file (struct symbols_cache *cache, const gchar *filename, gint fd, rspamd_mempool_t *pool) +static gboolean +create_cache_file (struct symbols_cache *cache, + const gchar *filename, + gint fd, + rspamd_mempool_t *pool) { - GChecksum *cksum; - u_char *digest; - gsize cklen; - GList *cur; - struct cache_item *item; + GChecksum *cksum; + u_char *digest; + gsize cklen; + GList *cur; + struct cache_item *item; /* Calculate checksum */ cksum = get_mem_cksum (cache); @@ -269,23 +288,33 @@ enum rspamd_symbol_type { }; static void -register_symbol_common (struct symbols_cache **cache, const gchar *name, double weight, gint priority, - symbol_func_t func, gpointer user_data, enum rspamd_symbol_type type) +register_symbol_common (struct symbols_cache **cache, + const gchar *name, + double weight, + gint priority, + symbol_func_t func, + gpointer user_data, + enum rspamd_symbol_type type) { - struct cache_item *item = NULL; - struct symbols_cache *pcache = *cache; - GList **target; - double *w; + struct cache_item *item = NULL; + struct symbols_cache *pcache = *cache; + GList **target; + double *w; if (*cache == NULL) { pcache = g_new0 (struct symbols_cache, 1); *cache = pcache; - pcache->static_pool = rspamd_mempool_new (rspamd_mempool_suggest_size ()); - pcache->items_by_symbol = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); + pcache->static_pool = + rspamd_mempool_new (rspamd_mempool_suggest_size ()); + pcache->items_by_symbol = g_hash_table_new (rspamd_str_hash, + rspamd_str_equal); } - - item = rspamd_mempool_alloc0 (pcache->static_pool, sizeof (struct cache_item)); - item->s = rspamd_mempool_alloc0 (pcache->static_pool, sizeof (struct saved_cache_item)); + + item = rspamd_mempool_alloc0 (pcache->static_pool, + sizeof (struct cache_item)); + item->s = + rspamd_mempool_alloc0 (pcache->static_pool, + sizeof (struct saved_cache_item)); rspamd_strlcpy (item->s->symbol, name, sizeof (item->s->symbol)); item->func = func; item->user_data = user_data; @@ -303,7 +332,10 @@ register_symbol_common (struct symbols_cache **cache, const gchar *name, double } /* Handle weight using default metric */ - if (pcache->cfg && pcache->cfg->default_metric && (w = g_hash_table_lookup (pcache->cfg->default_metric->symbols, name)) != NULL) { + if (pcache->cfg && pcache->cfg->default_metric && + (w = + g_hash_table_lookup (pcache->cfg->default_metric->symbols, + name)) != NULL) { item->s->weight = weight * (*w); } else { @@ -339,58 +371,100 @@ register_symbol_common (struct symbols_cache **cache, const gchar *name, double void register_symbol (struct symbols_cache **cache, const gchar *name, double weight, - symbol_func_t func, gpointer user_data) + symbol_func_t func, gpointer user_data) { - register_symbol_common (cache, name, weight, 0, func, user_data, SYMBOL_TYPE_NORMAL); + register_symbol_common (cache, + name, + weight, + 0, + func, + user_data, + SYMBOL_TYPE_NORMAL); } void -register_virtual_symbol (struct symbols_cache **cache, const gchar *name, double weight) +register_virtual_symbol (struct symbols_cache **cache, + const gchar *name, + double weight) { - register_symbol_common (cache, name, weight, 0, NULL, NULL, SYMBOL_TYPE_VIRTUAL); + register_symbol_common (cache, + name, + weight, + 0, + NULL, + NULL, + SYMBOL_TYPE_VIRTUAL); } void -register_callback_symbol (struct symbols_cache **cache, const gchar *name, double weight, - symbol_func_t func, gpointer user_data) +register_callback_symbol (struct symbols_cache **cache, + const gchar *name, + double weight, + symbol_func_t func, + gpointer user_data) { - register_symbol_common (cache, name, weight, 0, func, user_data, SYMBOL_TYPE_CALLBACK); + register_symbol_common (cache, + name, + weight, + 0, + func, + user_data, + SYMBOL_TYPE_CALLBACK); } void -register_callback_symbol_priority (struct symbols_cache **cache, const gchar *name, double weight, gint priority, - symbol_func_t func, gpointer user_data) +register_callback_symbol_priority (struct symbols_cache **cache, + const gchar *name, + double weight, + gint priority, + symbol_func_t func, + gpointer user_data) { - register_symbol_common (cache, name, weight, priority, func, user_data, SYMBOL_TYPE_CALLBACK); + register_symbol_common (cache, + name, + weight, + priority, + func, + user_data, + SYMBOL_TYPE_CALLBACK); } void -register_dynamic_symbol (rspamd_mempool_t *dynamic_pool, struct symbols_cache **cache, - const gchar *name, double weight, symbol_func_t func, - gpointer user_data, GList *networks) +register_dynamic_symbol (rspamd_mempool_t *dynamic_pool, + struct symbols_cache **cache, + const gchar *name, + double weight, + symbol_func_t func, + gpointer user_data, + GList *networks) { - struct cache_item *item = NULL; - struct symbols_cache *pcache = *cache; - GList *t, *cur; - uintptr_t r; - double *w; - guint32 mask = 0xFFFFFFFF; - struct dynamic_map_item *it; - gint rr; + struct cache_item *item = NULL; + struct symbols_cache *pcache = *cache; + GList *t, *cur; + uintptr_t r; + double *w; + guint32 mask = 0xFFFFFFFF; + struct dynamic_map_item *it; + gint rr; if (*cache == NULL) { pcache = g_new0 (struct symbols_cache, 1); *cache = pcache; - pcache->static_pool = rspamd_mempool_new (rspamd_mempool_suggest_size ()); + pcache->static_pool = + rspamd_mempool_new (rspamd_mempool_suggest_size ()); } - + item = rspamd_mempool_alloc0 (dynamic_pool, sizeof (struct cache_item)); - item->s = rspamd_mempool_alloc (dynamic_pool, sizeof (struct saved_cache_item)); + item->s = + rspamd_mempool_alloc (dynamic_pool, sizeof (struct saved_cache_item)); rspamd_strlcpy (item->s->symbol, name, sizeof (item->s->symbol)); item->func = func; item->user_data = user_data; /* Handle weight using default metric */ - if (pcache->cfg && pcache->cfg->default_metric && (w = g_hash_table_lookup (pcache->cfg->default_metric->symbols, name)) != NULL) { + if (pcache->cfg && pcache->cfg->default_metric && + (w = + g_hash_table_lookup (pcache->cfg->default_metric->symbols, + name)) != NULL) { item->s->weight = weight * (*w); } else { @@ -402,7 +476,7 @@ register_dynamic_symbol (rspamd_mempool_t *dynamic_pool, struct symbols_cache ** pcache->used_items++; msg_debug ("used items: %d, added symbol: %s", (*cache)->used_items, name); rspamd_set_counter (item->s->symbol, 0); - + g_hash_table_insert (pcache->items_by_symbol, item->s->symbol, item); if (networks == NULL) { @@ -420,50 +494,81 @@ register_dynamic_symbol (rspamd_mempool_t *dynamic_pool, struct symbols_cache ** r = ntohl (it->addr.s_addr & mask); if (it->negative) { /* For negatve items insert into list and into negative cache map */ - if ((r = radix32tree_find (pcache->negative_dynamic_map, r)) != RADIX_NO_VALUE) { + if ((r = + radix32tree_find (pcache->negative_dynamic_map, + r)) != RADIX_NO_VALUE) { t = (GList *)((gpointer)r); t = g_list_prepend (t, item); /* Replace pointers in radix tree and in destructor function */ - rspamd_mempool_replace_destructor (dynamic_pool, (rspamd_mempool_destruct_t)g_list_free, (gpointer)r, t); - rr = radix32tree_replace (pcache->negative_dynamic_map, ntohl (it->addr.s_addr), mask, (uintptr_t)t); + rspamd_mempool_replace_destructor (dynamic_pool, + (rspamd_mempool_destruct_t)g_list_free, (gpointer)r, t); + rr = radix32tree_replace (pcache->negative_dynamic_map, + ntohl (it->addr.s_addr), + mask, + (uintptr_t)t); if (rr == -1) { - msg_warn ("cannot replace ip to tree: %s, mask %X", inet_ntoa (it->addr), mask); + msg_warn ("cannot replace ip to tree: %s, mask %X", + inet_ntoa (it->addr), + mask); } } else { t = g_list_prepend (NULL, item); - rspamd_mempool_add_destructor (dynamic_pool, (rspamd_mempool_destruct_t)g_list_free, t); - rr = radix32tree_insert (pcache->negative_dynamic_map, ntohl (it->addr.s_addr), mask, (uintptr_t)t); + rspamd_mempool_add_destructor (dynamic_pool, + (rspamd_mempool_destruct_t)g_list_free, t); + rr = radix32tree_insert (pcache->negative_dynamic_map, + ntohl (it->addr.s_addr), + mask, + (uintptr_t)t); if (rr == -1) { - msg_warn ("cannot insert ip to tree: %s, mask %X", inet_ntoa (it->addr), mask); + msg_warn ("cannot insert ip to tree: %s, mask %X", + inet_ntoa (it->addr), + mask); } else if (rr == 1) { - msg_warn ("ip %s, mask %X, value already exists", inet_ntoa (it->addr), mask); + msg_warn ("ip %s, mask %X, value already exists", + inet_ntoa (it->addr), + mask); } } /* Insert into list */ - pcache->dynamic_items = g_list_prepend (pcache->dynamic_items, item); + pcache->dynamic_items = g_list_prepend (pcache->dynamic_items, + item); } else { - if ((r = radix32tree_find (pcache->dynamic_map, r)) != RADIX_NO_VALUE) { + if ((r = + radix32tree_find (pcache->dynamic_map, + r)) != RADIX_NO_VALUE) { t = (GList *)((gpointer)r); t = g_list_prepend (t, item); /* Replace pointers in radix tree and in destructor function */ - rspamd_mempool_replace_destructor (dynamic_pool, (rspamd_mempool_destruct_t)g_list_free, (gpointer)r, t); - rr = radix32tree_replace (pcache->dynamic_map, ntohl (it->addr.s_addr), mask, (uintptr_t)t); + rspamd_mempool_replace_destructor (dynamic_pool, + (rspamd_mempool_destruct_t)g_list_free, (gpointer)r, t); + rr = + radix32tree_replace (pcache->dynamic_map, + ntohl (it->addr.s_addr), mask, (uintptr_t)t); if (rr == -1) { - msg_warn ("cannot replace ip to tree: %s, mask %X", inet_ntoa (it->addr), mask); + msg_warn ("cannot replace ip to tree: %s, mask %X", + inet_ntoa (it->addr), + mask); } } else { t = g_list_prepend (NULL, item); - rspamd_mempool_add_destructor (dynamic_pool, (rspamd_mempool_destruct_t)g_list_free, t); - rr = radix32tree_insert (pcache->dynamic_map, ntohl (it->addr.s_addr), mask, (uintptr_t)t); + rspamd_mempool_add_destructor (dynamic_pool, + (rspamd_mempool_destruct_t)g_list_free, t); + rr = + radix32tree_insert (pcache->dynamic_map, + ntohl (it->addr.s_addr), mask, (uintptr_t)t); if (rr == -1) { - msg_warn ("cannot insert ip to tree: %s, mask %X", inet_ntoa (it->addr), mask); + msg_warn ("cannot insert ip to tree: %s, mask %X", + inet_ntoa (it->addr), + mask); } else if (rr == 1) { - msg_warn ("ip %s, mask %X, value already exists", inet_ntoa (it->addr), mask); + msg_warn ("ip %s, mask %X, value already exists", + inet_ntoa (it->addr), + mask); } } } @@ -493,8 +598,8 @@ remove_dynamic_rules (struct symbols_cache *cache) static void free_cache (gpointer arg) { - struct symbols_cache *cache = arg; - + struct symbols_cache *cache = arg; + if (cache->map != NULL) { unmap_cache_file (cache); } @@ -521,15 +626,18 @@ free_cache (gpointer arg) } gboolean -init_symbols_cache (rspamd_mempool_t * pool, struct symbols_cache *cache, struct rspamd_config *cfg, - const gchar *filename, gboolean ignore_checksum) +init_symbols_cache (rspamd_mempool_t * pool, + struct symbols_cache *cache, + struct rspamd_config *cfg, + const gchar *filename, + gboolean ignore_checksum) { - struct stat st; - gint fd; - GChecksum *cksum; - u_char *mem_sum, *file_sum; - gsize cklen; - gboolean res; + struct stat st; + gint fd; + GChecksum *cksum; + u_char *mem_sum, *file_sum; + gsize cklen; + gboolean res; if (cache == NULL) { return FALSE; @@ -545,14 +653,19 @@ init_symbols_cache (rspamd_mempool_t * pool, struct symbols_cache *cache, struct post_cache_init (cache); return TRUE; } - + /* First of all try to stat file */ if (stat (filename, &st) == -1) { /* Check errno */ if (errno == ENOENT) { /* Try to create file */ - if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { - msg_info ("cannot create file %s, error %d, %s", filename, errno, strerror (errno)); + if ((fd = + open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | + S_IRUSR)) == -1) { + msg_info ("cannot create file %s, error %d, %s", + filename, + errno, + strerror (errno)); return FALSE; } else { @@ -560,13 +673,19 @@ init_symbols_cache (rspamd_mempool_t * pool, struct symbols_cache *cache, struct } } else { - msg_info ("cannot stat file %s, error %d, %s", filename, errno, strerror (errno)); + msg_info ("cannot stat file %s, error %d, %s", + filename, + errno, + strerror (errno)); return FALSE; } } else { if ((fd = open (filename, O_RDWR)) == -1) { - msg_info ("cannot open file %s, error %d, %s", filename, errno, strerror (errno)); + msg_info ("cannot open file %s, error %d, %s", + filename, + errno, + strerror (errno)); return FALSE; } } @@ -589,8 +708,13 @@ init_symbols_cache (rspamd_mempool_t * pool, struct symbols_cache *cache, struct if (errno == EINVAL) { /* Try to create file */ msg_info ("recreate cache file"); - if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { - msg_info ("cannot create file %s, error %d, %s", filename, errno, strerror (errno)); + if ((fd = + open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | + S_IRUSR)) == -1) { + msg_info ("cannot create file %s, error %d, %s", + filename, + errno, + strerror (errno)); return FALSE; } else { @@ -600,7 +724,8 @@ init_symbols_cache (rspamd_mempool_t * pool, struct symbols_cache *cache, struct close (fd); g_free (mem_sum); g_checksum_free (cksum); - msg_err ("cannot seek to read checksum, %d, %s", errno, strerror (errno)); + msg_err ("cannot seek to read checksum, %d, %s", errno, + strerror (errno)); return FALSE; } file_sum = g_malloc (cklen); @@ -620,8 +745,13 @@ init_symbols_cache (rspamd_mempool_t * pool, struct symbols_cache *cache, struct g_checksum_free (cksum); msg_info ("checksum mismatch, recreating file"); /* Reopen with rw permissions */ - if ((fd = open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | S_IRUSR)) == -1) { - msg_info ("cannot create file %s, error %d, %s", filename, errno, strerror (errno)); + if ((fd = + open (filename, O_RDWR | O_TRUNC | O_CREAT, S_IWUSR | + S_IRUSR)) == -1) { + msg_info ("cannot create file %s, error %d, %s", + filename, + errno, + strerror (errno)); return FALSE; } else { @@ -636,7 +766,9 @@ init_symbols_cache (rspamd_mempool_t * pool, struct symbols_cache *cache, struct /* MMap cache file and copy saved_cache structures */ res = mmap_cache_file (cache, fd, pool); - rspamd_mempool_add_destructor (pool, (rspamd_mempool_destruct_t)free_cache, cache); + rspamd_mempool_add_destructor (pool, + (rspamd_mempool_destruct_t)free_cache, + cache); return res; } @@ -648,10 +780,12 @@ check_dynamic_item (struct rspamd_task *task, struct symbols_cache *cache) /* TODO: radix doesn't support ipv6 addrs */ return NULL; #else - GList *res = NULL; - uintptr_t r; + GList *res = NULL; + uintptr_t r; if (cache->dynamic_map != NULL && task->from_addr.s_addr != INADDR_NONE) { - if ((r = radix32tree_find (cache->dynamic_map, ntohl (task->from_addr.s_addr))) != RADIX_NO_VALUE) { + if ((r = + radix32tree_find (cache->dynamic_map, + ntohl (task->from_addr.s_addr))) != RADIX_NO_VALUE) { res = (GList *)((gpointer)r); return res; } @@ -664,18 +798,23 @@ check_dynamic_item (struct rspamd_task *task, struct symbols_cache *cache) } static gboolean -check_negative_dynamic_item (struct rspamd_task *task, struct symbols_cache *cache, struct cache_item *item) +check_negative_dynamic_item (struct rspamd_task *task, + struct symbols_cache *cache, + struct cache_item *item) { #ifdef HAVE_INET_PTON /* TODO: radix doesn't support ipv6 addrs */ return FALSE; #else - GList *res = NULL; - uintptr_t r; - - if (cache->negative_dynamic_map != NULL && task->from_addr.s_addr != INADDR_NONE) { - if ((r = radix32tree_find (cache->negative_dynamic_map, ntohl (task->from_addr.s_addr))) != RADIX_NO_VALUE) { + GList *res = NULL; + uintptr_t r; + + if (cache->negative_dynamic_map != NULL && task->from_addr.s_addr != + INADDR_NONE) { + if ((r = + radix32tree_find (cache->negative_dynamic_map, + ntohl (task->from_addr.s_addr))) != RADIX_NO_VALUE) { res = (GList *)((gpointer)r); while (res) { if (res->data == (gpointer)item) { @@ -693,7 +832,7 @@ check_negative_dynamic_item (struct rspamd_task *task, struct symbols_cache *cac static gboolean check_debug_symbol (struct rspamd_config *cfg, const gchar *symbol) { - GList *cur; + GList *cur; cur = cfg->debug_symbols; while (cur) { @@ -736,11 +875,13 @@ rspamd_symbols_cache_metric_cb (gpointer k, gpointer v, gpointer ud) } gboolean -validate_cache (struct symbols_cache *cache, struct rspamd_config *cfg, gboolean strict) +validate_cache (struct symbols_cache *cache, + struct rspamd_config *cfg, + gboolean strict) { - struct cache_item *item; - GList *cur, *p, *metric_symbols; - gboolean res; + struct cache_item *item; + GList *cur, *p, *metric_symbols; + gboolean res; if (cache == NULL) { msg_err ("empty cache is invalid"); @@ -752,13 +893,16 @@ validate_cache (struct symbols_cache *cache, struct rspamd_config *cfg, gboolean while (cur) { item = cur->data; if (!item->is_callback) { - if (g_hash_table_lookup (cfg->metrics_symbols, item->s->symbol) == NULL) { + if (g_hash_table_lookup (cfg->metrics_symbols, + item->s->symbol) == NULL) { if (strict) { - msg_warn ("no weight registered for symbol %s", item->s->symbol); + msg_warn ("no weight registered for symbol %s", + item->s->symbol); return FALSE; } else { - msg_info ("no weight registered for symbol %s", item->s->symbol); + msg_info ("no weight registered for symbol %s", + item->s->symbol); } } } @@ -768,13 +912,16 @@ validate_cache (struct symbols_cache *cache, struct rspamd_config *cfg, gboolean while (cur) { item = cur->data; if (!item->is_callback) { - if (g_hash_table_lookup (cfg->metrics_symbols, item->s->symbol) == NULL) { + if (g_hash_table_lookup (cfg->metrics_symbols, + item->s->symbol) == NULL) { if (strict) { - msg_warn ("no weight registered for symbol %s", item->s->symbol); + msg_warn ("no weight registered for symbol %s", + item->s->symbol); return FALSE; } else { - msg_info ("no weight registered for symbol %s", item->s->symbol); + msg_info ("no weight registered for symbol %s", + item->s->symbol); } } } @@ -807,7 +954,9 @@ validate_cache (struct symbols_cache *cache, struct rspamd_config *cfg, gboolean } } if (!res) { - msg_warn ("symbol '%s' is registered in metric but not found in cache", cur->data); + msg_warn ( + "symbol '%s' is registered in metric but not found in cache", + cur->data); if (strict) { return FALSE; } @@ -819,10 +968,14 @@ validate_cache (struct symbols_cache *cache, struct rspamd_config *cfg, gboolean /* Now adjust symbol weights according to default metric */ if (cfg->default_metric != NULL) { - g_hash_table_foreach (cfg->default_metric->symbols, rspamd_symbols_cache_metric_cb, cache); + g_hash_table_foreach (cfg->default_metric->symbols, + rspamd_symbols_cache_metric_cb, + cache); /* Resort caches */ - cache->negative_items = g_list_sort (cache->negative_items, cache_logic_cmp); - cache->static_items = g_list_sort (cache->static_items, cache_logic_cmp); + cache->negative_items = g_list_sort (cache->negative_items, + cache_logic_cmp); + cache->static_items = + g_list_sort (cache->static_items, cache_logic_cmp); } return TRUE; @@ -840,16 +993,18 @@ struct symbol_callback_data { }; gboolean -call_symbol_callback (struct rspamd_task * task, struct symbols_cache * cache, gpointer *save) +call_symbol_callback (struct rspamd_task * task, + struct symbols_cache * cache, + gpointer *save) { #ifdef HAVE_CLOCK_GETTIME - struct timespec ts1, ts2; + struct timespec ts1, ts2; #else - struct timeval tv1, tv2; + struct timeval tv1, tv2; #endif - guint64 diff; - struct cache_item *item = NULL; - struct symbol_callback_data *s = *save; + guint64 diff; + struct cache_item *item = NULL; + struct symbol_callback_data *s = *save; if (s == NULL) { if (cache == NULL) { @@ -863,14 +1018,17 @@ call_symbol_callback (struct rspamd_task * task, struct symbols_cache * cache, g post_cache_init (cache); rspamd_mempool_wunlock_rwlock (cache->lock); } - s = rspamd_mempool_alloc0 (task->task_pool, sizeof (struct symbol_callback_data)); + s = + rspamd_mempool_alloc0 (task->task_pool, + sizeof (struct symbol_callback_data)); *save = s; if (cache->negative_items != NULL) { s->list_pointer = g_list_first (cache->negative_items); s->saved_item = s->list_pointer->data; s->state = CACHE_STATE_NEGATIVE; } - else if ((s->list_pointer = check_dynamic_item (task, cache)) || cache->dynamic_items != NULL) { + else if ((s->list_pointer = + check_dynamic_item (task, cache)) || cache->dynamic_items != NULL) { if (s->list_pointer == NULL) { s->list_pointer = g_list_first (cache->dynamic_items); s->saved_item = s->list_pointer->data; @@ -898,63 +1056,47 @@ call_symbol_callback (struct rspamd_task * task, struct symbols_cache * cache, g return FALSE; } switch (s->state) { - case CACHE_STATE_NEGATIVE: - s->list_pointer = g_list_next (s->list_pointer); - if (s->list_pointer == NULL) { - if ((s->list_pointer = check_dynamic_item (task, cache)) || cache->dynamic_items != NULL) { - if (s->list_pointer == NULL) { - s->list_pointer = g_list_first (cache->dynamic_items); - s->saved_item = s->list_pointer->data; - s->state = CACHE_STATE_DYNAMIC; - } - else { - s->saved_item = s->list_pointer->data; - s->state = CACHE_STATE_DYNAMIC_MAP; - } + case CACHE_STATE_NEGATIVE: + s->list_pointer = g_list_next (s->list_pointer); + if (s->list_pointer == NULL) { + if ((s->list_pointer = + check_dynamic_item (task, + cache)) || cache->dynamic_items != NULL) { + if (s->list_pointer == NULL) { + s->list_pointer = g_list_first (cache->dynamic_items); + s->saved_item = s->list_pointer->data; + s->state = CACHE_STATE_DYNAMIC; } else { - s->state = CACHE_STATE_STATIC; - s->list_pointer = g_list_first (cache->static_items); - if (s->list_pointer) { - s->saved_item = s->list_pointer->data; - } - else { - return FALSE; - } + s->saved_item = s->list_pointer->data; + s->state = CACHE_STATE_DYNAMIC_MAP; } } else { - s->saved_item = s->list_pointer->data; - } - item = s->saved_item; - break; - case CACHE_STATE_DYNAMIC_MAP: - s->list_pointer = g_list_next (s->list_pointer); - if (s->list_pointer == NULL) { - s->list_pointer = g_list_first (cache->dynamic_items); + s->state = CACHE_STATE_STATIC; + s->list_pointer = g_list_first (cache->static_items); if (s->list_pointer) { s->saved_item = s->list_pointer->data; - s->state = CACHE_STATE_DYNAMIC; } else { - s->state = CACHE_STATE_STATIC; - s->list_pointer = g_list_first (cache->static_items); - if (s->list_pointer) { - s->saved_item = s->list_pointer->data; - } - else { - return FALSE; - } + return FALSE; } } - else { + } + else { + s->saved_item = s->list_pointer->data; + } + item = s->saved_item; + break; + case CACHE_STATE_DYNAMIC_MAP: + s->list_pointer = g_list_next (s->list_pointer); + if (s->list_pointer == NULL) { + s->list_pointer = g_list_first (cache->dynamic_items); + if (s->list_pointer) { s->saved_item = s->list_pointer->data; + s->state = CACHE_STATE_DYNAMIC; } - item = s->saved_item; - break; - case CACHE_STATE_DYNAMIC: - s->list_pointer = g_list_next (s->list_pointer); - if (s->list_pointer == NULL) { + else { s->state = CACHE_STATE_STATIC; s->list_pointer = g_list_first (cache->static_items); if (s->list_pointer) { @@ -964,39 +1106,58 @@ call_symbol_callback (struct rspamd_task * task, struct symbols_cache * cache, g return FALSE; } } - else { - s->saved_item = s->list_pointer->data; - /* Skip items that are in negative map */ - while (s->list_pointer != NULL && check_negative_dynamic_item (task, cache, s->saved_item)) { - s->list_pointer = g_list_next (s->list_pointer); - if (s->list_pointer != NULL) { - s->saved_item = s->list_pointer->data; - } - } - if (s->list_pointer == NULL) { - s->state = CACHE_STATE_STATIC; - s->list_pointer = g_list_first (cache->static_items); - if (s->list_pointer) { - s->saved_item = s->list_pointer->data; - } - else { - return FALSE; - } - } - } - item = s->saved_item; - break; - case CACHE_STATE_STATIC: - /* Next pointer */ - s->list_pointer = g_list_next (s->list_pointer); + } + else { + s->saved_item = s->list_pointer->data; + } + item = s->saved_item; + break; + case CACHE_STATE_DYNAMIC: + s->list_pointer = g_list_next (s->list_pointer); + if (s->list_pointer == NULL) { + s->state = CACHE_STATE_STATIC; + s->list_pointer = g_list_first (cache->static_items); if (s->list_pointer) { s->saved_item = s->list_pointer->data; } else { return FALSE; } - item = s->saved_item; - break; + } + else { + s->saved_item = s->list_pointer->data; + /* Skip items that are in negative map */ + while (s->list_pointer != NULL && + check_negative_dynamic_item (task, cache, s->saved_item)) { + s->list_pointer = g_list_next (s->list_pointer); + if (s->list_pointer != NULL) { + s->saved_item = s->list_pointer->data; + } + } + if (s->list_pointer == NULL) { + s->state = CACHE_STATE_STATIC; + s->list_pointer = g_list_first (cache->static_items); + if (s->list_pointer) { + s->saved_item = s->list_pointer->data; + } + else { + return FALSE; + } + } + } + item = s->saved_item; + break; + case CACHE_STATE_STATIC: + /* Next pointer */ + s->list_pointer = g_list_next (s->list_pointer); + if (s->list_pointer) { + s->saved_item = s->list_pointer->data; + } + else { + return FALSE; + } + item = s->saved_item; + break; } } if (!item) { @@ -1007,9 +1168,9 @@ call_symbol_callback (struct rspamd_task * task, struct symbols_cache * cache, g # ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &ts1); # elif defined(HAVE_CLOCK_VIRTUAL) - clock_gettime (CLOCK_VIRTUAL, &ts1); + clock_gettime (CLOCK_VIRTUAL, &ts1); # else - clock_gettime (CLOCK_REALTIME, &ts1); + clock_gettime (CLOCK_REALTIME, &ts1); # endif #else if (gettimeofday (&tv1, NULL) == -1) { @@ -1030,9 +1191,9 @@ call_symbol_callback (struct rspamd_task * task, struct symbols_cache * cache, g # ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &ts2); # elif defined(HAVE_CLOCK_VIRTUAL) - clock_gettime (CLOCK_VIRTUAL, &ts2); + clock_gettime (CLOCK_VIRTUAL, &ts2); # else - clock_gettime (CLOCK_REALTIME, &ts2); + clock_gettime (CLOCK_REALTIME, &ts2); # endif #else if (gettimeofday (&tv2, NULL) == -1) { @@ -1041,9 +1202,12 @@ call_symbol_callback (struct rspamd_task * task, struct symbols_cache * cache, g #endif #ifdef HAVE_CLOCK_GETTIME - diff = (ts2.tv_sec - ts1.tv_sec) * 1000000 + (ts2.tv_nsec - ts1.tv_nsec) / 1000; + diff = + (ts2.tv_sec - + ts1.tv_sec) * 1000000 + (ts2.tv_nsec - ts1.tv_nsec) / 1000; #else - diff = (tv2.tv_sec - tv1.tv_sec) * 1000000 + (tv2.tv_usec - tv1.tv_usec); + diff = + (tv2.tv_sec - tv1.tv_sec) * 1000000 + (tv2.tv_usec - tv1.tv_usec); #endif item->s->avg_time = rspamd_set_counter (item->s->symbol, diff); } diff --git a/src/libserver/symbols_cache.h b/src/libserver/symbols_cache.h index b3835824d..e1fd3286f 100644 --- a/src/libserver/symbols_cache.h +++ b/src/libserver/symbols_cache.h @@ -32,7 +32,7 @@ struct cache_item { struct dynamic_map_item *networks; guint32 networks_number; gboolean is_dynamic; - + /* Callback data */ symbol_func_t func; gpointer user_data; @@ -53,7 +53,7 @@ struct symbols_cache { /* Items that have negative weights */ GList *negative_items; - + /* Radix map of dynamic rules with ip mappings */ radix_tree_t *dynamic_map; radix_tree_t *negative_dynamic_map; @@ -77,8 +77,11 @@ struct symbols_cache { /** * Load symbols cache from file, must be called _after_ init_symbols_cache */ -gboolean init_symbols_cache (rspamd_mempool_t *pool, struct symbols_cache *cache, struct rspamd_config *cfg, - const gchar *filename, gboolean ignore_checksum); +gboolean init_symbols_cache (rspamd_mempool_t *pool, + struct symbols_cache *cache, + struct rspamd_config *cfg, + const gchar *filename, + gboolean ignore_checksum); /** * Register function for symbols parsing @@ -86,15 +89,20 @@ gboolean init_symbols_cache (rspamd_mempool_t *pool, struct symbols_cache *cache * @param func pointer to handler * @param user_data pointer to user_data */ -void register_symbol (struct symbols_cache **cache, const gchar *name, double weight, - symbol_func_t func, gpointer user_data); +void register_symbol (struct symbols_cache **cache, + const gchar *name, + double weight, + symbol_func_t func, + gpointer user_data); /** * Register virtual symbol * @param name name of symbol */ -void register_virtual_symbol (struct symbols_cache **cache, const gchar *name, double weight); +void register_virtual_symbol (struct symbols_cache **cache, + const gchar *name, + double weight); /** * Register callback function for symbols parsing @@ -102,8 +110,11 @@ void register_virtual_symbol (struct symbols_cache **cache, const gchar *name, d * @param func pointer to handler * @param user_data pointer to user_data */ -void register_callback_symbol (struct symbols_cache **cache, const gchar *name, double weight, - symbol_func_t func, gpointer user_data); +void register_callback_symbol (struct symbols_cache **cache, + const gchar *name, + double weight, + symbol_func_t func, + gpointer user_data); /** * Register function for symbols parsing with strict priority @@ -111,8 +122,12 @@ void register_callback_symbol (struct symbols_cache **cache, const gchar *name, * @param func pointer to handler * @param user_data pointer to user_data */ -void register_callback_symbol_priority (struct symbols_cache **cache, const gchar *name, double weight, - gint priority, symbol_func_t func, gpointer user_data); +void register_callback_symbol_priority (struct symbols_cache **cache, + const gchar *name, + double weight, + gint priority, + symbol_func_t func, + gpointer user_data); /** * Register function for dynamic symbols parsing @@ -120,9 +135,13 @@ void register_callback_symbol_priority (struct symbols_cache **cache, const gcha * @param func pointer to handler * @param user_data pointer to user_data */ -void register_dynamic_symbol (rspamd_mempool_t *pool, struct symbols_cache **cache, const gchar *name, - double weight, symbol_func_t func, - gpointer user_data, GList *networks); +void register_dynamic_symbol (rspamd_mempool_t *pool, + struct symbols_cache **cache, + const gchar *name, + double weight, + symbol_func_t func, + gpointer user_data, + GList *networks); /** * Call function for cached symbol using saved callback @@ -130,7 +149,9 @@ void register_dynamic_symbol (rspamd_mempool_t *pool, struct symbols_cache **cac * @param cache symbols cache * @param saved_item pointer to currently saved item */ -gboolean call_symbol_callback (struct rspamd_task *task, struct symbols_cache *cache, gpointer *save); +gboolean call_symbol_callback (struct rspamd_task *task, + struct symbols_cache *cache, + gpointer *save); /** * Remove all dynamic rules from cache @@ -144,7 +165,9 @@ void remove_dynamic_rules (struct symbols_cache *cache); * @param cfg configuration * @param strict do strict checks - symbols MUST be described in metrics */ -gboolean validate_cache (struct symbols_cache *cache, struct rspamd_config *cfg, gboolean strict); +gboolean validate_cache (struct symbols_cache *cache, + struct rspamd_config *cfg, + gboolean strict); #endif diff --git a/src/libserver/task.c b/src/libserver/task.c index a46a70604..4e71a1710 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -21,12 +21,12 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "task.h" -#include "main.h" #include "filter.h" -#include "protocol.h" -#include "message.h" #include "lua/lua_common.h" +#include "main.h" +#include "message.h" +#include "protocol.h" +#include "task.h" /* * Destructor for recipients list in a task @@ -34,7 +34,7 @@ static void rcpt_destruct (void *pointer) { - struct rspamd_task *task = (struct rspamd_task *) pointer; + struct rspamd_task *task = (struct rspamd_task *) pointer; if (task->rcpt) { g_list_free (task->rcpt); @@ -44,10 +44,10 @@ rcpt_destruct (void *pointer) /* * Create new task */ -struct rspamd_task * +struct rspamd_task * rspamd_task_new (struct rspamd_worker *worker) { - struct rspamd_task *new_task; + struct rspamd_task *new_task; new_task = g_slice_alloc0 (sizeof (struct rspamd_task)); @@ -60,9 +60,9 @@ rspamd_task_new (struct rspamd_worker *worker) # ifdef HAVE_CLOCK_PROCESS_CPUTIME_ID clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &new_task->ts); # elif defined(HAVE_CLOCK_VIRTUAL) - clock_gettime (CLOCK_VIRTUAL, &new_task->ts); + clock_gettime (CLOCK_VIRTUAL, &new_task->ts); # else - clock_gettime (CLOCK_REALTIME, &new_task->ts); + clock_gettime (CLOCK_REALTIME, &new_task->ts); # endif #endif if (gettimeofday (&new_task->tv, NULL) == -1) { @@ -73,27 +73,28 @@ rspamd_task_new (struct rspamd_worker *worker) /* Add destructor for recipients list (it would be better to use anonymous function here */ rspamd_mempool_add_destructor (new_task->task_pool, - (rspamd_mempool_destruct_t) rcpt_destruct, new_task); + (rspamd_mempool_destruct_t) rcpt_destruct, new_task); new_task->results = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); rspamd_mempool_add_destructor (new_task->task_pool, - (rspamd_mempool_destruct_t) g_hash_table_destroy, - new_task->results); + (rspamd_mempool_destruct_t) g_hash_table_destroy, + new_task->results); new_task->re_cache = g_hash_table_new (rspamd_str_hash, rspamd_str_equal); rspamd_mempool_add_destructor (new_task->task_pool, - (rspamd_mempool_destruct_t) g_hash_table_destroy, - new_task->re_cache); - new_task->raw_headers = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal); + (rspamd_mempool_destruct_t) g_hash_table_destroy, + new_task->re_cache); + new_task->raw_headers = g_hash_table_new (rspamd_strcase_hash, + rspamd_strcase_equal); rspamd_mempool_add_destructor (new_task->task_pool, - (rspamd_mempool_destruct_t) g_hash_table_destroy, - new_task->raw_headers); + (rspamd_mempool_destruct_t) g_hash_table_destroy, + new_task->raw_headers); new_task->emails = g_tree_new (compare_email_func); rspamd_mempool_add_destructor (new_task->task_pool, - (rspamd_mempool_destruct_t) g_tree_destroy, - new_task->emails); + (rspamd_mempool_destruct_t) g_tree_destroy, + new_task->emails); new_task->urls = g_tree_new (compare_url_func); rspamd_mempool_add_destructor (new_task->task_pool, - (rspamd_mempool_destruct_t) g_tree_destroy, - new_task->urls); + (rspamd_mempool_destruct_t) g_tree_destroy, + new_task->urls); new_task->sock = -1; new_task->is_mime = TRUE; new_task->is_json = TRUE; @@ -123,7 +124,7 @@ rspamd_task_reply (struct rspamd_task *task) gboolean rspamd_task_fin (void *arg) { - struct rspamd_task *task = (struct rspamd_task *) arg; + struct rspamd_task *task = (struct rspamd_task *) arg; gint r; GError *err = NULL; @@ -205,7 +206,7 @@ rspamd_task_fin (void *arg) void rspamd_task_restore (void *arg) { - struct rspamd_task *task = (struct rspamd_task *) arg; + struct rspamd_task *task = (struct rspamd_task *) arg; /* Call post filters */ if (task->state == WAIT_POST_FILTER && !task->skip_extra_filters) { @@ -220,8 +221,8 @@ rspamd_task_restore (void *arg) void rspamd_task_free (struct rspamd_task *task, gboolean is_soft) { - GList *part; - struct mime_part *p; + GList *part; + struct mime_part *p; if (task) { debug_task ("free pointer %p", task); @@ -257,14 +258,16 @@ rspamd_task_free (struct rspamd_task *task, gboolean is_soft) } } -void rspamd_task_free_hard (gpointer ud) +void +rspamd_task_free_hard (gpointer ud) { struct rspamd_task *task = ud; rspamd_task_free (task, FALSE); } -void rspamd_task_free_soft (gpointer ud) +void +rspamd_task_free_soft (gpointer ud) { struct rspamd_task *task = ud; @@ -274,8 +277,8 @@ void rspamd_task_free_soft (gpointer ud) gboolean rspamd_task_process (struct rspamd_task *task, - struct rspamd_http_message *msg, GThreadPool *classify_pool, - gboolean process_extra_filters) + struct rspamd_http_message *msg, GThreadPool *classify_pool, + gboolean process_extra_filters) { gint r; GError *err = NULL; diff --git a/src/libserver/task.h b/src/libserver/task.h index 4c3462d9a..0f782b657 100644 --- a/src/libserver/task.h +++ b/src/libserver/task.h @@ -24,11 +24,11 @@ #define TASK_H_ #include "config.h" -#include "http.h" +#include "dns.h" #include "events.h" -#include "util.h" +#include "http.h" #include "mem_pool.h" -#include "dns.h" +#include "util.h" enum rspamd_command { CMD_CHECK, @@ -62,7 +62,7 @@ struct custom_command { * Worker task structure */ struct rspamd_task { - struct rspamd_worker *worker; /**< pointer to worker object */ + struct rspamd_worker *worker; /**< pointer to worker object */ enum { READ_MESSAGE, WAIT_PRE_FILTER, @@ -70,82 +70,82 @@ struct rspamd_task { WAIT_POST_FILTER, WRITE_REPLY, CLOSING_CONNECTION - } state; /**< current session state */ - enum rspamd_command cmd; /**< command */ - struct custom_command *custom_cmd; /**< custom command if any */ - gint sock; /**< socket descriptor */ + } state; /**< current session state */ + enum rspamd_command cmd; /**< command */ + struct custom_command *custom_cmd; /**< custom command if any */ + gint sock; /**< socket descriptor */ gboolean is_mime; /**< if this task is mime task */ - gboolean is_json; /**< output is JSON */ - gboolean skip_extra_filters; /**< skip pre and post filters */ + gboolean is_json; /**< output is JSON */ + gboolean skip_extra_filters; /**< skip pre and post filters */ gboolean is_skipped; /**< whether message was skipped by configuration */ - gchar *helo; /**< helo header value */ - gchar *from; /**< from header value */ - gchar *queue_id; /**< queue id if specified */ - const gchar *message_id; /**< message id */ - GList *rcpt; /**< recipients list */ - guint nrcpt; /**< number of recipients */ - rspamd_inet_addr_t from_addr; /**< from addr for a task */ - rspamd_inet_addr_t client_addr; /**< address of connected socket */ - gchar *deliver_to; /**< address to deliver */ - gchar *user; /**< user to deliver */ - gchar *subject; /**< subject (for non-mime) */ - gchar *hostname; /**< hostname reported by MTA */ - GString *msg; /**< message buffer */ - struct rspamd_http_connection *http_conn; /**< HTTP server connection */ - struct rspamd_async_session* s; /**< async session object */ - gint parts_count; /**< mime parts count */ - GMimeMessage *message; /**< message, parsed with GMime */ - GMimeObject *parser_parent_part; /**< current parent part */ - InternetAddressList *rcpts; /**< list of all recipients */ - GList *parts; /**< list of parsed parts */ - GList *text_parts; /**< list of text parts */ - gchar *raw_headers_str; /**< list of raw headers */ - GList *received; /**< list of received headers */ - GTree *urls; /**< list of parsed urls */ - GTree *emails; /**< list of parsed emails */ - GList *images; /**< list of images */ - GHashTable *raw_headers; /**< list of raw headers */ - GHashTable *results; /**< hash table of metric_result indexed by - * metric's name */ - GHashTable *tokens; /**< hash table of tokens indexed by tokenizer - * pointer */ - GList *messages; /**< list of messages that would be reported */ - GHashTable *re_cache; /**< cache for matched or not matched regexps */ - struct rspamd_config *cfg; /**< pointer to config object */ - gchar *last_error; /**< last error */ - gint error_code; /**< code of last error */ - rspamd_mempool_t *task_pool; /**< memory pool for task */ + gchar *helo; /**< helo header value */ + gchar *from; /**< from header value */ + gchar *queue_id; /**< queue id if specified */ + const gchar *message_id; /**< message id */ + GList *rcpt; /**< recipients list */ + guint nrcpt; /**< number of recipients */ + rspamd_inet_addr_t from_addr; /**< from addr for a task */ + rspamd_inet_addr_t client_addr; /**< address of connected socket */ + gchar *deliver_to; /**< address to deliver */ + gchar *user; /**< user to deliver */ + gchar *subject; /**< subject (for non-mime) */ + gchar *hostname; /**< hostname reported by MTA */ + GString *msg; /**< message buffer */ + struct rspamd_http_connection *http_conn; /**< HTTP server connection */ + struct rspamd_async_session * s; /**< async session object */ + gint parts_count; /**< mime parts count */ + GMimeMessage *message; /**< message, parsed with GMime */ + GMimeObject *parser_parent_part; /**< current parent part */ + InternetAddressList *rcpts; /**< list of all recipients */ + GList *parts; /**< list of parsed parts */ + GList *text_parts; /**< list of text parts */ + gchar *raw_headers_str; /**< list of raw headers */ + GList *received; /**< list of received headers */ + GTree *urls; /**< list of parsed urls */ + GTree *emails; /**< list of parsed emails */ + GList *images; /**< list of images */ + GHashTable *raw_headers; /**< list of raw headers */ + GHashTable *results; /**< hash table of metric_result indexed by + * metric's name */ + GHashTable *tokens; /**< hash table of tokens indexed by tokenizer + * pointer */ + GList *messages; /**< list of messages that would be reported */ + GHashTable *re_cache; /**< cache for matched or not matched regexps */ + struct rspamd_config *cfg; /**< pointer to config object */ + gchar *last_error; /**< last error */ + gint error_code; /**< code of last error */ + rspamd_mempool_t *task_pool; /**< memory pool for task */ #ifdef HAVE_CLOCK_GETTIME - struct timespec ts; /**< time of connection */ + struct timespec ts; /**< time of connection */ #endif - struct timeval tv; /**< time of connection */ - guint32 scan_milliseconds; /**< how much milliseconds passed */ - gboolean pass_all_filters; /**< pass task throught every rule */ - gboolean no_log; /**< do not log or write this task to the history */ - guint32 parser_recursion; /**< for avoiding recursion stack overflow */ - gboolean (*fin_callback)(void *arg); /**< calback for filters finalizing */ - void *fin_arg; /**< argument for fin callback */ + struct timeval tv; /**< time of connection */ + guint32 scan_milliseconds; /**< how much milliseconds passed */ + gboolean pass_all_filters; /**< pass task throught every rule */ + gboolean no_log; /**< do not log or write this task to the history */ + guint32 parser_recursion; /**< for avoiding recursion stack overflow */ + gboolean (*fin_callback)(void *arg); /**< calback for filters finalizing */ + void *fin_arg; /**< argument for fin callback */ - guint32 dns_requests; /**< number of DNS requests per this task */ + guint32 dns_requests; /**< number of DNS requests per this task */ - struct rspamd_dns_resolver *resolver; /**< DNS resolver */ - struct event_base *ev_base; /**< Event base */ + struct rspamd_dns_resolver *resolver; /**< DNS resolver */ + struct event_base *ev_base; /**< Event base */ - GThreadPool *classify_pool; /**< A pool of classify threads */ + GThreadPool *classify_pool; /**< A pool of classify threads */ struct { - enum rspamd_metric_action action; /**< Action of pre filters */ - gchar *str; /**< String describing action */ - } pre_result; /**< Result of pre-filters */ + enum rspamd_metric_action action; /**< Action of pre filters */ + gchar *str; /**< String describing action */ + } pre_result; /**< Result of pre-filters */ - ucl_object_t *settings; /**< Settings applied to task */ + ucl_object_t *settings; /**< Settings applied to task */ }; /** * Construct new task for worker */ -struct rspamd_task* rspamd_task_new (struct rspamd_worker *worker); +struct rspamd_task * rspamd_task_new (struct rspamd_worker *worker); /** * Destroy task object and remove its IO dispatcher if it exists */ @@ -173,7 +173,7 @@ gboolean rspamd_task_fin (void *arg); * @return task has been successfully parsed and processed */ gboolean rspamd_task_process (struct rspamd_task *task, - struct rspamd_http_message *msg, GThreadPool *classify_pool, - gboolean process_extra_filters); + struct rspamd_http_message *msg, GThreadPool *classify_pool, + gboolean process_extra_filters); #endif /* TASK_H_ */ diff --git a/src/libserver/url.c b/src/libserver/url.c index c4313e8a9..b481f6e2b 100644 --- a/src/libserver/url.c +++ b/src/libserver/url.c @@ -23,12 +23,12 @@ */ #include "config.h" -#include "url.h" -#include "util.h" #include "fstring.h" #include "main.h" #include "message.h" #include "trie.h" +#include "url.h" +#include "util.h" #define POST_CHAR 1 #define POST_CHAR_S "\001" @@ -38,16 +38,16 @@ #define HIGHEST_PORT 65535 #define uri_port_is_valid(port) \ - (LOWEST_PORT <= (port) && (port) <= HIGHEST_PORT) + (LOWEST_PORT <= (port) && (port) <= HIGHEST_PORT) struct _proto { - guchar *name; - gint port; - uintptr_t *unused; - guint need_slashes:1; - guint need_slash_after_host:1; - guint free_syntax:1; - guint need_ssl:1; + guchar *name; + gint port; + uintptr_t *unused; + guint need_slashes : 1; + guint need_slash_after_host : 1; + guint free_syntax : 1; + guint need_ssl : 1; }; typedef struct url_match_s { @@ -64,314 +64,627 @@ typedef struct url_match_s { struct url_matcher { const gchar *pattern; const gchar *prefix; - gboolean (*start)(const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match); - gboolean (*end)(const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match); + gboolean (*start)(const gchar *begin, const gchar *end, const gchar *pos, + url_match_t *match); + gboolean (*end)(const gchar *begin, const gchar *end, const gchar *pos, + url_match_t *match); gint flags; }; -static gboolean url_file_start (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match); -static gboolean url_file_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match); - -static gboolean url_web_start (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match); -static gboolean url_web_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match); - -static gboolean url_tld_start (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match); -static gboolean url_tld_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match); - -static gboolean url_email_start (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match); -static gboolean url_email_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match); +static gboolean url_file_start (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match); +static gboolean url_file_end (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match); + +static gboolean url_web_start (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match); +static gboolean url_web_end (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match); + +static gboolean url_tld_start (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match); +static gboolean url_tld_end (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match); + +static gboolean url_email_start (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match); +static gboolean url_email_end (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match); struct url_matcher matchers[] = { - /* Common prefixes */ - { "file://", "", url_file_start, url_file_end, 0 }, - { "ftp://", "", url_web_start, url_web_end, 0 }, - { "sftp://", "", url_web_start, url_web_end, 0 }, - { "http://", "", url_web_start, url_web_end, 0 }, - { "https://", "", url_web_start, url_web_end, 0 }, - { "news://", "", url_web_start, url_web_end, 0 }, - { "nntp://", "", url_web_start, url_web_end, 0 }, - { "telnet://", "", url_web_start, url_web_end, 0 }, - { "webcal://", "", url_web_start, url_web_end, 0 }, - { "mailto://", "", url_email_start, url_email_end, 0 }, - { "callto://", "", url_web_start, url_web_end, 0 }, - { "h323:", "", url_web_start, url_web_end, 0 }, - { "sip:", "", url_web_start, url_web_end, 0 }, - { "www.", "http://", url_web_start, url_web_end, 0 }, - { "ftp.", "ftp://", url_web_start, url_web_end, URL_FLAG_NOHTML }, - /* TLD domains parts */ - { ".ac", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ad", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ae", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".aero", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".af", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ag", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ai", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".al", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".am", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".an", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ao", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".aq", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ar", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".arpa", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".as", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".asia", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".at", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".au", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".aw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ax", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".az", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ba", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bb", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bd", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".be", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bf", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bg", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bh", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bi", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".biz", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bj", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bn", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bo", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".br", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bs", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bt", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bv", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".by", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".bz", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ca", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cat", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cc", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cd", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cf", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cg", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ch", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ci", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ck", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cl", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cn", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".co", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".com", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".coop", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cu", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cv", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cx", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cy", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".cz", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".de", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".dj", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".dk", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".dm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".do", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".dz", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ec", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".edu", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ee", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".eg", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".er", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".es", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".et", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".eu", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".fi", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".fj", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".fk", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".fm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".fo", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".fr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ga", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gb", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gd", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ge", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gf", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gg", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gh", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gi", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gl", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gn", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gov", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gp", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gq", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gs", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gt", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gu", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".gy", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".hk", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".hm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".hn", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".hr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ht", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".hu", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".id", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ie", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".il", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".im", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".in", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".info", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".int", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".io", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".iq", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ir", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".is", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".it", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".je", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".jm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".jo", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".jobs", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".jp", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ke", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".kg", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".kh", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ki", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".km", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".kn", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".kp", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".kr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".kw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ky", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".kz", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".la", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".lb", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".lc", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".li", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".lk", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".lr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ls", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".lt", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".lu", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".lv", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ly", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ma", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mc", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".md", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".me", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mg", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mh", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mil", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mk", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ml", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mn", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mo", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mobi", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mp", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mq", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ms", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mt", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mu", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".museum", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mv", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mx", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".my", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".mz", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".na", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".name", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".nc", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ne", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".net", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".nf", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ng", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ni", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".nl", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".no", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".np", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".nr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".nu", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".nz", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".om", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".org", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pa", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pe", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pf", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pg", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ph", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pk", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pl", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pn", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pro", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ps", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pt", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".pw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".py", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".qa", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".re", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ro", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".rs", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ru", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".rw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sa", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sb", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sc", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sd", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".se", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sg", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sh", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".si", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sj", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sk", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sl", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sn", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".so", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".st", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".su", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sv", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sx", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sy", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".sz", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tc", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".td", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tel", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tf", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tg", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".th", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tj", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tk", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tl", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tn", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".to", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tp", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tr", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".travel", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tt", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tv", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".tz", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ua", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ug", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".uk", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".us", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".uy", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".uz", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".va", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".vc", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ve", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".vg", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".vi", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".vn", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".vu", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".wf", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ws", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".xxx", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".ye", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".yt", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".za", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".zm", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - { ".zw", "http://", url_tld_start, url_tld_end, URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, - /* Likely emails */ - { "@", "mailto://",url_email_start, url_email_end, URL_FLAG_NOHTML } + /* Common prefixes */ + { "file://", "", url_file_start, url_file_end, + 0 }, + { "ftp://", "", url_web_start, url_web_end, + 0 }, + { "sftp://", "", url_web_start, url_web_end, + 0 }, + { "http://", "", url_web_start, url_web_end, + 0 }, + { "https://", "", url_web_start, url_web_end, + 0 }, + { "news://", "", url_web_start, url_web_end, + 0 }, + { "nntp://", "", url_web_start, url_web_end, + 0 }, + { "telnet://", "", url_web_start, url_web_end, + 0 }, + { "webcal://", "", url_web_start, url_web_end, + 0 }, + { "mailto://", "", url_email_start, url_email_end, + 0 }, + { "callto://", "", url_web_start, url_web_end, + 0 }, + { "h323:", "", url_web_start, url_web_end, + 0 }, + { "sip:", "", url_web_start, url_web_end, + 0 }, + { "www.", "http://", url_web_start, url_web_end, + 0 }, + { "ftp.", "ftp://", url_web_start, url_web_end, + URL_FLAG_NOHTML }, + /* TLD domains parts */ + { ".ac", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ad", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ae", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".aero", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".af", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ag", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ai", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".al", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".am", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".an", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ao", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".aq", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ar", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".arpa", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".as", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".asia", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".at", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".au", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".aw", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ax", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".az", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ba", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bb", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bd", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".be", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bf", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bg", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bh", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bi", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".biz", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bj", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bn", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bo", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".br", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bs", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bt", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bv", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bw", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".by", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".bz", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ca", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cat", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cc", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cd", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cf", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cg", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ch", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ci", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ck", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cl", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cn", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".co", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".com", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".coop", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cu", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cv", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cw", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cx", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cy", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".cz", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".de", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".dj", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".dk", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".dm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".do", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".dz", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ec", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".edu", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ee", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".eg", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".er", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".es", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".et", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".eu", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".fi", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".fj", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".fk", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".fm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".fo", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".fr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ga", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gb", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gd", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ge", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gf", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gg", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gh", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gi", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gl", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gn", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gov", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gp", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gq", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gs", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gt", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gu", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gw", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".gy", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".hk", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".hm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".hn", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".hr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ht", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".hu", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".id", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ie", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".il", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".im", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".in", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".info", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".int", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".io", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".iq", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ir", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".is", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".it", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".je", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".jm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".jo", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".jobs", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".jp", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ke", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".kg", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".kh", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ki", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".km", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".kn", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".kp", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".kr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".kw", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ky", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".kz", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".la", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".lb", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".lc", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".li", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".lk", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".lr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ls", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".lt", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".lu", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".lv", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ly", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ma", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mc", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".md", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".me", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mg", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mh", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mil", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mk", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ml", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mn", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mo", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mobi", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mp", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mq", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ms", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mt", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mu", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".museum", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mv", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mw", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mx", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".my", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".mz", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".na", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".name", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".nc", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ne", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".net", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".nf", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ng", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ni", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".nl", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".no", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".np", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".nr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".nu", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".nz", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".om", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".org", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pa", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pe", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pf", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pg", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ph", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pk", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pl", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pn", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pro", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ps", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pt", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".pw", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".py", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".qa", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".re", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ro", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".rs", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ru", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".rw", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sa", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sb", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sc", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sd", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".se", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sg", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sh", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".si", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sj", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sk", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sl", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sn", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".so", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".st", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".su", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sv", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sx", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sy", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".sz", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tc", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".td", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tel", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tf", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tg", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".th", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tj", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tk", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tl", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tn", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".to", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tp", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tr", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".travel", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tt", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tv", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tw", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".tz", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ua", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ug", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".uk", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".us", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".uy", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".uz", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".va", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".vc", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ve", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".vg", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".vi", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".vn", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".vu", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".wf", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ws", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".xxx", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".ye", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".yt", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".za", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".zm", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + { ".zw", "http://", url_tld_start, url_tld_end, + URL_FLAG_NOHTML | URL_FLAG_STRICT_MATCH }, + /* Likely emails */ + { "@", "mailto://",url_email_start, url_email_end, + URL_FLAG_NOHTML } }; struct url_match_scanner { @@ -382,7 +695,7 @@ struct url_match_scanner { struct url_match_scanner *url_scanner = NULL; -static const struct _proto protocol_backends[] = { +static const struct _proto protocol_backends[] = { {"file", 0, NULL, 1, 0, 0, 0}, {"ftp", 21, NULL, 1, 0, 0, 0}, {"http", 80, NULL, 1, 0, 0, 0}, @@ -395,7 +708,8 @@ static const struct _proto protocol_backends[] = { /* Convert an ASCII hex digit to the corresponding number between 0 and 15. H should be a hexadecimal digit that satisfies isxdigit; otherwise, the result is undefined. */ -#define XDIGIT_TO_NUM(h) ((h) < 'A' ? (h) - '0' : g_ascii_toupper (h) - 'A' + 10) +#define XDIGIT_TO_NUM(h) ((h) < 'A' ? (h) - '0' : g_ascii_toupper (h) - 'A' + \ + 10) #define X2DIGITS_TO_NUM(h1, h2) ((XDIGIT_TO_NUM (h1) << 4) + XDIGIT_TO_NUM (h2)) /* The reverse of the above: convert a number in the [0, 16) range to the ASCII representation of the corresponding hexadecimal digit. @@ -404,45 +718,47 @@ static const struct _proto protocol_backends[] = { #define XNUM_TO_digit(x) ("0123456789abcdef"[x] + 0) static guchar url_scanner_table[256] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 1, 1, 9, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 24,128,160,128,128,128,128,128,160,160,128,128,160,192,160,160, - 68, 68, 68, 68, 68, 68, 68, 68, 68, 68,160,160, 32,128, 32,128, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 1, 1, 9, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 24,128,160,128,128,128,128,128,160,160,128,128,160,192,160,160, + 68, 68, 68, 68, 68, 68, 68, 68, 68, 68,160,160, 32,128, 32,128, 160, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, - 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66,160,160,160,128,192, + 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66,160,160,160,128,192, 128, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, - 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66,128,128,128,128, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, 66,128,128,128,128, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; enum { - IS_CTRL = (1 << 0), - IS_ALPHA = (1 << 1), - IS_DIGIT = (1 << 2), - IS_LWSP = (1 << 3), - IS_SPACE = (1 << 4), - IS_SPECIAL = (1 << 5), - IS_DOMAIN = (1 << 6), - IS_URLSAFE = (1 << 7) + IS_CTRL = (1 << 0), + IS_ALPHA = (1 << 1), + IS_DIGIT = (1 << 2), + IS_LWSP = (1 << 3), + IS_SPACE = (1 << 4), + IS_SPECIAL = (1 << 5), + IS_DOMAIN = (1 << 6), + IS_URLSAFE = (1 << 7) }; #define is_ctrl(x) ((url_scanner_table[(guchar)(x)] & IS_CTRL) != 0) #define is_lwsp(x) ((url_scanner_table[(guchar)(x)] & IS_LWSP) != 0) -#define is_atom(x) ((url_scanner_table[(guchar)(x)] & (IS_SPECIAL|IS_SPACE|IS_CTRL)) == 0) +#define is_atom(x) ((url_scanner_table[(guchar)(x)] & (IS_SPECIAL | IS_SPACE | \ + IS_CTRL)) == 0) #define is_alpha(x) ((url_scanner_table[(guchar)(x)] & IS_ALPHA) != 0) #define is_digit(x) ((url_scanner_table[(guchar)(x)] & IS_DIGIT) != 0) #define is_domain(x) ((url_scanner_table[(guchar)(x)] & IS_DOMAIN) != 0) -#define is_urlsafe(x) ((url_scanner_table[(guchar)(x)] & (IS_ALPHA|IS_DIGIT|IS_URLSAFE)) != 0) +#define is_urlsafe(x) ((url_scanner_table[(guchar)(x)] & (IS_ALPHA | IS_DIGIT | \ + IS_URLSAFE)) != 0) -const gchar * +const gchar * url_strerror (enum uri_errno err) { switch (err) { @@ -477,7 +793,7 @@ url_strerror (enum uri_errno err) static gint check_uri_file (gchar *name) { - static const gchar chars[] = POST_CHAR_S "#?"; + static const gchar chars[] = POST_CHAR_S "#?"; return strcspn (name, chars); } @@ -485,30 +801,41 @@ check_uri_file (gchar *name) static gint url_init (void) { - guint i; - gchar patbuf[128]; + guint i; + gchar patbuf[128]; if (url_scanner == NULL) { url_scanner = g_malloc (sizeof (struct url_match_scanner)); url_scanner->matchers = matchers; url_scanner->matchers_count = G_N_ELEMENTS (matchers); url_scanner->patterns = rspamd_trie_create (TRUE); - for (i = 0; i < url_scanner->matchers_count; i ++) { + for (i = 0; i < url_scanner->matchers_count; i++) { if (matchers[i].flags & URL_FLAG_STRICT_MATCH) { /* Insert more specific patterns */ /* some.tld/ */ - rspamd_snprintf (patbuf, sizeof (patbuf), "%s/", matchers[i].pattern); + rspamd_snprintf (patbuf, + sizeof (patbuf), + "%s/", + matchers[i].pattern); rspamd_trie_insert (url_scanner->patterns, patbuf, i); /* some.tld */ - rspamd_snprintf (patbuf, sizeof (patbuf), "%s ", matchers[i].pattern); + rspamd_snprintf (patbuf, + sizeof (patbuf), + "%s ", + matchers[i].pattern); rspamd_trie_insert (url_scanner->patterns, patbuf, i); /* some.tld: */ - rspamd_snprintf (patbuf, sizeof (patbuf), "%s:", matchers[i].pattern); + rspamd_snprintf (patbuf, + sizeof (patbuf), + "%s:", + matchers[i].pattern); rspamd_trie_insert (url_scanner->patterns, patbuf, i); } else { - rspamd_trie_insert (url_scanner->patterns, matchers[i].pattern, i); + rspamd_trie_insert (url_scanner->patterns, + matchers[i].pattern, + i); } } } @@ -522,10 +849,10 @@ get_protocol (gchar *name, gint namelen) /* These are really enum protocol values but can take on negative * values and since 0 <= -1 for enum values it's better to use clean * integer type. */ - gint start, end; - enum protocol protocol; - guchar *pname; - gint pnamelen, minlen, compare; + gint start, end; + enum protocol protocol; + guchar *pname; + gint pnamelen, minlen, compare; /* Almost dichotomic search is used here */ /* Starting at the HTTP entry which is the most common that will make @@ -590,7 +917,7 @@ get_protocol_free_syntax (enum protocol protocol) static gint get_protocol_length (const gchar *url) { - gchar *end = (gchar *)url; + gchar *end = (gchar *)url; /* Seek the end of the protocol name if any. */ /* RFC1738: @@ -610,11 +937,12 @@ get_protocol_length (const gchar *url) static guint url_calculate_escaped_hostlen (gchar *host, guint hostlen) { - guint i, result = hostlen; - gchar *p = host, c; + guint i, result = hostlen; + gchar *p = host, c; for (i = 0; i < hostlen; i++, p++) { - if (*p == '%' && g_ascii_isxdigit (*(p + 1)) && g_ascii_isxdigit (*(p + 2)) && i < hostlen - 2) { + if (*p == '%' && g_ascii_isxdigit (*(p + 1)) && + g_ascii_isxdigit (*(p + 2)) && i < hostlen - 2) { c = X2DIGITS_TO_NUM (*(p + 1), *(p + 2)); if (c != '\0') { result -= 2; @@ -637,18 +965,19 @@ url_calculate_escaped_hostlen (gchar *host, guint hostlen) static void url_unescape (gchar *s) { - gchar *t = s; /* t - tortoise */ - gchar *h = s; /* h - hare */ + gchar *t = s; /* t - tortoise */ + gchar *h = s; /* h - hare */ for (; *h; h++, t++) { if (*h != '%') { - copychar: +copychar: *t = *h; } else { - gchar c; + gchar c; /* Do nothing if '%' is not followed by two hex digits. */ - if (!h[1] || !h[2] || !(g_ascii_isxdigit (h[1]) && g_ascii_isxdigit (h[2]))) + if (!h[1] || !h[2] || + !(g_ascii_isxdigit (h[1]) && g_ascii_isxdigit (h[2]))) goto copychar; c = X2DIGITS_TO_NUM (h[1], h[2]); /* Don't unescape %00 because there is no way to insert it @@ -665,8 +994,8 @@ url_unescape (gchar *s) static void url_strip (gchar *s) { - gchar *t = s; /* t - tortoise */ - gchar *h = s; /* h - hare */ + gchar *t = s; /* t - tortoise */ + gchar *h = s; /* h - hare */ while (*h) { if (g_ascii_isgraph (*h)) { @@ -678,17 +1007,17 @@ url_strip (gchar *s) *t = '\0'; } -static gchar * +static gchar * url_escape_1 (const gchar *s, gint allow_passthrough, rspamd_mempool_t * pool) { - const gchar *p1; - gchar *p2, *newstr; - gint newlen; - gint addition = 0; + const gchar *p1; + gchar *p2, *newstr; + gint newlen; + gint addition = 0; for (p1 = s; *p1; p1++) if (!is_urlsafe (*p1)) { - addition += 2; /* Two more characters (hex digits) */ + addition += 2; /* Two more characters (hex digits) */ } if (!addition) { @@ -708,7 +1037,7 @@ url_escape_1 (const gchar *s, gint allow_passthrough, rspamd_mempool_t * pool) while (*p1) { /* Quote the characters that match the test mask. */ if (!is_urlsafe (*p1)) { - guchar c = *p1++; + guchar c = *p1++; *p2++ = '%'; *p2++ = XNUM_TO_DIGIT (c >> 4); *p2++ = XNUM_TO_DIGIT (c & 0xf); @@ -724,7 +1053,7 @@ url_escape_1 (const gchar *s, gint allow_passthrough, rspamd_mempool_t * pool) /* URL-escape the unsafe characters (see urlchr_table) in a given string, returning a freshly allocated string. */ -gchar * +gchar * url_escape (const gchar *s, rspamd_mempool_t * pool) { return url_escape_1 (s, 0, pool); @@ -747,7 +1076,7 @@ char_needs_escaping (const gchar *p) return TRUE; } } - else if (! is_urlsafe (*p)) { + else if (!is_urlsafe (*p)) { return TRUE; } return FALSE; @@ -755,16 +1084,16 @@ char_needs_escaping (const gchar *p) /* Translate a %-escaped (but possibly non-conformant) input string S into a %-escaped (and conformant) output string. -*/ + */ -static gchar * +static gchar * reencode_escapes (gchar *s, rspamd_mempool_t * pool) { - const gchar *p1; - gchar *newstr, *p2; - gint oldlen, newlen; + const gchar *p1; + gchar *newstr, *p2; + gint oldlen, newlen; - gint encode_count = 0; + gint encode_count = 0; /* First pass: inspect the string to see if there's anything to do, and to calculate the new length. */ @@ -789,7 +1118,7 @@ reencode_escapes (gchar *s, rspamd_mempool_t * pool) while (*p1) if (char_needs_escaping (p1)) { - guchar c = *p1++; + guchar c = *p1++; *p2++ = '%'; *p2++ = XNUM_TO_DIGIT (c >> 4); *p2++ = XNUM_TO_DIGIT (c & 0xf); @@ -809,10 +1138,10 @@ reencode_escapes (gchar *s, rspamd_mempool_t * pool) static void unescape_single_char (gchar *str, gchar chr) { - const gchar c1 = XNUM_TO_DIGIT (chr >> 4); - const gchar c2 = XNUM_TO_DIGIT (chr & 0xf); - gchar *h = str; /* hare */ - gchar *t = str; /* tortoise */ + const gchar c1 = XNUM_TO_DIGIT (chr >> 4); + const gchar c2 = XNUM_TO_DIGIT (chr & 0xf); + gchar *h = str; /* hare */ + gchar *t = str; /* tortoise */ for (; *h; h++, t++) { if (h[0] == '%' && h[1] == c1 && h[2] == c2) { @@ -835,10 +1164,10 @@ unescape_single_char (gchar *str, gchar chr) static gboolean path_simplify (gchar *path) { - gchar *h = path; /* hare */ - gchar *t = path; /* tortoise */ - gchar *beg = path; /* boundary for backing the tortoise */ - gchar *end = path + strlen (path); + gchar *h = path; /* hare */ + gchar *t = path; /* tortoise */ + gchar *beg = path; /* boundary for backing the tortoise */ + gchar *end = path + strlen (path); while (h < end) { /* Hare should be at the beginning of a path element. */ @@ -852,7 +1181,7 @@ path_simplify (gchar *path) if (t > beg) { /* Move backwards until T hits the beginning of the previous path element or the beginning of path. */ - for (--t; t > beg && t[-1] != '/'; t--); + for (--t; t > beg && t[-1] != '/'; t--) ; } else { /* If we're at the beginning, copy the "../" literally @@ -864,7 +1193,7 @@ path_simplify (gchar *path) h += 3; } else { - regular: +regular: /* A regular path element. If H hasn't advanced past T, simply skip to the next path element. Otherwise, copy the path element until the next slash. */ @@ -894,10 +1223,10 @@ path_simplify (gchar *path) enum uri_errno parse_uri (struct uri *uri, gchar *uristring, rspamd_mempool_t * pool) { - guchar *prefix_end, *host_end, *p; - guchar *lbracket, *rbracket; - gint datalen, n, addrlen; - guchar *frag_or_post, *user_end, *port_end; + guchar *prefix_end, *host_end, *p; + guchar *lbracket, *rbracket; + gint datalen, n, addrlen; + guchar *frag_or_post, *user_end, *port_end; memset (uri, 0, sizeof (*uri)); @@ -910,11 +1239,13 @@ parse_uri (struct uri *uri, gchar *uristring, rspamd_mempool_t * pool) uri->protocollen = get_protocol_length (struri (uri)); /* Assume http as default protocol */ - if (!uri->protocollen || (uri->protocol = get_protocol (struri (uri), uri->protocollen)) == PROTOCOL_UNKNOWN) { + if (!uri->protocollen || + (uri->protocol = + get_protocol (struri (uri), uri->protocollen)) == PROTOCOL_UNKNOWN) { /* Make exception for numeric urls */ p = uri->string; while (*p && (g_ascii_isalnum (*p) || *p == ':')) { - p ++; + p++; } if (*p == '\0') { return URI_ERRNO_INVALID_PROTOCOL; @@ -929,7 +1260,7 @@ parse_uri (struct uri *uri, gchar *uristring, rspamd_mempool_t * pool) /* Figure out whether the protocol is known */ msg_debug ("getting protocol from url: %d", uri->protocol); - prefix_end = struri (uri) + uri->protocollen; /* ':' */ + prefix_end = struri (uri) + uri->protocollen; /* ':' */ /* Check if there's a digit after the protocol name. */ if (g_ascii_isdigit (*prefix_end)) { @@ -1010,7 +1341,7 @@ parse_uri (struct uri *uri, gchar *uristring, rspamd_mempool_t * pool) /* Possibly skip auth part */ host_end = prefix_end + strcspn (prefix_end, "@"); - if (prefix_end + strcspn (prefix_end, "/?") > host_end && *host_end) { /* we have auth info here */ + if (prefix_end + strcspn (prefix_end, "/?") > host_end && *host_end) { /* we have auth info here */ /* Allow '@' in the password component */ while (strcspn (host_end + 1, "@") < strcspn (host_end + 1, "/?")) @@ -1054,7 +1385,7 @@ parse_uri (struct uri *uri, gchar *uristring, rspamd_mempool_t * pool) return URI_ERRNO_TRAILING_DOTS; } - if (*host_end == ':') { /* we have port here */ + if (*host_end == ':') { /* we have port here */ port_end = host_end + 1 + strcspn (host_end + 1, "/"); host_end++; @@ -1089,7 +1420,8 @@ parse_uri (struct uri *uri, gchar *uristring, rspamd_mempool_t * pool) host_end++; } - else if (get_protocol_need_slash_after_host (uri->protocol) && *host_end != '?') { + else if (get_protocol_need_slash_after_host (uri->protocol) && *host_end != + '?') { /* The need for slash after the host component depends on the * need for a host component. -- The dangerous mind of Jonah */ if (!uri->hostlen) @@ -1114,7 +1446,7 @@ parse_uri (struct uri *uri, gchar *uristring, rspamd_mempool_t * pool) } convert_to_lowercase (uri->string, uri->protocollen); - convert_to_lowercase (uri->host, uri->hostlen); + convert_to_lowercase (uri->host, uri->hostlen); /* Decode %HH sequences in host name. This is important not so much to support %HH sequences in host names (which other browser don't), but to support binary characters (which will have been @@ -1132,12 +1464,12 @@ parse_uri (struct uri *uri, gchar *uristring, rspamd_mempool_t * pool) } static const gchar url_braces[] = { - '(', ')' , - '{', '}' , - '[', ']' , - '<', '>' , - '|', '|' , - '\'', '\'' + '(', ')', + '{', '}', + '[', ']', + '<', '>', + '|', '|', + '\'', '\'' }; static gboolean @@ -1156,22 +1488,28 @@ is_open_brace (gchar c) } static gboolean -url_file_start (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match) +url_file_start (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match) { match->m_begin = pos; return TRUE; } static gboolean -url_file_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match) +url_file_end (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match) { - const gchar *p; - gchar stop; - guint i; + const gchar *p; + gchar stop; + guint i; p = pos + strlen (match->pattern); stop = *p; if (*p == '/') { - p ++; + p++; } for (i = 0; i < G_N_ELEMENTS (url_braces) / 2; i += 2) { @@ -1182,7 +1520,7 @@ url_file_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_ } while (p < end && *p != stop && is_urlsafe (*p)) { - p ++; + p++; } if (p == begin) { @@ -1195,14 +1533,18 @@ url_file_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_ } static gboolean -url_tld_start (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match) +url_tld_start (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match) { - const gchar *p = pos; + const gchar *p = pos; /* Try to find the start of the url by finding any non-urlsafe character or whitespace/punctuation */ while (p >= begin) { - if ((!is_domain (*p) && *p != '.' && *p != '/') || g_ascii_isspace (*p)) { - p ++; + if ((!is_domain (*p) && *p != '.' && + *p != '/') || g_ascii_isspace (*p)) { + p++; if (!g_ascii_isalnum (*p)) { /* Urls cannot start with strange symbols */ return FALSE; @@ -1228,16 +1570,19 @@ url_tld_start (const gchar *begin, const gchar *end, const gchar *pos, url_match /* Urls cannot contain '/' in their body */ return FALSE; } - p --; + p--; } return FALSE; } static gboolean -url_tld_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match) +url_tld_end (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match) { - const gchar *p; + const gchar *p; /* A url must be finished by tld, so it must be followed by space character */ p = pos + strlen (match->pattern); @@ -1250,7 +1595,10 @@ url_tld_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t p = match->m_begin; /* Check common prefix */ if (g_ascii_strncasecmp (p, "http://", sizeof ("http://") - 1) == 0) { - return url_web_end (begin, end, match->m_begin + sizeof ("http://") - 1, match); + return url_web_end (begin, + end, + match->m_begin + sizeof ("http://") - 1, + match); } else { return url_web_end (begin, end, match->m_begin, match); @@ -1261,10 +1609,15 @@ url_tld_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t } static gboolean -url_web_start (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match) +url_web_start (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match) { /* Check what we have found */ - if (pos > begin && (g_ascii_strncasecmp (pos, "www", 3) == 0 || g_ascii_strncasecmp (pos, "ftp", 3) == 0)) { + if (pos > begin && + (g_ascii_strncasecmp (pos, "www", + 3) == 0 || g_ascii_strncasecmp (pos, "ftp", 3) == 0)) { if (!is_open_brace (*(pos - 1)) && !g_ascii_isspace (*(pos - 1))) { return FALSE; } @@ -1279,13 +1632,16 @@ url_web_start (const gchar *begin, const gchar *end, const gchar *pos, url_match } static gboolean -url_web_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match) +url_web_end (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match) { - const gchar *p, *c; - gchar open_brace = '\0', close_brace = '\0'; - gint brace_stack = 0; - gboolean passwd = FALSE; - guint port, i; + const gchar *p, *c; + gchar open_brace = '\0', close_brace = '\0'; + gint brace_stack = 0; + gboolean passwd = FALSE; + guint port, i; p = pos + strlen (match->pattern); for (i = 0; i < G_N_ELEMENTS (url_braces) / 2; i += 2) { @@ -1311,7 +1667,8 @@ url_web_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t p++; } - if ((p + 1) < end && *p == '.' && (is_atom (*(p + 1)) || *(p + 1) == '/')) { + if ((p + 1) < end && *p == '.' && + (is_atom (*(p + 1)) || *(p + 1) == '/')) { p++; } } @@ -1338,7 +1695,9 @@ domain: p++; } - if ((p + 1) < end && *p == '.' && (is_domain (*(p + 1)) || *(p + 1) == '/' || (*(p + 1) & 0x80))) { + if ((p + 1) < end && *p == '.' && + (is_domain (*(p + 1)) || *(p + 1) == '/' || + (*(p + 1) & 0x80))) { p++; } } @@ -1372,7 +1731,7 @@ domain: } } else { - passwd: +passwd: passwd = TRUE; c = p; @@ -1396,7 +1755,7 @@ domain: break; } - /* we have a '/' so there could be a path - fall through */ + /* we have a '/' so there could be a path - fall through */ case '/': /* we've detected a path component to our url */ p++; case '?': @@ -1434,15 +1793,18 @@ domain: static gboolean -url_email_start (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match) +url_email_start (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match) { - const gchar *p; + const gchar *p; /* Check what we have found */ if (pos > begin && *pos == '@') { /* Try to extract it with username */ p = pos - 1; while (p > begin && (is_domain (*p) || *p == '.' || *p == '_')) { - p --; + p--; } if (!is_domain (*p) && p != pos - 1) { match->m_begin = p + 1; @@ -1464,10 +1826,13 @@ url_email_start (const gchar *begin, const gchar *end, const gchar *pos, url_mat } static gboolean -url_email_end (const gchar *begin, const gchar *end, const gchar *pos, url_match_t *match) +url_email_end (const gchar *begin, + const gchar *end, + const gchar *pos, + url_match_t *match) { - const gchar *p; - gboolean got_at = FALSE; + const gchar *p; + gboolean got_at = FALSE; p = pos + strlen (match->pattern); if (*pos == '@') { @@ -1475,12 +1840,12 @@ url_email_end (const gchar *begin, const gchar *end, const gchar *pos, url_match } while (p < end && (is_domain (*p) || *p == '_' - || (*p == '@' && !got_at) || - (*p == '.' && p + 1 < end && is_domain (*(p + 1))))) { + || (*p == '@' && !got_at) || + (*p == '.' && p + 1 < end && is_domain (*(p + 1))))) { if (*p == '@') { got_at = TRUE; } - p ++; + p++; } match->m_len = p - match->m_begin; match->add_prefix = TRUE; @@ -1488,13 +1853,16 @@ url_email_end (const gchar *begin, const gchar *end, const gchar *pos, url_match } void -url_parse_text (rspamd_mempool_t * pool, struct rspamd_task *task, struct mime_text_part *part, gboolean is_html) +url_parse_text (rspamd_mempool_t * pool, + struct rspamd_task *task, + struct mime_text_part *part, + gboolean is_html) { - gint rc; - gchar *url_str = NULL, *url_start, *url_end; - struct uri *new; - struct process_exception *ex; - gchar *p, *end, *begin; + gint rc; + gchar *url_str = NULL, *url_start, *url_end; + struct uri *new; + struct process_exception *ex; + gchar *p, *end, *begin; if (!part->orig->data || part->orig->len == 0) { @@ -1514,15 +1882,19 @@ url_parse_text (rspamd_mempool_t * pool, struct rspamd_task *task, struct mime_t p = begin; } while (p < end) { - if (url_try_text (pool, p, end - p, &url_start, &url_end, &url_str, is_html)) { + if (url_try_text (pool, p, end - p, &url_start, &url_end, &url_str, + is_html)) { if (url_str != NULL) { new = rspamd_mempool_alloc0 (pool, sizeof (struct uri)); - ex = rspamd_mempool_alloc0 (pool, sizeof (struct process_exception)); + ex = + rspamd_mempool_alloc0 (pool, + sizeof (struct process_exception)); if (new != NULL) { g_strstrip (url_str); rc = parse_uri (new, url_str, pool); - if ((rc == URI_ERRNO_OK || rc == URI_ERRNO_NO_SLASHES || rc == URI_ERRNO_NO_HOST_SLASH) && - new->hostlen > 0) { + if ((rc == URI_ERRNO_OK || rc == URI_ERRNO_NO_SLASHES || + rc == URI_ERRNO_NO_HOST_SLASH) && + new->hostlen > 0) { ex->pos = url_start - begin; ex->len = url_end - url_start; if (new->protocol == PROTOCOL_MAILTO) { @@ -1537,10 +1909,14 @@ url_parse_text (rspamd_mempool_t * pool, struct rspamd_task *task, struct mime_t g_tree_insert (task->urls, new, new); } } - part->urls_offset = g_list_prepend (part->urls_offset, ex); + part->urls_offset = g_list_prepend ( + part->urls_offset, + ex); } else if (rc != URI_ERRNO_OK) { - msg_info ("extract of url '%s' failed: %s", url_str, url_strerror (rc)); + msg_info ("extract of url '%s' failed: %s", + url_str, + url_strerror (rc)); } } } @@ -1554,21 +1930,30 @@ url_parse_text (rspamd_mempool_t * pool, struct rspamd_task *task, struct mime_t /* Handle offsets of this part */ if (part->urls_offset != NULL) { part->urls_offset = g_list_reverse (part->urls_offset); - rspamd_mempool_add_destructor (task->task_pool, (rspamd_mempool_destruct_t)g_list_free, part->urls_offset); + rspamd_mempool_add_destructor (task->task_pool, + (rspamd_mempool_destruct_t)g_list_free, part->urls_offset); } } gboolean -url_try_text (rspamd_mempool_t *pool, const gchar *begin, gsize len, gchar **start, gchar **fin, gchar **url_str, gboolean is_html) +url_try_text (rspamd_mempool_t *pool, + const gchar *begin, + gsize len, + gchar **start, + gchar **fin, + gchar **url_str, + gboolean is_html) { - const gchar *end, *pos; - gint idx, l; - struct url_matcher *matcher; - url_match_t m; + const gchar *end, *pos; + gint idx, l; + struct url_matcher *matcher; + url_match_t m; end = begin + len; if (url_init () == 0) { - if ((pos = rspamd_trie_lookup (url_scanner->patterns, begin, len, &idx)) == NULL) { + if ((pos = + rspamd_trie_lookup (url_scanner->patterns, begin, len, + &idx)) == NULL) { return FALSE; } else { @@ -1580,11 +1965,17 @@ url_try_text (rspamd_mempool_t *pool, const gchar *begin, gsize len, gchar **sta m.pattern = matcher->pattern; m.prefix = matcher->prefix; m.add_prefix = FALSE; - if (matcher->start (begin, end, pos, &m) && matcher->end (begin, end, pos, &m)) { + if (matcher->start (begin, end, pos, + &m) && matcher->end (begin, end, pos, &m)) { if (m.add_prefix) { l = m.m_len + 1 + strlen (m.prefix); *url_str = rspamd_mempool_alloc (pool, l); - rspamd_snprintf (*url_str, l, "%s%*s", m.prefix, m.m_len, m.m_begin); + rspamd_snprintf (*url_str, + l, + "%s%*s", + m.prefix, + m.m_len, + m.m_begin); } else { *url_str = rspamd_mempool_alloc (pool, m.m_len + 1); diff --git a/src/libserver/url.h b/src/libserver/url.h index 60535ba5c..d8877f279 100644 --- a/src/libserver/url.h +++ b/src/libserver/url.h @@ -43,24 +43,24 @@ struct uri { guint fragmentlen; /* Flags */ - gboolean ipv6; /* URI contains IPv6 host */ - gboolean form; /* URI originated from form */ + gboolean ipv6; /* URI contains IPv6 host */ + gboolean form; /* URI originated from form */ gboolean is_phished; /* URI maybe phishing */ }; enum uri_errno { - URI_ERRNO_OK, /* Parsing went well */ - URI_ERRNO_EMPTY, /* The URI string was empty */ - URI_ERRNO_INVALID_PROTOCOL, /* No protocol was found */ - URI_ERRNO_NO_SLASHES, /* Slashes after protocol missing */ - URI_ERRNO_TOO_MANY_SLASHES, /* Too many slashes after protocol */ - URI_ERRNO_TRAILING_DOTS, /* '.' after host */ - URI_ERRNO_NO_HOST, /* Host part is missing */ - URI_ERRNO_NO_PORT_COLON, /* ':' after host without port */ - URI_ERRNO_NO_HOST_SLASH, /* Slash after host missing */ - URI_ERRNO_IPV6_SECURITY, /* IPv6 security bug detected */ - URI_ERRNO_INVALID_PORT, /* Port number is bad */ - URI_ERRNO_INVALID_PORT_RANGE /* Port number is not within 0-65535 */ + URI_ERRNO_OK, /* Parsing went well */ + URI_ERRNO_EMPTY, /* The URI string was empty */ + URI_ERRNO_INVALID_PROTOCOL, /* No protocol was found */ + URI_ERRNO_NO_SLASHES, /* Slashes after protocol missing */ + URI_ERRNO_TOO_MANY_SLASHES, /* Too many slashes after protocol */ + URI_ERRNO_TRAILING_DOTS, /* '.' after host */ + URI_ERRNO_NO_HOST, /* Host part is missing */ + URI_ERRNO_NO_PORT_COLON, /* ':' after host without port */ + URI_ERRNO_NO_HOST_SLASH, /* Slash after host missing */ + URI_ERRNO_IPV6_SECURITY, /* IPv6 security bug detected */ + URI_ERRNO_INVALID_PORT, /* Port number is bad */ + URI_ERRNO_INVALID_PORT_RANGE /* Port number is not within 0-65535 */ }; enum protocol { @@ -81,7 +81,10 @@ enum protocol { * @param part current text part * @param is_html turn on html euristic */ -void url_parse_text (rspamd_mempool_t *pool, struct rspamd_task *task, struct mime_text_part *part, gboolean is_html); +void url_parse_text (rspamd_mempool_t *pool, + struct rspamd_task *task, + struct mime_text_part *part, + gboolean is_html); /* * Parse a single url into an uri structure @@ -89,7 +92,9 @@ void url_parse_text (rspamd_mempool_t *pool, struct rspamd_task *task, struct mi * @param uristring text form of url * @param uri url object, must be pre allocated */ -enum uri_errno parse_uri(struct uri *uri, gchar *uristring, rspamd_mempool_t *pool); +enum uri_errno parse_uri (struct uri *uri, + gchar *uristring, + rspamd_mempool_t *pool); /* * Try to extract url from a text @@ -101,11 +106,17 @@ enum uri_errno parse_uri(struct uri *uri, gchar *uristring, rspamd_mempool_t *po * @param url_str storage for url string(or NULL) * @return TRUE if url is found in specified text */ -gboolean url_try_text (rspamd_mempool_t *pool, const gchar *begin, gsize len, gchar **start, gchar **end, gchar **url_str, gboolean is_html); +gboolean url_try_text (rspamd_mempool_t *pool, + const gchar *begin, + gsize len, + gchar **start, + gchar **end, + gchar **url_str, + gboolean is_html); /* * Return text representation of url parsing error */ -const gchar* url_strerror (enum uri_errno err); +const gchar * url_strerror (enum uri_errno err); #endif diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c index 74a83cb70..075e499ec 100644 --- a/src/libserver/worker_util.c +++ b/src/libserver/worker_util.c @@ -22,28 +22,28 @@ */ #include "config.h" +#include "lua/lua_common.h" #include "main.h" #include "message.h" -#include "lua/lua_common.h" -extern struct rspamd_main *rspamd_main; +extern struct rspamd_main *rspamd_main; /** * Return worker's control structure by its type * @param type * @return worker's control structure or NULL */ -worker_t* +worker_t * rspamd_get_worker_by_type (GQuark type) { - worker_t **cur; + worker_t **cur; cur = &workers[0]; while (*cur) { if (g_quark_from_string ((*cur)->name) == type) { return *cur; } - cur ++; + cur++; } return NULL; @@ -52,18 +52,21 @@ rspamd_get_worker_by_type (GQuark type) double rspamd_set_counter (const gchar *name, guint32 value) { - struct counter_data *cd; - double alpha; - gchar *key; + struct counter_data *cd; + double alpha; + gchar *key; cd = rspamd_hash_lookup (rspamd_main->counters, (gpointer) name); if (cd == NULL) { - cd = rspamd_mempool_alloc_shared (rspamd_main->counters->pool, sizeof (struct counter_data)); + cd = + rspamd_mempool_alloc_shared (rspamd_main->counters->pool, + sizeof (struct counter_data)); cd->value = value; cd->number = 0; key = rspamd_mempool_strdup_shared (rspamd_main->counters->pool, name); - rspamd_hash_insert (rspamd_main->counters, (gpointer) key, (gpointer) cd); + rspamd_hash_insert (rspamd_main->counters, (gpointer) key, + (gpointer) cd); } else { /* Calculate new value */ @@ -88,7 +91,7 @@ static void worker_sig_handler (gint signo, siginfo_t * info, void *unused) #endif { - struct timeval tv; + struct timeval tv; switch (signo) { case SIGINT: @@ -113,9 +116,9 @@ worker_sig_handler (gint signo, siginfo_t * info, void *unused) static void worker_sigusr2_handler (gint fd, short what, void *arg) { - struct rspamd_worker *worker = (struct rspamd_worker *) arg; + struct rspamd_worker *worker = (struct rspamd_worker *) arg; /* Do not accept new connections, preparing to end worker's process */ - struct timeval tv; + struct timeval tv; if (!wanna_die) { tv.tv_sec = SOFT_SHUTDOWN_TIME; @@ -135,7 +138,7 @@ worker_sigusr2_handler (gint fd, short what, void *arg) static void worker_sigusr1_handler (gint fd, short what, void *arg) { - struct rspamd_worker *worker = (struct rspamd_worker *) arg; + struct rspamd_worker *worker = (struct rspamd_worker *) arg; reopen_log (worker->srv->logger); @@ -144,16 +147,16 @@ worker_sigusr1_handler (gint fd, short what, void *arg) struct event_base * rspamd_prepare_worker (struct rspamd_worker *worker, const char *name, - void (*accept_handler)(int, short, void *)) + void (*accept_handler)(int, short, void *)) { - struct event_base *ev_base; - struct event *accept_event; - struct sigaction signals; - GList *cur; - gint listen_socket; + struct event_base *ev_base; + struct event *accept_event; + struct sigaction signals; + GList *cur; + gint listen_socket; #ifdef WITH_PROFILER - extern void _start (void), etext (void); + extern void _start (void), etext (void); monstartup ((u_long) & _start, (u_long) & etext); #endif @@ -173,23 +176,24 @@ rspamd_prepare_worker (struct rspamd_worker *worker, const char *name, if (listen_socket != -1) { accept_event = g_slice_alloc0 (sizeof (struct event)); event_set (accept_event, listen_socket, EV_READ | EV_PERSIST, - accept_handler, worker); + accept_handler, worker); event_base_set (ev_base, accept_event); event_add (accept_event, NULL); - worker->accept_events = g_list_prepend (worker->accept_events, accept_event); + worker->accept_events = g_list_prepend (worker->accept_events, + accept_event); } cur = g_list_next (cur); } /* SIGUSR2 handler */ signal_set (&worker->sig_ev_usr2, SIGUSR2, worker_sigusr2_handler, - (void *) worker); + (void *) worker); event_base_set (ev_base, &worker->sig_ev_usr2); signal_add (&worker->sig_ev_usr2, NULL); /* SIGUSR1 handler */ signal_set (&worker->sig_ev_usr1, SIGUSR1, worker_sigusr1_handler, - (void *) worker); + (void *) worker); event_base_set (ev_base, &worker->sig_ev_usr1); signal_add (&worker->sig_ev_usr1, NULL); @@ -199,8 +203,8 @@ rspamd_prepare_worker (struct rspamd_worker *worker, const char *name, void rspamd_worker_stop_accept (struct rspamd_worker *worker) { - GList *cur; - struct event *event; + GList *cur; + struct event *event; /* Remove all events */ cur = worker->accept_events; @@ -218,8 +222,8 @@ rspamd_worker_stop_accept (struct rspamd_worker *worker) void rspamd_controller_send_error (struct rspamd_http_connection_entry *entry, - gint code, - const gchar *error_msg) + gint code, + const gchar *error_msg) { struct rspamd_http_message *msg; @@ -230,14 +234,20 @@ rspamd_controller_send_error (struct rspamd_http_connection_entry *entry, msg->status = g_string_new (error_msg); rspamd_printf_gstring (msg->body, "{\"error\":\"%s\"}", error_msg); rspamd_http_connection_reset (entry->conn); - rspamd_http_connection_write_message (entry->conn, msg, NULL, - "application/json", entry, entry->conn->fd, entry->rt->ptv, entry->rt->ev_base); + rspamd_http_connection_write_message (entry->conn, + msg, + NULL, + "application/json", + entry, + entry->conn->fd, + entry->rt->ptv, + entry->rt->ev_base); entry->is_reply = TRUE; } void rspamd_controller_send_string (struct rspamd_http_connection_entry *entry, - const gchar *str) + const gchar *str) { struct rspamd_http_message *msg; @@ -246,14 +256,20 @@ rspamd_controller_send_string (struct rspamd_http_connection_entry *entry, msg->code = 200; msg->body = g_string_new (str); rspamd_http_connection_reset (entry->conn); - rspamd_http_connection_write_message (entry->conn, msg, NULL, - "application/json", entry, entry->conn->fd, entry->rt->ptv, entry->rt->ev_base); + rspamd_http_connection_write_message (entry->conn, + msg, + NULL, + "application/json", + entry, + entry->conn->fd, + entry->rt->ptv, + entry->rt->ev_base); entry->is_reply = TRUE; } void rspamd_controller_send_ucl (struct rspamd_http_connection_entry *entry, - ucl_object_t *obj) + ucl_object_t *obj) { struct rspamd_http_message *msg; @@ -263,7 +279,13 @@ rspamd_controller_send_ucl (struct rspamd_http_connection_entry *entry, msg->body = g_string_sized_new (BUFSIZ); rspamd_ucl_emit_gstring (obj, UCL_EMIT_JSON_COMPACT, msg->body); rspamd_http_connection_reset (entry->conn); - rspamd_http_connection_write_message (entry->conn, msg, NULL, - "application/json", entry, entry->conn->fd, entry->rt->ptv, entry->rt->ev_base); + rspamd_http_connection_write_message (entry->conn, + msg, + NULL, + "application/json", + entry, + entry->conn->fd, + entry->rt->ptv, + entry->rt->ev_base); entry->is_reply = TRUE; } diff --git a/src/libserver/worker_util.h b/src/libserver/worker_util.h index 2af32d1e8..e0b92be60 100644 --- a/src/libserver/worker_util.h +++ b/src/libserver/worker_util.h @@ -24,15 +24,15 @@ #define WORKER_UTIL_H_ #include "config.h" -#include "util.h" #include "http.h" +#include "util.h" /** * Return worker's control structure by its type * @param type * @return worker's control structure or NULL */ -worker_t* rspamd_get_worker_by_type (GQuark type); +worker_t * rspamd_get_worker_by_type (GQuark type); /** * Set counter for a symbol @@ -57,7 +57,7 @@ struct rspamd_worker; */ struct event_base * rspamd_prepare_worker (struct rspamd_worker *worker, const char *name, - void (*accept_handler)(int, short, void *)); + void (*accept_handler)(int, short, void *)); /** * Stop accepting new connections for a worker @@ -66,9 +66,9 @@ rspamd_prepare_worker (struct rspamd_worker *worker, const char *name, void rspamd_worker_stop_accept (struct rspamd_worker *worker); typedef gint (*rspamd_controller_func_t) ( - struct rspamd_http_connection_entry *conn_ent, - struct rspamd_http_message *msg, - struct module_ctx *ctx); + struct rspamd_http_connection_entry *conn_ent, + struct rspamd_http_message *msg, + struct module_ctx *ctx); struct rspamd_custom_controller_command { const gchar *command; @@ -85,8 +85,8 @@ struct rspamd_custom_controller_command { * @param error_msg error message */ void rspamd_controller_send_error (struct rspamd_http_connection_entry *entry, - gint code, - const gchar *error_msg); + gint code, + const gchar *error_msg); /** * Send a custom string using HTTP @@ -94,7 +94,7 @@ void rspamd_controller_send_error (struct rspamd_http_connection_entry *entry, * @param str string to send */ void rspamd_controller_send_string (struct rspamd_http_connection_entry *entry, - const gchar *str); + const gchar *str); /** * Send UCL using HTTP and JSON serialization @@ -102,6 +102,6 @@ void rspamd_controller_send_string (struct rspamd_http_connection_entry *entry, * @param obj object to send */ void rspamd_controller_send_ucl (struct rspamd_http_connection_entry *entry, - ucl_object_t *obj); + ucl_object_t *obj); #endif /* WORKER_UTIL_H_ */ |