From 9942bf2d277be84a322cdd3713cc51b8c6a1ec89 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Tue, 18 Sep 2012 22:56:43 +0400 Subject: [PATCH] Restore fuzzy learn operations. --- src/client/rspamc.c | 82 +++++++++++--- src/plugins/fuzzy_check.c | 233 ++++++++++++++++++++++++++++---------- 2 files changed, 243 insertions(+), 72 deletions(-) diff --git a/src/client/rspamc.c b/src/client/rspamc.c index 38855e7c4..877b0285f 100644 --- a/src/client/rspamc.c +++ b/src/client/rspamc.c @@ -23,6 +23,7 @@ */ #include "config.h" +#include "util.h" #include "../../lib/client/librspamdclient.h" #define PRINT_FUNC printf @@ -32,7 +33,6 @@ static gchar *connect_str = "localhost"; static gchar *password = NULL; -static gchar *statfile = NULL; static gchar *ip = NULL; static gchar *from = NULL; static gchar *deliver_to = NULL; @@ -508,6 +508,16 @@ fuzzy_rspamd_stdin (gboolean delete) gchar *in_buf; gint r = 0, len; GError *err = NULL; + GHashTable *params; + GList *results, *cur; + gchar valuebuf[sizeof("65535")], flagbuf[sizeof("65535")]; + struct rspamd_controller_result *res; + + params = g_hash_table_new (g_str_hash, g_str_equal); + rspamd_snprintf (valuebuf, sizeof (valuebuf), "%d", weight); + rspamd_snprintf (flagbuf, sizeof (flagbuf), "%d", flag); + g_hash_table_insert (params, "Value", valuebuf); + g_hash_table_insert (params, "Flag", flagbuf); /* Add server */ add_rspamd_server (TRUE); @@ -525,33 +535,77 @@ fuzzy_rspamd_stdin (gboolean delete) in_buf = g_realloc (in_buf, len); } } - /* TODO: write this function */ + results = rspamd_controller_command_memory (client, delete ? "fuzzy_del" : "fuzzy_add", password, params, in_buf, r, &err); + g_hash_table_destroy (params); + if (results == NULL || err != NULL) { + if (err != NULL) { + fprintf (stderr, "cannot process fuzzy for message: %s\n", err->message); + } + else { + fprintf (stderr, "cannot process fuzzy for message\n"); + } + exit (EXIT_FAILURE); + } + else { + cur = results; + while (cur) { + res = cur->data; + if (tty) { + printf ("\033[1m"); + } + PRINT_FUNC ("Results for host: %s: %d, %s\n", res->server_name, res->code, res->result->str); + if (tty) { + printf ("\033[0m"); + } + rspamd_free_controller_result (res); + cur = g_list_next (cur); + } + g_list_free (results); + } } static void fuzzy_rspamd_file (const gchar *file, gboolean delete) { GError *err = NULL; - /* TODO: write this function */ -#if 0 - if (!rspamd_fuzzy_file (client, file, password, weight, flag, delete, &err)) { + GHashTable *params; + GList *results, *cur; + gchar valuebuf[sizeof("65535")], flagbuf[sizeof("65535")]; + struct rspamd_controller_result *res; + + params = g_hash_table_new (g_str_hash, g_str_equal); + rspamd_snprintf (valuebuf, sizeof (valuebuf), "%d", weight); + rspamd_snprintf (flagbuf, sizeof (flagbuf), "%d", flag); + g_hash_table_insert (params, "Value", valuebuf); + g_hash_table_insert (params, "Flag", flagbuf); + + results = rspamd_controller_command_file (client, delete ? "fuzzy_del" : "fuzzy_add", password, params, file, &err); + g_hash_table_destroy (params); + if (results == NULL || err != NULL) { if (err != NULL) { - fprintf (stderr, "cannot learn message: %s\n", err->message); + fprintf (stderr, "cannot process fuzzy for message: %s\n", err->message); } else { - fprintf (stderr, "cannot learn message\n"); + fprintf (stderr, "cannot process fuzzy for message\n"); } + exit (EXIT_FAILURE); } else { - if (tty) { - printf ("\033[1m"); - } - PRINT_FUNC ("learn ok\n"); - if (tty) { - printf ("\033[0m"); + cur = results; + while (cur) { + res = cur->data; + if (tty) { + printf ("\033[1m"); + } + PRINT_FUNC ("Results for host: %s: %d, %s\n", res->server_name, res->code, res->result->str); + if (tty) { + printf ("\033[0m"); + } + rspamd_free_controller_result (res); + cur = g_list_next (cur); } + g_list_free (results); } -#endif } static void diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 406efae77..51bdd58c3 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -110,7 +110,8 @@ struct fuzzy_learn_session { gint cmd; gint value; gint flag; - gint *saved; + gint *saved; + GError **err; struct timeval tv; struct controller_session *session; struct storage_server *server; @@ -550,7 +551,7 @@ fuzzy_learn_callback (gint fd, short what, void *arg) { struct fuzzy_learn_session *session = arg; struct fuzzy_cmd cmd; - gchar buf[64]; + gchar buf[512]; gint r; if (what == EV_WRITE) { @@ -561,6 +562,9 @@ fuzzy_learn_callback (gint fd, short what, void *arg) cmd.value = session->value; cmd.flag = session->flag; if (write (fd, &cmd, sizeof (struct fuzzy_cmd)) == -1) { + if (*(session->err) == NULL) { + g_set_error (session->err, g_quark_from_static_string ("fuzzy check"), 404, "write socket error: %s", strerror (errno)); + } goto err; } else { @@ -571,22 +575,21 @@ fuzzy_learn_callback (gint fd, short what, void *arg) } else if (what == EV_READ) { if (read (fd, buf, sizeof (buf)) == -1) { + msg_info ("cannot add fuzzy hash for message <%s>", session->task->message_id); + if (*(session->err) == NULL) { + g_set_error (session->err, g_quark_from_static_string ("fuzzy check"), 404, "read socket error: %s", strerror (errno)); + } goto err; } else if (buf[0] == 'O' && buf[1] == 'K') { msg_info ("added fuzzy hash '%s' to list: %d for message <%s>", fuzzy_to_string (session->h), session->flag, session->task->message_id); - r = rspamd_snprintf (buf, sizeof (buf), "OK" CRLF "END" CRLF); - if (! rspamd_dispatcher_write (session->session->dispatcher, buf, r, FALSE, FALSE)) { - return; - } goto ok; } else { msg_info ("cannot add fuzzy hash for message <%s>", session->task->message_id); - r = rspamd_snprintf (buf, sizeof (buf), "ERR" CRLF "END" CRLF); - if (! rspamd_dispatcher_write (session->session->dispatcher, buf, r, FALSE, FALSE)) { - return; + if (*(session->err) == NULL) { + g_set_error (session->err, g_quark_from_static_string ("fuzzy check"), 500, "add fuzzy error"); } goto ok; } @@ -598,13 +601,37 @@ fuzzy_learn_callback (gint fd, short what, void *arg) return; - err: +err: msg_err ("got error in IO with server %s:%d, %d, %s", session->server->name, session->server->port, errno, strerror (errno)); - r = rspamd_snprintf (buf, sizeof (buf), "Error" CRLF "END" CRLF); - if (! rspamd_dispatcher_write (session->session->dispatcher, buf, r, FALSE, FALSE)) { - return; +ok: + if (--(*(session->saved)) == 0) { + session->session->state = STATE_REPLY; + if (*(session->err) != NULL) { + if (session->session->restful) { + r = rspamd_snprintf (buf, sizeof (buf), "HTTP/1.0 %d Write hash error: %s" CRLF CRLF, (*session->err)->code, (*session->err)->message); + } + else { + r = rspamd_snprintf (buf, sizeof (buf), "write error: %s" CRLF "END" CRLF, (*session->err)->message); + } + g_error_free (*session->err); + if (! rspamd_dispatcher_write (session->session->dispatcher, buf, r, FALSE, FALSE)) { + return; + } + } + else { + if (session->session->restful) { + r = rspamd_snprintf (buf, sizeof (buf), "HTTP/1.0 200 OK" CRLF CRLF); + } + else { + r = rspamd_snprintf (buf, sizeof (buf), "OK" CRLF "END" CRLF); + } + if (! rspamd_dispatcher_write (session->session->dispatcher, buf, r, FALSE, FALSE)) { + return; + } + } + rspamd_dispatcher_restore (session->session->dispatcher); + } - ok: remove_normal_event (session->session->s, fuzzy_learn_fin, session); } @@ -736,7 +763,7 @@ fuzzy_symbol_callback (struct worker_task *task, void *unused) static inline gboolean register_fuzzy_controller_call (struct controller_session *session, struct worker_task *task, fuzzy_hash_t *h, - gint cmd, gint value, gint flag, gint *saved) + gint cmd, gint value, gint flag, gint *saved, GError **err) { struct fuzzy_learn_session *s; struct storage_server *selected; @@ -758,11 +785,17 @@ register_fuzzy_controller_call (struct controller_session *session, struct worke if ((sock = make_udp_socket (&selected->addr, selected->port, FALSE, TRUE)) == -1) { msg_warn ("cannot connect to %s, %d, %s", selected->name, errno, strerror (errno)); session->state = STATE_REPLY; - r = rspamd_snprintf (out_buf, sizeof (out_buf), "no hashes written" CRLF "END" CRLF); + if (session->restful) { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 404 No hashes have been written" CRLF CRLF); + } + else { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "no hashes have been written" CRLF "END" CRLF); + } if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { return FALSE; } free_task (task, FALSE); + rspamd_dispatcher_restore (session->dispatcher); return FALSE; } else { @@ -780,6 +813,7 @@ register_fuzzy_controller_call (struct controller_session *session, struct worke s->flag = flag; s->saved = saved; s->fd = sock; + s->err = err; event_add (&s->ev, &s->tv); (*saved)++; register_async_event (session->s, fuzzy_learn_fin, s, g_quark_from_static_string ("fuzzy check")); @@ -797,6 +831,7 @@ fuzzy_process_handler (struct controller_session *session, f_str_t * in) struct mime_part *mime_part; struct rspamd_image *image; GList *cur; + GError **err; gint r, cmd = 0, value = 0, flag = 0, *saved, *sargs; gchar out_buf[BUFSIZ], *checksum; fuzzy_hash_t fake_fuzzy; @@ -821,15 +856,22 @@ fuzzy_process_handler (struct controller_session *session, f_str_t * in) saved = memory_pool_alloc0 (session->session_pool, sizeof (gint)); + err = memory_pool_alloc0 (session->session_pool, sizeof (GError **)); r = process_message (task); if (r == -1) { msg_warn ("processing of message failed"); free_task (task, FALSE); session->state = STATE_REPLY; - r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot process message" CRLF "END" CRLF); + if (session->restful) { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Cannot process message" CRLF CRLF); + } + else { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot process message" CRLF "END" CRLF); + } if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { msg_warn ("write error"); } + rspamd_dispatcher_restore (session->dispatcher); return; } else { @@ -842,24 +884,36 @@ fuzzy_process_handler (struct controller_session *session, f_str_t * in) cur = g_list_next (cur); continue; } - if (! register_fuzzy_controller_call (session, task, part->fuzzy, cmd, value, flag, saved)) { + if (! register_fuzzy_controller_call (session, task, part->fuzzy, cmd, value, flag, saved, err)) { /* Cannot write hash */ session->state = STATE_REPLY; - r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); + if (session->restful) { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Cannot write fuzzy hash" CRLF CRLF); + } + else { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); + } if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { return; } + rspamd_dispatcher_restore (session->dispatcher); free_task (task, FALSE); return; } - if (! register_fuzzy_controller_call (session, task, part->double_fuzzy, cmd, value, flag, saved)) { + if (! register_fuzzy_controller_call (session, task, part->double_fuzzy, cmd, value, flag, saved, err)) { /* Cannot write hash */ session->state = STATE_REPLY; - r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); + if (session->restful) { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Cannot write fuzzy hash" CRLF CRLF); + } + else { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); + } if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { return; } free_task (task, FALSE); + rspamd_dispatcher_restore (session->dispatcher); return; } cur = g_list_next (cur); @@ -876,15 +930,21 @@ fuzzy_process_handler (struct controller_session *session, f_str_t * in) fake_fuzzy.block_size = 0; bzero (fake_fuzzy.hash_pipe, sizeof (fake_fuzzy.hash_pipe)); rspamd_strlcpy (fake_fuzzy.hash_pipe, checksum, sizeof (fake_fuzzy.hash_pipe)); - if (! register_fuzzy_controller_call (session, task, &fake_fuzzy, cmd, value, flag, saved)) { + if (! register_fuzzy_controller_call (session, task, &fake_fuzzy, cmd, value, flag, saved, err)) { /* Cannot write hash */ session->state = STATE_REPLY; - r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); + if (session->restful) { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Cannot write fuzzy hash" CRLF CRLF); + } + else { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); + } if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { return; } g_free (checksum); free_task (task, FALSE); + rspamd_dispatcher_restore (session->dispatcher); return; } @@ -906,15 +966,21 @@ fuzzy_process_handler (struct controller_session *session, f_str_t * in) fake_fuzzy.block_size = 0; bzero (fake_fuzzy.hash_pipe, sizeof (fake_fuzzy.hash_pipe)); rspamd_strlcpy (fake_fuzzy.hash_pipe, checksum, sizeof (fake_fuzzy.hash_pipe)); - if (! register_fuzzy_controller_call (session, task, &fake_fuzzy, cmd, value, flag, saved)) { + if (! register_fuzzy_controller_call (session, task, &fake_fuzzy, cmd, value, flag, saved, err)) { /* Cannot write hash */ session->state = STATE_REPLY; - r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); + if (session->restful) { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Cannot write fuzzy hash" CRLF CRLF); + } + else { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "cannot write fuzzy hash" CRLF "END" CRLF); + } if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { return; } g_free (checksum); free_task (task, FALSE); + rspamd_dispatcher_restore (session->dispatcher); return; } msg_info ("save hash of part of type: %s/%s: [%s] to list %d", @@ -931,10 +997,16 @@ fuzzy_process_handler (struct controller_session *session, f_str_t * in) if (*saved == 0) { session->state = STATE_REPLY; - r = rspamd_snprintf (out_buf, sizeof (out_buf), "no hashes written" CRLF "END" CRLF); + if (session->restful) { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 404 No hashes have been written" CRLF CRLF); + } + else { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "no hashes have been written" CRLF "END" CRLF); + } if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { return; } + rspamd_dispatcher_restore (session->dispatcher); } } @@ -945,45 +1017,90 @@ fuzzy_controller_handler (gchar **args, struct controller_session *session, gint guint32 size; gint r, value = 1, flag = 0, *sargs; - /* Process size */ - arg = args[0]; - if (!arg || *arg == '\0') { - msg_info ("empty content length"); - r = rspamd_snprintf (out_buf, sizeof (out_buf), "fuzzy command requires length as argument" CRLF "END" CRLF); - if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { + if (session->restful) { + /* Get size */ + arg = g_hash_table_lookup (session->kwargs, "content-length"); + if (!arg || *arg == '\0') { + msg_info ("empty content length"); + r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Fuzzy command requires Content-Length" CRLF CRLF); + if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { + return; + } + session->state = STATE_REPLY; + rspamd_dispatcher_restore (session->dispatcher); return; } - session->state = STATE_REPLY; - return; - } - errno = 0; - size = strtoul (arg, &err_str, 10); - if (errno != 0 || (err_str && *err_str != '\0')) { - r = rspamd_snprintf (out_buf, sizeof (out_buf), "learn size is invalid" CRLF); - if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { + errno = 0; + size = strtoul (arg, &err_str, 10); + if (errno != 0 || (err_str && *err_str != '\0')) { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "HTTP/1.0 500 Learn size is invalid" CRLF CRLF); + if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { + return; + } + session->state = STATE_REPLY; + rspamd_dispatcher_restore (session->dispatcher); return; } - session->state = STATE_REPLY; - return; - } - /* Process value */ - arg = args[1]; - if (arg && *arg != '\0') { - errno = 0; - value = strtol (arg, &err_str, 10); - if (errno != 0 || *err_str != '\0') { - msg_info ("error converting numeric argument %s", arg); - value = 0; + arg = g_hash_table_lookup (session->kwargs, "value"); + if (arg) { + errno = 0; + value = strtol (arg, &err_str, 10); + if (errno != 0 || *err_str != '\0') { + msg_info ("error converting numeric argument %s", arg); + value = 0; + } + } + arg = g_hash_table_lookup (session->kwargs, "flag"); + if (arg) { + errno = 0; + flag = strtol (arg, &err_str, 10); + if (errno != 0 || *err_str != '\0') { + msg_info ("error converting numeric argument %s", arg); + flag = 0; + } } } - /* Process flag */ - arg = args[2]; - if (arg && *arg != '\0') { + else { + /* Process size */ + arg = args[0]; + if (!arg || *arg == '\0') { + msg_info ("empty content length"); + r = rspamd_snprintf (out_buf, sizeof (out_buf), "fuzzy command requires length as argument" CRLF "END" CRLF); + if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { + return; + } + session->state = STATE_REPLY; + return; + } errno = 0; - flag = strtol (arg, &err_str, 10); - if (errno != 0 || *err_str != '\0') { - msg_info ("error converting numeric argument %s", arg); - flag = 0; + size = strtoul (arg, &err_str, 10); + if (errno != 0 || (err_str && *err_str != '\0')) { + r = rspamd_snprintf (out_buf, sizeof (out_buf), "learn size is invalid" CRLF); + if (! rspamd_dispatcher_write (session->dispatcher, out_buf, r, FALSE, FALSE)) { + return; + } + session->state = STATE_REPLY; + return; + } + /* Process value */ + arg = args[1]; + if (arg && *arg != '\0') { + errno = 0; + value = strtol (arg, &err_str, 10); + if (errno != 0 || *err_str != '\0') { + msg_info ("error converting numeric argument %s", arg); + value = 0; + } + } + /* Process flag */ + arg = args[2]; + if (arg && *arg != '\0') { + errno = 0; + flag = strtol (arg, &err_str, 10); + if (errno != 0 || *err_str != '\0') { + msg_info ("error converting numeric argument %s", arg); + flag = 0; + } } } -- 2.39.5