Restore expired/count statistics.

diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index 921f1796486a92c57257d630019968f9ee93a3fe..7b3c58c43d1833b1cbf0cd7810d74192f005f0fd 100644 (file)
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -222,6 +222,7 @@ rspamd_fuzzy_process_command (struct fuzzy_session *session)
                        rep.value = 403;
                        rep.prob = 0.0;
                }
+               server_stat->fuzzy_hashes = rspamd_fuzzy_backend_count (session->ctx->backend);
        }
 
        rspamd_fuzzy_write_reply (session, &rep);
@@ -324,7 +325,9 @@ sync_callback (gint fd, short what, void *arg)
        evtimer_add (&tev, &tmv);
 
        /* Call backend sync */
-       rspamd_fuzzy_backend_sync (ctx->backend);
+       rspamd_fuzzy_backend_sync (ctx->backend, ctx->expire);
+
+       server_stat->fuzzy_hashes_expired = rspamd_fuzzy_backend_expired (ctx->backend);
 }
 
 gpointer
@@ -418,6 +421,8 @@ start_fuzzy (struct rspamd_worker *worker)
                exit (EXIT_FAILURE);
        }
 
+       server_stat->fuzzy_hashes = rspamd_fuzzy_backend_count (ctx->backend);
+
        /* Timer event */
        evtimer_set (&tev, sync_callback, worker);
        event_base_set (ctx->ev_base, &tev);
@@ -444,7 +449,7 @@ start_fuzzy (struct rspamd_worker *worker)
 
        event_base_loop (ctx->ev_base, 0);
 
-       rspamd_fuzzy_backend_sync (ctx->backend);
+       rspamd_fuzzy_backend_sync (ctx->backend, ctx->expire);
        rspamd_fuzzy_backend_close (ctx->backend);
        rspamd_log_close (rspamd_main->logger);
        exit (EXIT_SUCCESS);

diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c
index 2df970543b025380c7dd440dc5728506d66053c7..200af4a1c99f8e5a3a403f603053b90710a3e88e 100644 (file)
--- a/src/libserver/fuzzy_backend.c
+++ b/src/libserver/fuzzy_backend.c
@@ -41,6 +41,8 @@ struct rspamd_legacy_fuzzy_node {
 struct rspamd_fuzzy_backend {
        sqlite3 *db;
        char *path;
+       gsize count;
+       gsize expired;
 };
 
 
@@ -61,6 +63,7 @@ const char *create_tables_sql =
 const char *create_index_sql =
                "BEGIN;"
                "CREATE UNIQUE INDEX IF NOT EXISTS d ON digests(digest);"
+               "CREATE INDEX IF NOT EXISTS t ON digests(time);"
                "CREATE UNIQUE INDEX IF NOT EXISTS s ON shingles(value, number);"
                "COMMIT;";
 enum rspamd_fuzzy_statement_idx {
@@ -73,6 +76,9 @@ enum rspamd_fuzzy_statement_idx {
        RSPAMD_FUZZY_BACKEND_CHECK,
        RSPAMD_FUZZY_BACKEND_CHECK_SHINGLE,
        RSPAMD_FUZZY_BACKEND_DELETE,
+       RSPAMD_FUZZY_BACKEND_COUNT,
+       RSPAMD_FUZZY_BACKEND_EXPIRE,
+       RSPAMD_FUZZY_BACKEND_VACUUM,
        RSPAMD_FUZZY_BACKEND_MAX
 };
 static struct rspamd_fuzzy_stmts {
@@ -148,6 +154,27 @@ static struct rspamd_fuzzy_stmts {
                .args = "D",
                .stmt = NULL,
                .result = SQLITE_DONE
+       },
+       {
+               .idx = RSPAMD_FUZZY_BACKEND_COUNT,
+               .sql = "SELECT COUNT(*) FROM digests;",
+               .args = "",
+               .stmt = NULL,
+               .result = SQLITE_ROW
+       },
+       {
+               .idx = RSPAMD_FUZZY_BACKEND_EXPIRE,
+               .sql = "DELETE FROM digests WHERE time < ?1;",
+               .args = "I",
+               .stmt = NULL,
+               .result = SQLITE_DONE
+       },
+       {
+               .idx = RSPAMD_FUZZY_BACKEND_VACUUM,
+               .sql = "VACUUM;",
+               .args = "",
+               .stmt = NULL,
+               .result = SQLITE_DONE
        }
 };
 
@@ -310,6 +337,8 @@ rspamd_fuzzy_backend_create_db (const gchar *path, gboolean add_index,
        bk = g_slice_alloc (sizeof (*bk));
        bk->path = g_strdup (path);
        bk->db = sqlite;
+       bk->expired = 0;
+       bk->count = 0;
 
        /*
         * Here we need to run create prior to preparing other statements
@@ -355,6 +384,16 @@ rspamd_fuzzy_backend_open_db (const gchar *path, GError **err)
        bk->path = g_strdup (path);
        bk = g_slice_alloc (sizeof (*bk));
        bk->db = sqlite;
+       bk->expired = 0;
+
+       /* Cleanup database */
+       rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_VACUUM, bk, NULL);
+
+       if (rspamd_fuzzy_backend_run_stmt (bk, RSPAMD_FUZZY_BACKEND_COUNT)
+                       == SQLITE_OK) {
+               bk->count = sqlite3_column_int64 (
+                               prepared_stmts[RSPAMD_FUZZY_BACKEND_COUNT].stmt, 0);
+       }
 
        rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_TRANSACTION_START,
                                bk, NULL);
@@ -542,6 +581,7 @@ rspamd_fuzzy_backend_add (struct rspamd_fuzzy_backend *backend,
                rc = rspamd_fuzzy_backend_run_stmt (backend, RSPAMD_FUZZY_BACKEND_INSERT,
                        (gint)cmd->flag, cmd->digest, (gint64)cmd->value, (gint64)time (NULL));
 
+               backend->count ++;
                if (cmd->shingles_count > 0) {
                        /* Add corresponding shingles */
                }
@@ -560,14 +600,22 @@ rspamd_fuzzy_backend_del (struct rspamd_fuzzy_backend *backend,
        rc = rspamd_fuzzy_backend_run_stmt (backend, RSPAMD_FUZZY_BACKEND_DELETE,
                        cmd->digest);
 
+       backend->count -= sqlite3_changes (backend->db);
+
        return (rc == SQLITE_OK);
 }
 
 gboolean
-rspamd_fuzzy_backend_sync (struct rspamd_fuzzy_backend *backend)
+rspamd_fuzzy_backend_sync (struct rspamd_fuzzy_backend *backend, gint64 expire)
 {
        gboolean ret = FALSE;
 
+       /* Perform expire */
+       if (expire > 0) {
+               rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_EXPIRE,
+                               backend, NULL);
+               backend->expired += sqlite3_changes (backend->db);
+       }
        ret = rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT,
                        backend, NULL);
 
@@ -596,3 +644,16 @@ rspamd_fuzzy_backend_close (struct rspamd_fuzzy_backend *backend)
                g_slice_free1 (sizeof (*backend), backend);
        }
 }
+
+
+gsize
+rspamd_fuzzy_backend_count (struct rspamd_fuzzy_backend *backend)
+{
+       return backend->count;
+}
+
+gsize
+rspamd_fuzzy_backend_expired (struct rspamd_fuzzy_backend *backend)
+{
+       return backend->expired;
+}

diff --git a/src/libserver/fuzzy_backend.h b/src/libserver/fuzzy_backend.h
index 61664c889c5eea1f78acd1d32e5ae7d621d8ded3..f0a965154169fdfec76ff4974c53fb207f05a765 100644 (file)
--- a/src/libserver/fuzzy_backend.h
+++ b/src/libserver/fuzzy_backend.h
@@ -74,7 +74,8 @@ gboolean rspamd_fuzzy_backend_del (
  * @param backend
  * @return
  */
-gboolean rspamd_fuzzy_backend_sync (struct rspamd_fuzzy_backend *backend);
+gboolean rspamd_fuzzy_backend_sync (struct rspamd_fuzzy_backend *backend,
+               gint64 expire);
 
 /**
  * Close storage
@@ -82,4 +83,7 @@ gboolean rspamd_fuzzy_backend_sync (struct rspamd_fuzzy_backend *backend);
  */
 void rspamd_fuzzy_backend_close (struct rspamd_fuzzy_backend *backend);
 
+gsize rspamd_fuzzy_backend_count (struct rspamd_fuzzy_backend *backend);
+gsize rspamd_fuzzy_backend_expired (struct rspamd_fuzzy_backend *backend);
+
 #endif /* FUZZY_BACKEND_H_ */