First attempt to clean orphaned elements.

diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c
index 016ee2b5afc8f64d9db375b6323e06b63d28e1d5..ce7d5b5d126c696ab9e453ca5643ffdadb1f9ba4 100644 (file)
--- a/src/libserver/fuzzy_backend.c
+++ b/src/libserver/fuzzy_backend.c
@@ -86,6 +86,7 @@ enum rspamd_fuzzy_statement_idx {
        RSPAMD_FUZZY_BACKEND_COUNT,
        RSPAMD_FUZZY_BACKEND_EXPIRE,
        RSPAMD_FUZZY_BACKEND_VACUUM,
+       RSPAMD_FUZZY_BACKEND_DELETE_ORPHANED,
        RSPAMD_FUZZY_BACKEND_MAX
 };
 static struct rspamd_fuzzy_stmts {
@@ -189,7 +190,14 @@ static struct rspamd_fuzzy_stmts {
                .args = "",
                .stmt = NULL,
                .result = SQLITE_DONE
-       }
+       },
+       {
+               .idx = RSPAMD_FUZZY_BACKEND_DELETE_ORPHANED,
+               .sql = "DELETE FROM shingles WHERE value=?1 AND number=?2;",
+               .args = "II",
+               .stmt = NULL,
+               .result = SQLITE_DONE
+       },
 };
 
 static GQuark
@@ -404,13 +412,26 @@ rspamd_fuzzy_backend_open_db (const gchar *path, GError **err)
 struct rspamd_fuzzy_backend*
 rspamd_fuzzy_backend_open (const gchar *path, GError **err)
 {
+       struct orphaned_shingle_elt {
+               gint64 value;
+               gint64 number;
+       };
+
        gchar *dir;
        gint fd;
        struct rspamd_fuzzy_backend *backend;
        static const char sqlite_wal[] = "PRAGMA journal_mode=\"wal\";",
                        fallback_journal[] = "PRAGMA journal_mode=\"off\";",
-                       foreign_keys[] = "PRAGMA foreign_keys=\"ON\";";
-       int rc;
+                       foreign_keys[] = "PRAGMA foreign_keys=\"ON\";",
+                       orphaned_shingles[] =   "SELECT shingles.value,shingles.number "
+                                                                       "FROM shingles "
+                                                                       "LEFT JOIN digests ON "
+                                                                       "shingles.digest_id=digests.id WHERE "
+                                                                       "digests.id IS NULL;";
+       sqlite3_stmt *stmt;
+       GArray *orphaned;
+       struct orphaned_shingle_elt orphaned_elt, *pelt;
+       gint rc, i;
 
        if (path == NULL) {
                g_set_error (err, rspamd_fuzzy_backend_quark (),
@@ -462,17 +483,60 @@ rspamd_fuzzy_backend_open (const gchar *path, GError **err)
        }
 
        if ((rc = sqlite3_exec (backend->db, sqlite_wal, NULL, NULL, NULL)) != SQLITE_OK) {
-               msg_warn_fuzzy_backend ("WAL mode is not supported (%d), locking issues might occur",
-                               rc);
+               msg_warn_fuzzy_backend ("WAL mode is not supported (%s), locking issues might occur",
+                               sqlite3_errmsg (backend->db));
                sqlite3_exec (backend->db, fallback_journal, NULL, NULL, NULL);
        }
 
        if ((rc = sqlite3_exec (backend->db, foreign_keys, NULL, NULL, NULL)) !=
                        SQLITE_OK) {
-               msg_warn_fuzzy_backend ("foreign keys are not supported", rc);
+               msg_warn_fuzzy_backend ("foreign keys are not supported: %s",
+                               sqlite3_errmsg (backend->db));
        }
 
        /* Cleanup database */
+       if ((rc = sqlite3_prepare_v2 (backend->db, orphaned_shingles, -1, &stmt,
+                               NULL)) != SQLITE_OK) {
+               msg_warn_fuzzy_backend ("cannot cleanup shingles: %s",
+                               sqlite3_errmsg (backend->db));
+       }
+       else {
+               orphaned = g_array_new (FALSE, FALSE, sizeof (struct orphaned_shingle_elt));
+
+               while (sqlite3_step (stmt) == SQLITE_ROW) {
+                       orphaned_elt.value = sqlite3_column_int64 (stmt, 0);
+                       orphaned_elt.number = sqlite3_column_int64 (stmt, 1);
+                       g_array_append_val (orphaned, orphaned_elt);
+               }
+
+               sqlite3_finalize (stmt);
+               msg_info_fuzzy_backend ("going to delete %ud orphaned shingles",
+                               orphaned->len);
+
+               if (orphaned->len > 0) {
+                       /* Need to delete orphaned elements */
+                       rspamd_fuzzy_backend_run_simple (
+                                       RSPAMD_FUZZY_BACKEND_TRANSACTION_START,
+                                       backend,
+                                       NULL);
+
+                       for (i = 0; i < (gint)orphaned->len; i++) {
+                               pelt = &g_array_index (orphaned, struct orphaned_shingle_elt, i);
+                               rspamd_fuzzy_backend_run_stmt (backend,
+                                               RSPAMD_FUZZY_BACKEND_DELETE_ORPHANED,
+                                               pelt->value, pelt->number);
+                       }
+
+                       rspamd_fuzzy_backend_run_simple (
+                                       RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT,
+                                       backend,
+                                       NULL);
+                       msg_info_fuzzy_backend ("deleted %ud orphaned shingles", orphaned->len);
+               }
+
+               g_array_free (orphaned, TRUE);
+       }
+
        rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_VACUUM, backend, NULL);
 
        if (rspamd_fuzzy_backend_run_stmt (backend, RSPAMD_FUZZY_BACKEND_COUNT)