From 9c44caea68a12837ab2c99fddd6ba5e7b907240f Mon Sep 17 00:00:00 2001
From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Mon, 28 Sep 2015 11:24:34 +0100
Subject: [PATCH] First attempt to clean orphaned elements.

---
 src/libserver/fuzzy_backend.c | 76 ++++++++++++++++++++++++++++++++---
 1 file changed, 70 insertions(+), 6 deletions(-)

diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c
index 016ee2b5a..ce7d5b5d1 100644
--- a/src/libserver/fuzzy_backend.c
+++ b/src/libserver/fuzzy_backend.c
@@ -86,6 +86,7 @@ enum rspamd_fuzzy_statement_idx {
 	RSPAMD_FUZZY_BACKEND_COUNT,
 	RSPAMD_FUZZY_BACKEND_EXPIRE,
 	RSPAMD_FUZZY_BACKEND_VACUUM,
+	RSPAMD_FUZZY_BACKEND_DELETE_ORPHANED,
 	RSPAMD_FUZZY_BACKEND_MAX
 };
 static struct rspamd_fuzzy_stmts {
@@ -189,7 +190,14 @@ static struct rspamd_fuzzy_stmts {
 		.args = "",
 		.stmt = NULL,
 		.result = SQLITE_DONE
-	}
+	},
+	{
+		.idx = RSPAMD_FUZZY_BACKEND_DELETE_ORPHANED,
+		.sql = "DELETE FROM shingles WHERE value=?1 AND number=?2;",
+		.args = "II",
+		.stmt = NULL,
+		.result = SQLITE_DONE
+	},
 };
 
 static GQuark
@@ -404,13 +412,26 @@ rspamd_fuzzy_backend_open_db (const gchar *path, GError **err)
 struct rspamd_fuzzy_backend*
 rspamd_fuzzy_backend_open (const gchar *path, GError **err)
 {
+	struct orphaned_shingle_elt {
+		gint64 value;
+		gint64 number;
+	};
+
 	gchar *dir;
 	gint fd;
 	struct rspamd_fuzzy_backend *backend;
 	static const char sqlite_wal[] = "PRAGMA journal_mode=\"wal\";",
 			fallback_journal[] = "PRAGMA journal_mode=\"off\";",
-			foreign_keys[] = "PRAGMA foreign_keys=\"ON\";";
-	int rc;
+			foreign_keys[] = "PRAGMA foreign_keys=\"ON\";",
+			orphaned_shingles[] =	"SELECT shingles.value,shingles.number "
+									"FROM shingles "
+									"LEFT JOIN digests ON "
+									"shingles.digest_id=digests.id WHERE "
+									"digests.id IS NULL;";
+	sqlite3_stmt *stmt;
+	GArray *orphaned;
+	struct orphaned_shingle_elt orphaned_elt, *pelt;
+	gint rc, i;
 
 	if (path == NULL) {
 		g_set_error (err, rspamd_fuzzy_backend_quark (),
@@ -462,17 +483,60 @@ rspamd_fuzzy_backend_open (const gchar *path, GError **err)
 	}
 
 	if ((rc = sqlite3_exec (backend->db, sqlite_wal, NULL, NULL, NULL)) != SQLITE_OK) {
-		msg_warn_fuzzy_backend ("WAL mode is not supported (%d), locking issues might occur",
-				rc);
+		msg_warn_fuzzy_backend ("WAL mode is not supported (%s), locking issues might occur",
+				sqlite3_errmsg (backend->db));
 		sqlite3_exec (backend->db, fallback_journal, NULL, NULL, NULL);
 	}
 
 	if ((rc = sqlite3_exec (backend->db, foreign_keys, NULL, NULL, NULL)) !=
 			SQLITE_OK) {
-		msg_warn_fuzzy_backend ("foreign keys are not supported", rc);
+		msg_warn_fuzzy_backend ("foreign keys are not supported: %s",
+				sqlite3_errmsg (backend->db));
 	}
 
 	/* Cleanup database */
+	if ((rc = sqlite3_prepare_v2 (backend->db, orphaned_shingles, -1, &stmt,
+				NULL)) != SQLITE_OK) {
+		msg_warn_fuzzy_backend ("cannot cleanup shingles: %s",
+				sqlite3_errmsg (backend->db));
+	}
+	else {
+		orphaned = g_array_new (FALSE, FALSE, sizeof (struct orphaned_shingle_elt));
+
+		while (sqlite3_step (stmt) == SQLITE_ROW) {
+			orphaned_elt.value = sqlite3_column_int64 (stmt, 0);
+			orphaned_elt.number = sqlite3_column_int64 (stmt, 1);
+			g_array_append_val (orphaned, orphaned_elt);
+		}
+
+		sqlite3_finalize (stmt);
+		msg_info_fuzzy_backend ("going to delete %ud orphaned shingles",
+				orphaned->len);
+
+		if (orphaned->len > 0) {
+			/* Need to delete orphaned elements */
+			rspamd_fuzzy_backend_run_simple (
+					RSPAMD_FUZZY_BACKEND_TRANSACTION_START,
+					backend,
+					NULL);
+
+			for (i = 0; i < (gint)orphaned->len; i++) {
+				pelt = &g_array_index (orphaned, struct orphaned_shingle_elt, i);
+				rspamd_fuzzy_backend_run_stmt (backend,
+						RSPAMD_FUZZY_BACKEND_DELETE_ORPHANED,
+						pelt->value, pelt->number);
+			}
+
+			rspamd_fuzzy_backend_run_simple (
+					RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT,
+					backend,
+					NULL);
+			msg_info_fuzzy_backend ("deleted %ud orphaned shingles", orphaned->len);
+		}
+
+		g_array_free (orphaned, TRUE);
+	}
+
 	rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_VACUUM, backend, NULL);
 
 	if (rspamd_fuzzy_backend_run_stmt (backend, RSPAMD_FUZZY_BACKEND_COUNT)
-- 
2.39.5