From: Vsevolod Stakhov Date: Sat, 20 Dec 2014 13:00:54 +0000 (+0000) Subject: Fix old database conversion. X-Git-Tag: 0.8.0~57 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=b0b003e18e947d33186746fc989216dc55918f61;p=rspamd.git Fix old database conversion. --- diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c index 6f0439a9d..c70afb445 100644 --- a/src/libserver/fuzzy_backend.c +++ b/src/libserver/fuzzy_backend.c @@ -43,10 +43,28 @@ struct rspamd_fuzzy_backend { char *path; }; + +const char *create_tables_sql = + "BEGIN;" + "CREATE TABLE digests(" + "id INTEGER PRIMARY KEY," + "flag INTEGER NOT NULL," + "digest TEXT NOT NULL," + "value INTEGER," + "time INTEGER);" + "CREATE TABLE shingles(" + "value INTEGER NOT NULL," + "number INTEGER NOT NULL," + "digest_id INTEGER REFERENCES digests(id) ON DELETE CASCADE " + "ON UPDATE CASCADE);" + "COMMIT;"; +const char *create_index_sql = + "BEGIN;" + "CREATE UNIQUE INDEX IF NOT EXISTS d ON digests(digest, flag);" + "CREATE UNIQUE INDEX IF NOT EXISTS s ON shingles(value, number);" + "COMMIT;"; enum rspamd_fuzzy_statement_idx { - RSPAMD_FUZZY_BACKEND_CREATE = 0, - RSPAMD_FUZZY_BACKEND_INDEX, - RSPAMD_FUZZY_BACKEND_TRANSACTION_START, + RSPAMD_FUZZY_BACKEND_TRANSACTION_START = 0, RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT, RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK, RSPAMD_FUZZY_BACKEND_INSERT, @@ -61,81 +79,66 @@ static struct rspamd_fuzzy_stmts { const gchar *sql; const gchar *args; sqlite3_stmt *stmt; + gint result; } prepared_stmts[RSPAMD_FUZZY_BACKEND_MAX] = { - { - .idx = RSPAMD_FUZZY_BACKEND_CREATE, - .sql = "CREATE TABLE digests(" - "id INTEGER PRIMARY KEY," - "flag INTEGER NOT NULL," - "digest TEXT NOT NULL," - "value INTEGER," - "time INTEGER);" - "" - "CREATE TABLE shingles(" - "value INTEGER NOT NULL," - "number INTEGER NOT NULL," - "digest_id INTEGER REFERENCES digests(id) ON DELETE CASCADE" - "ON UPDATE CASCADE);", - .args = "", - .stmt = NULL - }, - { - .idx = RSPAMD_FUZZY_BACKEND_INDEX, - .sql = "CREATE UNIQUE INDEX IF NOT EXISTS d ON digests(digest, flag);" - "CREATE UNIQUE INDEX IF NOT EXISTS s ON shingles(value, number);", - .args = "", - .stmt = NULL - }, { .idx = RSPAMD_FUZZY_BACKEND_TRANSACTION_START, .sql = "BEGIN TRANSACTION;", .args = "", - .stmt = NULL + .stmt = NULL, + .result = SQLITE_DONE }, { .idx = RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT, .sql = "COMMIT;", .args = "", - .stmt = NULL + .stmt = NULL, + .result = SQLITE_DONE }, { .idx = RSPAMD_FUZZY_BACKEND_TRANSACTION_ROLLBACK, .sql = "ROLLBACK;", .args = "", - .stmt = NULL + .stmt = NULL, + .result = SQLITE_DONE }, { .idx = RSPAMD_FUZZY_BACKEND_INSERT, .sql = "INSERT INTO digests(flag, digest, value, time) VALUES" "(?1, ?2, ?3, ?4);", .args = "STII", - .stmt = NULL + .stmt = NULL, + .result = SQLITE_DONE }, { .idx = RSPAMD_FUZZY_BACKEND_INSERT_SHINGLE, .sql = "INSERT INTO shingles(value, number, digest_id) " "VALUES (?1, ?2, ?3);", .args = "III", - .stmt = NULL + .stmt = NULL, + .result = SQLITE_DONE }, { .idx = RSPAMD_FUZZY_BACKEND_CHECK, .sql = "SELECT * FROM digests WHERE digest==?1 AND flag==?2;", .args = "TI", - .stmt = NULL + .stmt = NULL, + .result = SQLITE_ROW }, { .idx = RSPAMD_FUZZY_BACKEND_CHECK_SHINGLE, .sql = "", .args = "", - .stmt = NULL + .stmt = NULL, + .result = SQLITE_ROW }, { .idx = RSPAMD_FUZZY_BACKEND_DELETE, .sql = "", .args = "", - .stmt = NULL + .stmt = NULL, + .result = SQLITE_DONE } }; @@ -151,10 +154,14 @@ rspamd_fuzzy_backend_prepare_stmts (struct rspamd_fuzzy_backend *bk, GError **er int i; for (i = 0; i < RSPAMD_FUZZY_BACKEND_MAX; i ++) { + if (prepared_stmts[i].stmt != NULL) { + /* Skip already prepared statements */ + continue; + } if (sqlite3_prepare_v2 (bk->db, prepared_stmts[i].sql, -1, &prepared_stmts[i].stmt, NULL) != SQLITE_OK) { g_set_error (err, rspamd_fuzzy_backend_quark (), - -1, "Cannot initialize prepared sql %s: %s", + -1, "Cannot initialize prepared sql `%s`: %s", prepared_stmts[i].sql, sqlite3_errmsg (bk->db)); return FALSE; @@ -165,7 +172,7 @@ rspamd_fuzzy_backend_prepare_stmts (struct rspamd_fuzzy_backend *bk, GError **er } static int -rspamd_fuzzy_backend_run_stmt (int idx, ...) +rspamd_fuzzy_backend_run_stmt (struct rspamd_fuzzy_backend *bk, int idx, ...) { int retcode; va_list ap; @@ -179,6 +186,18 @@ rspamd_fuzzy_backend_run_stmt (int idx, ...) } stmt = prepared_stmts[idx].stmt; + if (stmt == NULL) { + if ((retcode = sqlite3_prepare_v2 (bk->db, prepared_stmts[idx].sql, -1, + &prepared_stmts[idx].stmt, NULL)) != SQLITE_OK) { + msg_err ("Cannot initialize prepared sql `%s`: %s", + prepared_stmts[idx].sql, sqlite3_errmsg (bk->db)); + + return retcode; + } + stmt = prepared_stmts[idx].stmt; + } + + msg_debug ("executing `%s`", prepared_stmts[idx].sql); argtypes = prepared_stmts[idx].args; sqlite3_reset (stmt); va_start (ap, idx); @@ -201,6 +220,10 @@ rspamd_fuzzy_backend_run_stmt (int idx, ...) va_end (ap); retcode = sqlite3_step (stmt); + if (retcode == prepared_stmts[idx].result) { + return SQLITE_OK; + } + return retcode; } @@ -223,10 +246,9 @@ static gboolean rspamd_fuzzy_backend_run_simple (int idx, struct rspamd_fuzzy_backend *bk, GError **err) { - if (rspamd_fuzzy_backend_run_stmt (idx) - != SQLITE_OK) { + if (rspamd_fuzzy_backend_run_stmt (bk, idx) != SQLITE_OK) { g_set_error (err, rspamd_fuzzy_backend_quark (), - -1, "Cannot execute sql %s: %s", + -1, "Cannot execute sql `%s`: %s", prepared_stmts[idx].sql, sqlite3_errmsg (bk->db)); return FALSE; @@ -235,6 +257,20 @@ rspamd_fuzzy_backend_run_simple (int idx, struct rspamd_fuzzy_backend *bk, return TRUE; } +static gboolean +rspamd_fuzzy_backend_run_sql (const gchar *sql, struct rspamd_fuzzy_backend *bk, + GError **err) +{ + if (sqlite3_exec (bk->db, sql, NULL, NULL, NULL) != SQLITE_OK) { + g_set_error (err, rspamd_fuzzy_backend_quark (), + -1, "Cannot execute raw sql `%s`: %s", + sql, sqlite3_errmsg (bk->db)); + return FALSE; + } + + return TRUE; +} + static struct rspamd_fuzzy_backend * rspamd_fuzzy_backend_create_db (const gchar *path, gboolean add_index, GError **err) @@ -257,25 +293,22 @@ rspamd_fuzzy_backend_create_db (const gchar *path, gboolean add_index, bk->path = g_strdup (path); bk->db = sqlite; - if (!rspamd_fuzzy_backend_prepare_stmts (bk, err)) { + /* + * Here we need to run create prior to preparing other statements + */ + if (!rspamd_fuzzy_backend_run_sql (create_tables_sql, bk, err)) { rspamd_fuzzy_backend_close (bk); - return NULL; } - if (rspamd_fuzzy_backend_run_stmt (RSPAMD_FUZZY_BACKEND_CREATE) - != SQLITE_OK) { - g_set_error (err, rspamd_fuzzy_backend_quark (), - -1, "Cannot execute init sql %s: %s", - prepared_stmts[RSPAMD_FUZZY_BACKEND_CREATE].sql, - sqlite3_errmsg (bk->db)); + if (!rspamd_fuzzy_backend_prepare_stmts (bk, err)) { rspamd_fuzzy_backend_close (bk); return NULL; } if (add_index) { - rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_INDEX, bk, NULL); + rspamd_fuzzy_backend_run_sql (create_index_sql, bk, NULL); } rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_TRANSACTION_START, @@ -335,9 +368,9 @@ rspamd_fuzzy_backend_convert (const gchar *path, int fd, GError **err) (void)fstat (fd, &st); (void)lseek (fd, 0, SEEK_SET); - off = sizeof (FUZZY_FILE_MAGIC) - 1; + off = sizeof (FUZZY_FILE_MAGIC); if ((map = mmap (NULL, st.st_size - off, PROT_READ, MAP_SHARED, fd, - off)) == MAP_FAILED) { + 0)) == MAP_FAILED) { g_set_error (err, rspamd_fuzzy_backend_quark (), errno, "Cannot mmap file %s: %s", path, strerror (errno)); @@ -346,29 +379,28 @@ rspamd_fuzzy_backend_convert (const gchar *path, int fd, GError **err) return FALSE; } - end = map + st.st_size - off; - p = map; + end = map + st.st_size; + p = map + off; rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_TRANSACTION_START, nbackend, NULL); while (p < end) { n = (struct rspamd_legacy_fuzzy_node *)p; /* Convert node flag, digest, value, time */ - if (rspamd_fuzzy_backend_run_stmt (RSPAMD_FUZZY_BACKEND_INSERT, + if (rspamd_fuzzy_backend_run_stmt (nbackend, RSPAMD_FUZZY_BACKEND_INSERT, (gint)n->flag, n->h.hash_pipe, (gint64)n->value, n->time) != SQLITE_OK) { msg_warn ("Cannot execute init sql %s: %s", - prepared_stmts[RSPAMD_FUZZY_BACKEND_CREATE].sql, + prepared_stmts[RSPAMD_FUZZY_BACKEND_INSERT].sql, sqlite3_errmsg (nbackend->db)); } p += sizeof (struct rspamd_legacy_fuzzy_node); } - munmap (map, st.st_size - off); + munmap (map, st.st_size); rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_TRANSACTION_COMMIT, nbackend, NULL); - rspamd_fuzzy_backend_run_simple (RSPAMD_FUZZY_BACKEND_INDEX, - nbackend, NULL); + rspamd_fuzzy_backend_run_sql (create_index_sql, nbackend, NULL); rspamd_fuzzy_backend_close (nbackend); rename (tmpdb, path);