From: Vsevolod Stakhov Date: Mon, 23 May 2016 15:37:16 +0000 (+0100) Subject: [Feature] Allow versioning for sqlite databases X-Git-Tag: 1.3.0~451 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=c15fd65bc3e7f447a816e9539afa92f444f53480;p=rspamd.git [Feature] Allow versioning for sqlite databases --- diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c index 26e595e5f..520eb3c02 100644 --- a/src/libserver/fuzzy_backend.c +++ b/src/libserver/fuzzy_backend.c @@ -52,13 +52,13 @@ static const guint max_retries = 10; static const char *create_tables_sql = "BEGIN;" - "CREATE TABLE digests(" + "CREATE TABLE IF NOT EXISTS digests(" "id INTEGER PRIMARY KEY," "flag INTEGER NOT NULL," "digest TEXT NOT NULL," "value INTEGER," "time INTEGER);" - "CREATE TABLE shingles(" + "CREATE TABLE IF NOT EXISTS shingles(" "value INTEGER NOT NULL," "number INTEGER NOT NULL," "digest_id INTEGER REFERENCES digests(id) ON DELETE CASCADE " @@ -410,7 +410,7 @@ rspamd_fuzzy_backend_open_db (const gchar *path, GError **err) bk->expired = 0; bk->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "fuzzy_backend"); bk->db = rspamd_sqlite3_open_or_create (bk->pool, bk->path, - create_tables_sql, err); + create_tables_sql, 0, err); if (bk->db == NULL) { rspamd_fuzzy_backend_close (bk); diff --git a/src/libstat/backends/sqlite3_backend.c b/src/libstat/backends/sqlite3_backend.c index f90e6b93a..30c9c74a2 100644 --- a/src/libstat/backends/sqlite3_backend.c +++ b/src/libstat/backends/sqlite3_backend.c @@ -438,7 +438,8 @@ rspamd_sqlite3_opendb (rspamd_mempool_t *pool, }; bk = g_slice_alloc0 (sizeof (*bk)); - bk->sqlite = rspamd_sqlite3_open_or_create (pool, path, create_tables_sql, err); + bk->sqlite = rspamd_sqlite3_open_or_create (pool, path, create_tables_sql, + 0, err); bk->pool = pool; if (bk->sqlite == NULL) { diff --git a/src/libstat/learn_cache/sqlite3_cache.c b/src/libstat/learn_cache/sqlite3_cache.c index 61335ab34..48cfe4af7 100644 --- a/src/libstat/learn_cache/sqlite3_cache.c +++ b/src/libstat/learn_cache/sqlite3_cache.c @@ -135,7 +135,7 @@ rspamd_stat_cache_sqlite3_init (struct rspamd_stat_ctx *ctx, rspamd_snprintf (dbpath, sizeof (dbpath), "%s", path); sqlite = rspamd_sqlite3_open_or_create (cfg->cfg_pool, - dbpath, create_tables_sql, &err); + dbpath, create_tables_sql, 0, &err); if (sqlite == NULL) { msg_err ("cannot open sqlite3 cache: %e", err); diff --git a/src/libutil/sqlite_utils.c b/src/libutil/sqlite_utils.c index 452559c76..e7768142c 100644 --- a/src/libutil/sqlite_utils.c +++ b/src/libutil/sqlite_utils.c @@ -248,7 +248,7 @@ rspamd_sqlite3_wait (rspamd_mempool_t *pool, const gchar *lock) sqlite3 * rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const - gchar *create_sql, GError **err) + gchar *create_sql, guint version, GError **err) { sqlite3 *sqlite; gint rc, flags, lock_fd; @@ -268,7 +268,8 @@ rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const other_pragmas[] = "PRAGMA read_uncommitted=\"ON\";" "PRAGMA cache_size=" - G_STRINGIFY(RSPAMD_SQLITE_CACHE_SIZE) ";"; + G_STRINGIFY(RSPAMD_SQLITE_CACHE_SIZE) ";", + db_version[] = "PRAGMA user_version;"; gboolean create = FALSE, has_lock = FALSE; flags = SQLITE_OPEN_READWRITE; @@ -401,6 +402,79 @@ rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const return NULL; } } + else if (has_lock && version > 0) { + /* Check user version */ + sqlite3_stmt *stmt = NULL; + guint32 db_ver; + GString *new_ver_sql; + + if (sqlite3_prepare (sqlite, db_version, -1, &stmt, NULL) != SQLITE_OK) { + msg_warn_pool_check ("Cannot get user version pragma", + sqlite3_errmsg (sqlite)); + } + else { + if (sqlite3_step (stmt) != SQLITE_ROW) { + msg_warn_pool_check ("Cannot get user version pragma, step failed", + sqlite3_errmsg (sqlite)); + sqlite3_finalize (stmt); + } + else { + db_ver = sqlite3_column_int (stmt, 0); + sqlite3_reset (stmt); + sqlite3_finalize (stmt); + + if (version > db_ver) { + msg_warn_pool_check ("Database version %ud is less than " + "desired version %ud, run create script", db_ver, + version); + + if (create_sql) { + if (sqlite3_exec (sqlite, create_sql, NULL, NULL, NULL) != SQLITE_OK) { + g_set_error (err, rspamd_sqlite3_quark (), + -1, "cannot execute create sql `%s`: %s", + create_sql, sqlite3_errmsg (sqlite)); + sqlite3_close (sqlite); + rspamd_file_unlock (lock_fd, FALSE); + unlink (lock_path); + if (lock_fd != -1) { + close (lock_fd); + } + + return NULL; + } + } + + new_ver_sql = g_string_new ("PRAGMA user_version="); + rspamd_printf_gstring (new_ver_sql, "%ud", version); + + if (sqlite3_exec (sqlite, new_ver_sql->str, NULL, NULL, NULL) + != SQLITE_OK) { + g_set_error (err, rspamd_sqlite3_quark (), + -1, "cannot execute update version sql `%s`: %s", + new_ver_sql->str, sqlite3_errmsg (sqlite)); + sqlite3_close (sqlite); + rspamd_file_unlock (lock_fd, FALSE); + unlink (lock_path); + if (lock_fd != -1) { + close (lock_fd); + } + + g_string_free (new_ver_sql, TRUE); + + return NULL; + } + + g_string_free (new_ver_sql, TRUE); + } + else if (db_ver > version) { + msg_warn_pool_check ("Database version %ud is more than " + "desired version %ud, this could cause" + " unexpected behaviour", db_ver, + version); + } + } + } + } if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) { msg_warn_pool_check ("WAL mode is not supported (%s), locking issues might occur", diff --git a/src/libutil/sqlite_utils.h b/src/libutil/sqlite_utils.h index f25be655e..78ee26fc6 100644 --- a/src/libutil/sqlite_utils.h +++ b/src/libutil/sqlite_utils.h @@ -69,7 +69,7 @@ void rspamd_sqlite3_close_prstmt (sqlite3 *db, GArray *stmts); * @return */ sqlite3 * rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const - gchar *create_sql, GError **err); + gchar *create_sql, guint32 version, GError **err); /** diff --git a/src/lua/lua_sqlite3.c b/src/lua/lua_sqlite3.c index ca8481117..e15673702 100644 --- a/src/lua/lua_sqlite3.c +++ b/src/lua/lua_sqlite3.c @@ -98,7 +98,7 @@ lua_sqlite3_open (lua_State *L) return 1; } - db = rspamd_sqlite3_open_or_create (NULL, path, NULL, &err); + db = rspamd_sqlite3_open_or_create (NULL, path, NULL, 0, &err); if (db == NULL) { if (err) { diff --git a/src/rspamadm/fuzzy_merge.c b/src/rspamadm/fuzzy_merge.c index 13bef43ef..931bea0f0 100644 --- a/src/rspamadm/fuzzy_merge.c +++ b/src/rspamadm/fuzzy_merge.c @@ -256,7 +256,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv) pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "fuzzy_merge"); dest_db = rspamd_sqlite3_open_or_create (pool, target, create_tables_sql, - &error); + 0, &error); if (dest_db == NULL) { rspamd_fprintf(stderr, "cannot open destination: %s\n", error->message); @@ -281,7 +281,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv) unique_ops = g_hash_table_new (rspamadm_op_hash, rspamadm_op_equal); for (i = 0; i < nsrc; i++) { - src = rspamd_sqlite3_open_or_create (pool, sources[i], NULL, &error); + src = rspamd_sqlite3_open_or_create (pool, sources[i], NULL, 0, &error); if (src == NULL) { rspamd_fprintf(stderr, "cannot open source %s: %s\n", sources[i], @@ -299,7 +299,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv) nsrc_shingles = 0; src = g_ptr_array_index (source_dbs, i); - + if (!quiet) { rspamd_printf ("reading data from %s\n", sources[i]); } @@ -449,7 +449,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv) sqlite3_finalize (stmt); sqlite3_close (src); } - + if (!quiet) { rspamd_printf ("start writing to %s, %ud ops pending\n", target, ops->len); }