[Feature] Allow versioning for sqlite databases

diff --git a/src/libserver/fuzzy_backend.c b/src/libserver/fuzzy_backend.c
index 26e595e5f5fc64541b422b0662ae4947c03f8943..520eb3c0221f91cdb7211d346207d1d25ceac249 100644 (file)
--- a/src/libserver/fuzzy_backend.c
+++ b/src/libserver/fuzzy_backend.c
@@ -52,13 +52,13 @@ static const guint max_retries = 10;
 
 static const char *create_tables_sql =
                "BEGIN;"
-               "CREATE TABLE digests("
+               "CREATE TABLE IF NOT EXISTS digests("
                "id INTEGER PRIMARY KEY,"
                "flag INTEGER NOT NULL,"
                "digest TEXT NOT NULL,"
                "value INTEGER,"
                "time INTEGER);"
-               "CREATE TABLE shingles("
+               "CREATE TABLE IF NOT EXISTS shingles("
                "value INTEGER NOT NULL,"
                "number INTEGER NOT NULL,"
                "digest_id INTEGER REFERENCES digests(id) ON DELETE CASCADE "
@@ -410,7 +410,7 @@ rspamd_fuzzy_backend_open_db (const gchar *path, GError **err)
        bk->expired = 0;
        bk->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "fuzzy_backend");
        bk->db = rspamd_sqlite3_open_or_create (bk->pool, bk->path,
-                       create_tables_sql, err);
+                       create_tables_sql, 0, err);
 
        if (bk->db == NULL) {
                rspamd_fuzzy_backend_close (bk);

diff --git a/src/libstat/backends/sqlite3_backend.c b/src/libstat/backends/sqlite3_backend.c
index f90e6b93a3628e834abb92285c61cefed96b8309..30c9c74a218f31b0f542f53132b940078ba07bad 100644 (file)
--- a/src/libstat/backends/sqlite3_backend.c
+++ b/src/libstat/backends/sqlite3_backend.c
@@ -438,7 +438,8 @@ rspamd_sqlite3_opendb (rspamd_mempool_t *pool,
        };
 
        bk = g_slice_alloc0 (sizeof (*bk));
-       bk->sqlite = rspamd_sqlite3_open_or_create (pool, path, create_tables_sql, err);
+       bk->sqlite = rspamd_sqlite3_open_or_create (pool, path, create_tables_sql,
+                       0, err);
        bk->pool = pool;
 
        if (bk->sqlite == NULL) {

diff --git a/src/libstat/learn_cache/sqlite3_cache.c b/src/libstat/learn_cache/sqlite3_cache.c
index 61335ab3420396838d37471e2ffd0ba33cffd256..48cfe4af78a2f8f403975e95218e9a215926458e 100644 (file)
--- a/src/libstat/learn_cache/sqlite3_cache.c
+++ b/src/libstat/learn_cache/sqlite3_cache.c
@@ -135,7 +135,7 @@ rspamd_stat_cache_sqlite3_init (struct rspamd_stat_ctx *ctx,
        rspamd_snprintf (dbpath, sizeof (dbpath), "%s", path);
 
        sqlite = rspamd_sqlite3_open_or_create (cfg->cfg_pool,
-                       dbpath, create_tables_sql, &err);
+                       dbpath, create_tables_sql, 0, &err);
 
        if (sqlite == NULL) {
                msg_err ("cannot open sqlite3 cache: %e", err);

diff --git a/src/libutil/sqlite_utils.c b/src/libutil/sqlite_utils.c
index 452559c76766906a5e1d446556ee5102b7ffaefa..e7768142cc1953ba420fa96f941f580011640699 100644 (file)
--- a/src/libutil/sqlite_utils.c
+++ b/src/libutil/sqlite_utils.c
@@ -248,7 +248,7 @@ rspamd_sqlite3_wait (rspamd_mempool_t *pool, const gchar *lock)
 
 sqlite3 *
 rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
-               gchar *create_sql, GError **err)
+               gchar *create_sql, guint version, GError **err)
 {
        sqlite3 *sqlite;
        gint rc, flags, lock_fd;
@@ -268,7 +268,8 @@ rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
 
                        other_pragmas[] =               "PRAGMA read_uncommitted=\"ON\";"
                                                                        "PRAGMA cache_size="
-                                                                       G_STRINGIFY(RSPAMD_SQLITE_CACHE_SIZE) ";";
+                                                                       G_STRINGIFY(RSPAMD_SQLITE_CACHE_SIZE) ";",
+                       db_version[] =                  "PRAGMA user_version;";
        gboolean create = FALSE, has_lock = FALSE;
 
        flags = SQLITE_OPEN_READWRITE;
@@ -401,6 +402,79 @@ rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
                        return NULL;
                }
        }
+       else if (has_lock && version > 0) {
+               /* Check user version */
+               sqlite3_stmt *stmt = NULL;
+               guint32 db_ver;
+               GString *new_ver_sql;
+
+               if (sqlite3_prepare (sqlite, db_version, -1, &stmt, NULL) != SQLITE_OK) {
+                       msg_warn_pool_check ("Cannot get user version pragma",
+                                                       sqlite3_errmsg (sqlite));
+               }
+               else {
+                       if (sqlite3_step (stmt) != SQLITE_ROW) {
+                               msg_warn_pool_check ("Cannot get user version pragma, step failed",
+                                                                                       sqlite3_errmsg (sqlite));
+                               sqlite3_finalize (stmt);
+                       }
+                       else {
+                               db_ver = sqlite3_column_int (stmt, 0);
+                               sqlite3_reset (stmt);
+                               sqlite3_finalize (stmt);
+
+                               if (version > db_ver) {
+                                       msg_warn_pool_check ("Database version %ud is less than "
+                                                       "desired version %ud, run create script", db_ver,
+                                                       version);
+
+                                       if (create_sql) {
+                                               if (sqlite3_exec (sqlite, create_sql, NULL, NULL, NULL) != SQLITE_OK) {
+                                                       g_set_error (err, rspamd_sqlite3_quark (),
+                                                                       -1, "cannot execute create sql `%s`: %s",
+                                                                       create_sql, sqlite3_errmsg (sqlite));
+                                                       sqlite3_close (sqlite);
+                                                       rspamd_file_unlock (lock_fd, FALSE);
+                                                       unlink (lock_path);
+                                                       if (lock_fd != -1) {
+                                                               close (lock_fd);
+                                                       }
+
+                                                       return NULL;
+                                               }
+                                       }
+
+                                       new_ver_sql = g_string_new ("PRAGMA user_version=");
+                                       rspamd_printf_gstring (new_ver_sql, "%ud", version);
+
+                                       if (sqlite3_exec (sqlite, new_ver_sql->str, NULL, NULL, NULL)
+                                                       != SQLITE_OK) {
+                                               g_set_error (err, rspamd_sqlite3_quark (),
+                                                               -1, "cannot execute update version sql `%s`: %s",
+                                                               new_ver_sql->str, sqlite3_errmsg (sqlite));
+                                               sqlite3_close (sqlite);
+                                               rspamd_file_unlock (lock_fd, FALSE);
+                                               unlink (lock_path);
+                                               if (lock_fd != -1) {
+                                                       close (lock_fd);
+                                               }
+
+                                               g_string_free (new_ver_sql, TRUE);
+
+                                               return NULL;
+                                       }
+
+                                       g_string_free (new_ver_sql, TRUE);
+                               }
+                               else if (db_ver > version) {
+                                       msg_warn_pool_check ("Database version %ud is more than "
+                                                       "desired version %ud, this could cause"
+                                                       " unexpected behaviour", db_ver,
+                                                       version);
+                               }
+                       }
+               }
+       }
 
        if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) {
                msg_warn_pool_check ("WAL mode is not supported (%s), locking issues might occur",

diff --git a/src/libutil/sqlite_utils.h b/src/libutil/sqlite_utils.h
index f25be655ec26b5260c6abc9ac1618a77cd8369ef..78ee26fc64d5a5400b7f02915575aad1f2ea8ead 100644 (file)
--- a/src/libutil/sqlite_utils.h
+++ b/src/libutil/sqlite_utils.h
@@ -69,7 +69,7 @@ void rspamd_sqlite3_close_prstmt (sqlite3 *db, GArray *stmts);
  * @return
  */
 sqlite3 * rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
-               gchar *create_sql, GError **err);
+               gchar *create_sql, guint32 version, GError **err);
 
 
 /**

diff --git a/src/lua/lua_sqlite3.c b/src/lua/lua_sqlite3.c
index ca8481117f48e026ef6b99e4526eee95862c64c9..e15673702a8565c57de55737bf2d3d2511859672 100644 (file)
--- a/src/lua/lua_sqlite3.c
+++ b/src/lua/lua_sqlite3.c
@@ -98,7 +98,7 @@ lua_sqlite3_open (lua_State *L)
                return 1;
        }
 
-       db = rspamd_sqlite3_open_or_create (NULL, path, NULL, &err);
+       db = rspamd_sqlite3_open_or_create (NULL, path, NULL, 0, &err);
 
        if (db == NULL) {
                if (err) {

diff --git a/src/rspamadm/fuzzy_merge.c b/src/rspamadm/fuzzy_merge.c
index 13bef43ef3d1ef8ba282c1acb74ad90394e9eda6..931bea0f08e3702c9e90a776305ed1fae7ed266a 100644 (file)
--- a/src/rspamadm/fuzzy_merge.c
+++ b/src/rspamadm/fuzzy_merge.c
@@ -256,7 +256,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv)
 
        pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "fuzzy_merge");
        dest_db = rspamd_sqlite3_open_or_create (pool, target, create_tables_sql,
-                       &error);
+                       0, &error);
 
        if (dest_db == NULL) {
                rspamd_fprintf(stderr, "cannot open destination: %s\n", error->message);
@@ -281,7 +281,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv)
        unique_ops = g_hash_table_new (rspamadm_op_hash, rspamadm_op_equal);
 
        for (i = 0; i < nsrc; i++) {
-               src = rspamd_sqlite3_open_or_create (pool, sources[i], NULL, &error);
+               src = rspamd_sqlite3_open_or_create (pool, sources[i], NULL, 0, &error);
 
                if (src == NULL) {
                        rspamd_fprintf(stderr, "cannot open source %s: %s\n", sources[i],
@@ -299,7 +299,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv)
                                nsrc_shingles = 0;
 
                src = g_ptr_array_index (source_dbs, i);
-               
+
                if (!quiet) {
                        rspamd_printf ("reading data from %s\n", sources[i]);
                }
@@ -449,7 +449,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv)
                sqlite3_finalize (stmt);
                sqlite3_close (src);
        }
-       
+
        if (!quiet) {
                rspamd_printf ("start writing to %s, %ud ops pending\n", target, ops->len);
        }