[Feature] Allow versioning for sqlite databases

src/libserver/fuzzy_backend.c

@@ -52,13 +52,13 @@ static const guint max_retries = 10;
 
 static const char *create_tables_sql =
 		"BEGIN;"
-		"CREATE TABLE digests("
+		"CREATE TABLE IF NOT EXISTS digests("
 		"id INTEGER PRIMARY KEY,"
 		"flag INTEGER NOT NULL,"
 		"digest TEXT NOT NULL,"
 		"value INTEGER,"
 		"time INTEGER);"
-		"CREATE TABLE shingles("
+		"CREATE TABLE IF NOT EXISTS shingles("
 		"value INTEGER NOT NULL,"
 		"number INTEGER NOT NULL,"
 		"digest_id INTEGER REFERENCES digests(id) ON DELETE CASCADE "
@@ -410,7 +410,7 @@ rspamd_fuzzy_backend_open_db (const gchar *path, GError **err)
 	bk->expired = 0;
 	bk->pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "fuzzy_backend");
 	bk->db = rspamd_sqlite3_open_or_create (bk->pool, bk->path,
-			create_tables_sql, err);
+			create_tables_sql, 0, err);
 
 	if (bk->db == NULL) {
 		rspamd_fuzzy_backend_close (bk);

src/libstat/backends/sqlite3_backend.c

@@ -438,7 +438,8 @@ rspamd_sqlite3_opendb (rspamd_mempool_t *pool,
 	};
 
 	bk = g_slice_alloc0 (sizeof (*bk));
-	bk->sqlite = rspamd_sqlite3_open_or_create (pool, path, create_tables_sql, err);
+	bk->sqlite = rspamd_sqlite3_open_or_create (pool, path, create_tables_sql,
+			0, err);
 	bk->pool = pool;
 
 	if (bk->sqlite == NULL) {

src/libstat/learn_cache/sqlite3_cache.c

@@ -135,7 +135,7 @@ rspamd_stat_cache_sqlite3_init (struct rspamd_stat_ctx *ctx,
 	rspamd_snprintf (dbpath, sizeof (dbpath), "%s", path);
 
 	sqlite = rspamd_sqlite3_open_or_create (cfg->cfg_pool,
-			dbpath, create_tables_sql, &err);
+			dbpath, create_tables_sql, 0, &err);
 
 	if (sqlite == NULL) {
 		msg_err ("cannot open sqlite3 cache: %e", err);

src/libutil/sqlite_utils.c

@@ -248,7 +248,7 @@ rspamd_sqlite3_wait (rspamd_mempool_t *pool, const gchar *lock)
 
 sqlite3 *
 rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
-		gchar *create_sql, GError **err)
+		gchar *create_sql, guint version, GError **err)
 {
 	sqlite3 *sqlite;
 	gint rc, flags, lock_fd;
@@ -268,7 +268,8 @@ rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
 
 			other_pragmas[] = 		"PRAGMA read_uncommitted=\"ON\";"
 									"PRAGMA cache_size="
-									G_STRINGIFY(RSPAMD_SQLITE_CACHE_SIZE) ";";
+									G_STRINGIFY(RSPAMD_SQLITE_CACHE_SIZE) ";",
+			db_version[] =			"PRAGMA user_version;";
 	gboolean create = FALSE, has_lock = FALSE;
 
 	flags = SQLITE_OPEN_READWRITE;
@@ -401,6 +402,79 @@ rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
 			return NULL;
 		}
 	}
+	else if (has_lock && version > 0) {
+		/* Check user version */
+		sqlite3_stmt *stmt = NULL;
+		guint32 db_ver;
+		GString *new_ver_sql;
+
+		if (sqlite3_prepare (sqlite, db_version, -1, &stmt, NULL) != SQLITE_OK) {
+			msg_warn_pool_check ("Cannot get user version pragma",
+							sqlite3_errmsg (sqlite));
+		}
+		else {
+			if (sqlite3_step (stmt) != SQLITE_ROW) {
+				msg_warn_pool_check ("Cannot get user version pragma, step failed",
+											sqlite3_errmsg (sqlite));
+				sqlite3_finalize (stmt);
+			}
+			else {
+				db_ver = sqlite3_column_int (stmt, 0);
+				sqlite3_reset (stmt);
+				sqlite3_finalize (stmt);
+
+				if (version > db_ver) {
+					msg_warn_pool_check ("Database version %ud is less than "
+							"desired version %ud, run create script", db_ver,
+							version);
+
+					if (create_sql) {
+						if (sqlite3_exec (sqlite, create_sql, NULL, NULL, NULL) != SQLITE_OK) {
+							g_set_error (err, rspamd_sqlite3_quark (),
+									-1, "cannot execute create sql `%s`: %s",
+									create_sql, sqlite3_errmsg (sqlite));
+							sqlite3_close (sqlite);
+							rspamd_file_unlock (lock_fd, FALSE);
+							unlink (lock_path);
+							if (lock_fd != -1) {
+								close (lock_fd);
+							}
+
+							return NULL;
+						}
+					}
+
+					new_ver_sql = g_string_new ("PRAGMA user_version=");
+					rspamd_printf_gstring (new_ver_sql, "%ud", version);
+
+					if (sqlite3_exec (sqlite, new_ver_sql->str, NULL, NULL, NULL)
+							!= SQLITE_OK) {
+						g_set_error (err, rspamd_sqlite3_quark (),
+								-1, "cannot execute update version sql `%s`: %s",
+								new_ver_sql->str, sqlite3_errmsg (sqlite));
+						sqlite3_close (sqlite);
+						rspamd_file_unlock (lock_fd, FALSE);
+						unlink (lock_path);
+						if (lock_fd != -1) {
+							close (lock_fd);
+						}
+
+						g_string_free (new_ver_sql, TRUE);
+
+						return NULL;
+					}
+
+					g_string_free (new_ver_sql, TRUE);
+				}
+				else if (db_ver > version) {
+					msg_warn_pool_check ("Database version %ud is more than "
+							"desired version %ud, this could cause"
+							" unexpected behaviour", db_ver,
+							version);
+				}
+			}
+		}
+	}
 
 	if (sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL) != SQLITE_OK) {
 		msg_warn_pool_check ("WAL mode is not supported (%s), locking issues might occur",

src/libutil/sqlite_utils.h

@@ -69,7 +69,7 @@ void rspamd_sqlite3_close_prstmt (sqlite3 *db, GArray *stmts);
  * @return
  */
 sqlite3 * rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
-		gchar *create_sql, GError **err);
+		gchar *create_sql, guint32 version, GError **err);
 
 
 /**

src/lua/lua_sqlite3.c

@@ -98,7 +98,7 @@ lua_sqlite3_open (lua_State *L)
 		return 1;
 	}
 
-	db = rspamd_sqlite3_open_or_create (NULL, path, NULL, &err);
+	db = rspamd_sqlite3_open_or_create (NULL, path, NULL, 0, &err);
 
 	if (db == NULL) {
 		if (err) {

src/rspamadm/fuzzy_merge.c

@@ -256,7 +256,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv)
 
 	pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), "fuzzy_merge");
 	dest_db = rspamd_sqlite3_open_or_create (pool, target, create_tables_sql,
-			&error);
+			0, &error);
 
 	if (dest_db == NULL) {
 		rspamd_fprintf(stderr, "cannot open destination: %s\n", error->message);
@@ -281,7 +281,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv)
 	unique_ops = g_hash_table_new (rspamadm_op_hash, rspamadm_op_equal);
 
 	for (i = 0; i < nsrc; i++) {
-		src = rspamd_sqlite3_open_or_create (pool, sources[i], NULL, &error);
+		src = rspamd_sqlite3_open_or_create (pool, sources[i], NULL, 0, &error);
 
 		if (src == NULL) {
 			rspamd_fprintf(stderr, "cannot open source %s: %s\n", sources[i],
@@ -299,7 +299,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv)
 				nsrc_shingles = 0;
 
 		src = g_ptr_array_index (source_dbs, i);
-		
+
 		if (!quiet) {
 			rspamd_printf ("reading data from %s\n", sources[i]);
 		}
@@ -449,7 +449,7 @@ rspamadm_fuzzy_merge (gint argc, gchar **argv)
 		sqlite3_finalize (stmt);
 		sqlite3_close (src);
 	}
-	
+
 	if (!quiet) {
 		rspamd_printf ("start writing to %s, %ud ops pending\n", target, ops->len);
 	}