123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629 |
- /*-
- * Copyright 2016 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- #include "config.h"
- #include "libserver/logger.h"
- #include "libutil/sqlite_utils.h"
- #include "unix-std.h"
-
-
- static GQuark
- rspamd_sqlite3_quark (void)
- {
- return g_quark_from_static_string ("rspamd-sqlite3");
- }
-
- GArray*
- rspamd_sqlite3_init_prstmt (sqlite3 *db,
- struct rspamd_sqlite3_prstmt *init_stmt,
- gint max_idx,
- GError **err)
- {
- gint i;
- GArray *res;
- struct rspamd_sqlite3_prstmt *nst;
-
- res = g_array_sized_new (FALSE, TRUE, sizeof (struct rspamd_sqlite3_prstmt),
- max_idx);
- g_array_set_size (res, max_idx);
-
- for (i = 0; i < max_idx; i ++) {
- nst = &g_array_index (res, struct rspamd_sqlite3_prstmt, i);
- memcpy (nst, &init_stmt[i], sizeof (*nst));
-
- if (sqlite3_prepare_v2 (db, init_stmt[i].sql, -1,
- &nst->stmt, NULL) != SQLITE_OK) {
- g_set_error (err, rspamd_sqlite3_quark (),
- -1, "Cannot initialize prepared sql `%s`: %s",
- nst->sql, sqlite3_errmsg (db));
- rspamd_sqlite3_close_prstmt (db, res);
-
- return NULL;
- }
- }
-
- return res;
- }
-
- int
- rspamd_sqlite3_run_prstmt (rspamd_mempool_t *pool, sqlite3 *db, GArray *stmts,
- gint idx, ...)
- {
- gint retcode;
- va_list ap;
- sqlite3_stmt *stmt;
- gint i, rowid, nargs, j;
- gint64 len;
- gpointer p;
- struct rspamd_sqlite3_prstmt *nst;
- const char *argtypes;
-
- if (idx < 0 || idx >= (gint)stmts->len) {
-
- return -1;
- }
-
- nst = &g_array_index (stmts, struct rspamd_sqlite3_prstmt, idx);
- stmt = nst->stmt;
-
- g_assert (nst != NULL);
-
- msg_debug_pool ("executing `%s`", nst->sql);
- argtypes = nst->args;
- sqlite3_clear_bindings (stmt);
- sqlite3_reset (stmt);
- va_start (ap, idx);
- nargs = 1;
-
- for (i = 0, rowid = 1; argtypes[i] != '\0'; i ++) {
- switch (argtypes[i]) {
- case 'T':
-
- for (j = 0; j < nargs; j ++, rowid ++) {
- sqlite3_bind_text (stmt, rowid, va_arg (ap, const char*), -1,
- SQLITE_STATIC);
- }
-
- nargs = 1;
- break;
- case 'V':
- case 'B':
-
- for (j = 0; j < nargs; j ++, rowid ++) {
- len = va_arg (ap, gint64);
- sqlite3_bind_text (stmt, rowid, va_arg (ap, const char*), len,
- SQLITE_STATIC);
- }
-
- nargs = 1;
- break;
- case 'I':
-
- for (j = 0; j < nargs; j ++, rowid ++) {
- sqlite3_bind_int64 (stmt, rowid, va_arg (ap, gint64));
- }
-
- nargs = 1;
- break;
- case 'S':
-
- for (j = 0; j < nargs; j ++, rowid ++) {
- sqlite3_bind_int (stmt, rowid, va_arg (ap, gint));
- }
-
- nargs = 1;
- break;
- case '*':
- nargs = va_arg (ap, gint);
- break;
- }
- }
-
- retcode = sqlite3_step (stmt);
-
- if (retcode == nst->result) {
- argtypes = nst->ret;
-
- for (i = 0; argtypes != NULL && argtypes[i] != '\0'; i ++) {
- switch (argtypes[i]) {
- case 'T':
- *va_arg (ap, char**) = g_strdup (sqlite3_column_text (stmt, i));
- break;
- case 'I':
- *va_arg (ap, gint64*) = sqlite3_column_int64 (stmt, i);
- break;
- case 'S':
- *va_arg (ap, int*) = sqlite3_column_int (stmt, i);
- break;
- case 'L':
- *va_arg (ap, gint64*) = sqlite3_last_insert_rowid (db);
- break;
- case 'B':
- len = sqlite3_column_bytes (stmt, i);
- g_assert (len >= 0);
- p = g_malloc (len);
- memcpy (p, sqlite3_column_blob (stmt, i), len);
- *va_arg (ap, gint64*) = len;
- *va_arg (ap, gpointer*) = p;
- break;
- }
- }
-
- if (!(nst->flags & RSPAMD_SQLITE3_STMT_MULTIPLE)) {
- sqlite3_clear_bindings (stmt);
- sqlite3_reset (stmt);
- }
-
- va_end (ap);
-
- return SQLITE_OK;
- }
- else if (retcode != SQLITE_DONE && retcode != SQLITE_OK && retcode != SQLITE_ROW) {
- msg_warn_pool ("failed to execute query %s: %d, %s", nst->sql,
- retcode, sqlite3_errmsg (db));
- }
-
- if (!(nst->flags & RSPAMD_SQLITE3_STMT_MULTIPLE)) {
- sqlite3_clear_bindings (stmt);
- sqlite3_reset (stmt);
- }
-
- va_end (ap);
-
- return retcode;
- }
-
- void
- rspamd_sqlite3_close_prstmt (sqlite3 *db, GArray *stmts)
- {
- guint i;
- struct rspamd_sqlite3_prstmt *nst;
-
- for (i = 0; i < stmts->len; i++) {
- nst = &g_array_index (stmts, struct rspamd_sqlite3_prstmt, i);
- if (nst->stmt != NULL) {
- sqlite3_finalize (nst->stmt);
- }
- }
-
- g_array_free (stmts, TRUE);
-
- return;
- }
-
- static gboolean
- rspamd_sqlite3_wait (rspamd_mempool_t *pool, const gchar *lock)
- {
- gint fd;
- pid_t pid;
- gssize r;
- struct timespec sleep_ts = {
- .tv_sec = 0,
- .tv_nsec = 1000000
- };
-
- while ((fd = open (lock, O_WRONLY|O_CREAT|O_EXCL, 00600)) == -1) {
- if (errno != EBUSY && errno != EEXIST) {
- msg_err_pool_check ("cannot open lock file %s: %s", lock,
- strerror (errno));
-
- return FALSE;
- }
-
- fd = open (lock, O_RDONLY);
-
- if (fd == -1) {
- msg_err_pool_check ("cannot open lock file %s: %s", lock,
- strerror (errno));
-
- return FALSE;
- }
-
- r = read (fd, &pid, sizeof (pid));
-
- if (r != sizeof (pid)) {
- msg_warn_pool_check ("stale lock file %s, removing", lock);
- unlink (lock);
- close (fd);
-
- return TRUE;
- }
-
- /* Now check for process existence */
- if (pid == getpid ()) {
- msg_warn_pool_check ("lock file %s, belongs to me, removing", lock);
- unlink (lock);
- close (fd);
-
- return TRUE;
- }
- else if (kill (pid, 0) == -1) {
- if (errno == ESRCH) {
- /* Process is already dead */
- msg_warn_pool_check ("stale lock file %s from pid: %P, removing",
- lock, pid);
- unlink (lock);
- close (fd);
-
- return TRUE;
- }
- }
-
- close (fd);
-
- if (nanosleep (&sleep_ts, NULL) == -1 && errno != EINTR) {
- msg_err_pool_check ("cannot sleep open lock file %s: %s", lock,
- strerror (errno));
-
- return FALSE;
- }
- }
-
- unlink (lock);
- close (fd);
-
- return TRUE;
- }
-
- #define RSPAMD_SQLITE_MMAP_LIMIT 268435456
- #define RSPAMD_SQLITE_CACHE_SIZE 262144
-
- sqlite3 *
- rspamd_sqlite3_open_or_create (rspamd_mempool_t *pool, const gchar *path, const
- gchar *create_sql, guint version, GError **err)
- {
- sqlite3 *sqlite;
- gint rc, flags, lock_fd;
- gchar lock_path[PATH_MAX], dbdir[PATH_MAX], *pdir;
- static const char sqlite_wal[] =
- "PRAGMA journal_mode=\"wal\";"
- "PRAGMA wal_autocheckpoint = 16;"
- "PRAGMA journal_size_limit = 1536;",
- exclusive_lock_sql[] = "PRAGMA locking_mode=\"exclusive\";",
-
- fsync_sql[] = "PRAGMA synchronous=\"NORMAL\";",
-
- foreign_keys[] = "PRAGMA foreign_keys=\"ON\";",
-
- #if defined(__LP64__) || defined(_LP64)
- enable_mmap[] = "PRAGMA mmap_size="
- G_STRINGIFY(RSPAMD_SQLITE_MMAP_LIMIT) ";",
- #endif
-
- other_pragmas[] = "PRAGMA read_uncommitted=\"ON\";"
- "PRAGMA cache_size="
- G_STRINGIFY(RSPAMD_SQLITE_CACHE_SIZE) ";",
- db_version[] = "PRAGMA user_version;";
- gboolean create = FALSE, has_lock = FALSE;
-
- flags = SQLITE_OPEN_READWRITE;
- #ifdef SQLITE_OPEN_SHAREDCACHE
- flags |= SQLITE_OPEN_SHAREDCACHE;
- #endif
- #ifdef SQLITE_OPEN_WAL
- flags |= SQLITE_OPEN_WAL;
- #endif
-
- rspamd_strlcpy (dbdir, path, sizeof (dbdir));
- pdir = dirname (dbdir);
-
- if (access (pdir, W_OK) == -1) {
- g_set_error (err, rspamd_sqlite3_quark (),
- errno, "cannot open sqlite directory %s: %s",
- pdir, strerror (errno));
-
- return NULL;
- }
-
- rspamd_snprintf (lock_path, sizeof (lock_path), "%s.lock", path);
-
- if (access (path, R_OK) == -1) {
- flags |= SQLITE_OPEN_CREATE;
- create = TRUE;
- }
-
-
- rspamd_snprintf (lock_path, sizeof (lock_path), "%s.lock", path);
- lock_fd = open (lock_path, O_WRONLY|O_CREAT|O_EXCL, 00600);
-
- if (lock_fd == -1) {
- if (errno == EEXIST || errno == EBUSY) {
- msg_debug_pool_check ("checking %s to wait for db being initialized", lock_path);
-
- if (!rspamd_sqlite3_wait(pool, lock_path)) {
- g_set_error(err, rspamd_sqlite3_quark(),
- errno, "cannot create sqlite file %s: %s",
- path, strerror(errno));
-
- return NULL;
- }
-
-
- /* At this point we have database created */
- create = FALSE;
- has_lock = FALSE;
- }
- else {
- g_set_error(err, rspamd_sqlite3_quark(),
- errno, "cannot lock sqlite file %s: %s",
- path, strerror(errno));
- }
- }
- else {
- pid_t myself = getpid ();
- msg_debug_pool_check ("locking %s to block other processes", lock_path);
- (void)write (lock_fd, &myself, sizeof (myself));
-
- g_assert (rspamd_file_lock (lock_fd, FALSE));
- has_lock = TRUE;
- }
-
- if ((rc = sqlite3_open_v2 (path, &sqlite,
- flags, NULL)) != SQLITE_OK) {
- #if SQLITE_VERSION_NUMBER >= 3008000
- g_set_error (err, rspamd_sqlite3_quark (),
- rc, "cannot open sqlite db %s: %s",
- path, sqlite3_errstr (rc));
- #else
- g_set_error (err, rspamd_sqlite3_quark (),
- rc, "cannot open sqlite db %s: %d",
- path, rc);
- #endif
-
- if (has_lock && lock_fd != -1) {
- msg_debug_pool_check ("removing lock from %s", lock_path);
- rspamd_file_unlock (lock_fd, FALSE);
- unlink (lock_path);
- close (lock_fd);
- }
-
- return NULL;
- }
-
- if (create && has_lock) {
- while ((rc = sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL)) != SQLITE_OK) {
- if (rc == SQLITE_BUSY) {
- struct timespec sleep_ts = {
- .tv_sec = 0,
- .tv_nsec = 1000000
- };
-
- nanosleep (&sleep_ts, NULL);
-
- continue;
- }
-
- msg_warn_pool_check ("WAL mode is not supported (%s), locking issues might occur",
- sqlite3_errmsg (sqlite));
- break;
- }
-
- if (sqlite3_exec (sqlite, exclusive_lock_sql, NULL, NULL, NULL) != SQLITE_OK) {
- msg_warn_pool_check ("cannot exclusively lock database to create schema: %s",
- sqlite3_errmsg (sqlite));
- }
-
- if (create_sql) {
- while ((rc = sqlite3_exec (sqlite, create_sql, NULL, NULL, NULL)) != SQLITE_OK) {
- if (rc == SQLITE_BUSY) {
- struct timespec sleep_ts = {
- .tv_sec = 0,
- .tv_nsec = 1000000
- };
-
- nanosleep (&sleep_ts, NULL);
-
- continue;
- }
-
- g_set_error (err, rspamd_sqlite3_quark (),
- -1, "cannot execute create sql `%s`: %s",
- create_sql, sqlite3_errmsg (sqlite));
- sqlite3_close (sqlite);
- rspamd_file_unlock (lock_fd, FALSE);
- unlink (lock_path);
- if (lock_fd != -1) {
- close (lock_fd);
- }
-
- return NULL;
- }
- }
-
- sqlite3_close (sqlite);
-
- /* Reopen in normal mode */
- msg_debug_pool_check ("reopening %s in normal mode", path);
- flags &= ~SQLITE_OPEN_CREATE;
-
- if ((rc = sqlite3_open_v2 (path, &sqlite,
- flags, NULL)) != SQLITE_OK) {
- #if SQLITE_VERSION_NUMBER >= 3008000
- g_set_error (err, rspamd_sqlite3_quark (),
- rc, "cannot open sqlite db after creation %s: %s",
- path, sqlite3_errstr (rc));
- #else
- g_set_error (err, rspamd_sqlite3_quark (),
- rc, "cannot open sqlite db after creation %s: %d",
- path, rc);
- #endif
- rspamd_file_unlock (lock_fd, FALSE);
- unlink (lock_path);
-
- if (lock_fd != -1) {
- close (lock_fd);
- }
-
- return NULL;
- }
- }
- else if (has_lock && version > 0) {
- /* Check user version */
- sqlite3_stmt *stmt = NULL;
- guint32 db_ver;
- GString *new_ver_sql;
-
- if (sqlite3_prepare (sqlite, db_version, -1, &stmt, NULL) != SQLITE_OK) {
- msg_warn_pool_check ("Cannot get user version pragma: %s",
- sqlite3_errmsg (sqlite));
- }
- else {
- if (sqlite3_step (stmt) != SQLITE_ROW) {
- msg_warn_pool_check ("Cannot get user version pragma, step failed: %s",
- sqlite3_errmsg (sqlite));
- sqlite3_finalize (stmt);
- }
- else {
- db_ver = sqlite3_column_int (stmt, 0);
- sqlite3_reset (stmt);
- sqlite3_finalize (stmt);
-
- if (version > db_ver) {
- msg_warn_pool_check ("Database version %ud is less than "
- "desired version %ud, run create script", db_ver,
- version);
-
- if (create_sql) {
- if (sqlite3_exec (sqlite, create_sql, NULL, NULL, NULL) != SQLITE_OK) {
- g_set_error (err, rspamd_sqlite3_quark (),
- -1, "cannot execute create sql `%s`: %s",
- create_sql, sqlite3_errmsg (sqlite));
- sqlite3_close (sqlite);
- rspamd_file_unlock (lock_fd, FALSE);
- unlink (lock_path);
- if (lock_fd != -1) {
- close (lock_fd);
- }
-
- return NULL;
- }
- }
-
- new_ver_sql = g_string_new ("PRAGMA user_version=");
- rspamd_printf_gstring (new_ver_sql, "%ud", version);
-
- if (sqlite3_exec (sqlite, new_ver_sql->str, NULL, NULL, NULL)
- != SQLITE_OK) {
- g_set_error (err, rspamd_sqlite3_quark (),
- -1, "cannot execute update version sql `%s`: %s",
- new_ver_sql->str, sqlite3_errmsg (sqlite));
- sqlite3_close (sqlite);
- rspamd_file_unlock (lock_fd, FALSE);
- unlink (lock_path);
- if (lock_fd != -1) {
- close (lock_fd);
- }
-
- g_string_free (new_ver_sql, TRUE);
-
- return NULL;
- }
-
- g_string_free (new_ver_sql, TRUE);
- }
- else if (db_ver > version) {
- msg_warn_pool_check ("Database version %ud is more than "
- "desired version %ud, this could cause"
- " unexpected behaviour", db_ver,
- version);
- }
- }
- }
- }
-
- while ((rc = sqlite3_exec (sqlite, sqlite_wal, NULL, NULL, NULL)) != SQLITE_OK) {
- if (rc == SQLITE_BUSY) {
- struct timespec sleep_ts = {
- .tv_sec = 0,
- .tv_nsec = 1000000
- };
-
- nanosleep (&sleep_ts, NULL);
-
- continue;
- }
-
- msg_warn_pool_check ("WAL mode is not supported (%s), locking issues might occur",
- sqlite3_errmsg (sqlite));
- break;
- }
-
- if (sqlite3_exec (sqlite, fsync_sql, NULL, NULL, NULL) != SQLITE_OK) {
- msg_warn_pool_check ("cannot set synchronous: %s",
- sqlite3_errmsg (sqlite));
- }
-
- if ((rc = sqlite3_exec (sqlite, foreign_keys, NULL, NULL, NULL)) !=
- SQLITE_OK) {
- msg_warn_pool_check ("cannot enable foreign keys: %s",
- sqlite3_errmsg (sqlite));
- }
-
- #if defined(__LP64__) || defined(_LP64)
- if ((rc = sqlite3_exec (sqlite, enable_mmap, NULL, NULL, NULL)) != SQLITE_OK) {
- msg_warn_pool_check ("cannot enable mmap: %s",
- sqlite3_errmsg (sqlite));
- }
- #endif
-
- if ((rc = sqlite3_exec (sqlite, other_pragmas, NULL, NULL, NULL)) !=
- SQLITE_OK) {
- msg_warn_pool_check ("cannot execute tuning pragmas: %s",
- sqlite3_errmsg (sqlite));
- }
-
- if (has_lock && lock_fd != -1) {
- msg_debug_pool_check ("removing lock from %s", lock_path);
- rspamd_file_unlock (lock_fd, FALSE);
- unlink (lock_path);
- close (lock_fd);
- }
-
- return sqlite;
- }
-
- gboolean
- rspamd_sqlite3_sync (sqlite3 *db, gint *wal_frames, gint *wal_checkpoints)
- {
- gint wf = 0, wc = 0, mode;
-
- #ifdef SQLITE_OPEN_WAL
- #ifdef SQLITE_CHECKPOINT_TRUNCATE
- mode = SQLITE_CHECKPOINT_TRUNCATE;
- #elif defined(SQLITE_CHECKPOINT_RESTART)
- mode = SQLITE_CHECKPOINT_RESTART;
- #elif defined(SQLITE_CHECKPOINT_FULL)
- mode = SQLITE_CHECKPOINT_FULL;
- #endif
- /* Perform wal checkpoint (might be long) */
- if (sqlite3_wal_checkpoint_v2 (db,
- NULL,
- mode,
- &wf,
- &wc) != SQLITE_OK) {
- return FALSE;
-
- }
- #endif
-
- if (wal_frames) {
- *wal_frames = wf;
- }
- if (wal_checkpoints) {
- *wal_checkpoints = wc;
- }
-
- return TRUE;
- }
|