/*- * Copyright 2016 Vsevolod Stakhov * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "config.h" #include "contrib/libev/ev.h" #include "redis_pool.h" #include "cfg_file.h" #include "contrib/hiredis/hiredis.h" #include "contrib/hiredis/async.h" #include "contrib/hiredis/adapters/libev.h" #include "cryptobox.h" #include "logger.h" struct rspamd_redis_pool_elt; enum rspamd_redis_pool_connection_state { RSPAMD_REDIS_POOL_CONN_INACTIVE = 0, RSPAMD_REDIS_POOL_CONN_ACTIVE, RSPAMD_REDIS_POOL_CONN_FINALISING }; struct rspamd_redis_pool_connection { struct redisAsyncContext *ctx; struct rspamd_redis_pool_elt *elt; GList *entry; ev_timer timeout; enum rspamd_redis_pool_connection_state state; gchar tag[MEMPOOL_UID_LEN]; ref_entry_t ref; }; struct rspamd_redis_pool_elt { struct rspamd_redis_pool *pool; guint64 key; GQueue *active; GQueue *inactive; }; struct rspamd_redis_pool { struct ev_loop *event_loop; struct rspamd_config *cfg; GHashTable *elts_by_key; GHashTable *elts_by_ctx; gdouble timeout; guint max_conns; }; static const gdouble default_timeout = 10.0; static const guint default_max_conns = 100; #define msg_err_rpool(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \ "redis_pool", conn->tag, \ G_STRFUNC, \ __VA_ARGS__) #define msg_warn_rpool(...) rspamd_default_log_function (G_LOG_LEVEL_WARNING, \ "redis_pool", conn->tag, \ G_STRFUNC, \ __VA_ARGS__) #define msg_info_rpool(...) rspamd_default_log_function (G_LOG_LEVEL_INFO, \ "redis_pool", conn->tag, \ G_STRFUNC, \ __VA_ARGS__) #define msg_debug_rpool(...) rspamd_conditional_debug_fast (NULL, NULL, \ rspamd_redis_pool_log_id, "redis_pool", conn->tag, \ G_STRFUNC, \ __VA_ARGS__) INIT_LOG_MODULE(redis_pool) static inline guint64 rspamd_redis_pool_get_key (const gchar *db, const gchar *password, const char *ip, int port) { rspamd_cryptobox_fast_hash_state_t st; rspamd_cryptobox_fast_hash_init (&st, rspamd_hash_seed ()); if (db) { rspamd_cryptobox_fast_hash_update (&st, db, strlen (db)); } if (password) { rspamd_cryptobox_fast_hash_update (&st, password, strlen (password)); } rspamd_cryptobox_fast_hash_update (&st, ip, strlen (ip)); rspamd_cryptobox_fast_hash_update (&st, &port, sizeof (port)); return rspamd_cryptobox_fast_hash_final (&st); } static void rspamd_redis_pool_conn_dtor (struct rspamd_redis_pool_connection *conn) { if (conn->state == RSPAMD_REDIS_POOL_CONN_ACTIVE) { msg_debug_rpool ("active connection removed"); if (conn->ctx) { if (!(conn->ctx->c.flags & REDIS_FREEING)) { redisAsyncContext *ac = conn->ctx; conn->ctx = NULL; g_hash_table_remove (conn->elt->pool->elts_by_ctx, ac); ac->onDisconnect = NULL; redisAsyncFree (ac); } } if (conn->entry) { g_queue_unlink (conn->elt->active, conn->entry); } } else { msg_debug_rpool ("inactive connection removed"); ev_timer_stop (conn->elt->pool->event_loop, &conn->timeout); if (conn->ctx && !(conn->ctx->c.flags & REDIS_FREEING)) { redisAsyncContext *ac = conn->ctx; /* To prevent on_disconnect here */ conn->state = RSPAMD_REDIS_POOL_CONN_FINALISING; g_hash_table_remove (conn->elt->pool->elts_by_ctx, ac); conn->ctx = NULL; ac->onDisconnect = NULL; redisAsyncFree (ac); } if (conn->entry) { g_queue_unlink (conn->elt->inactive, conn->entry); } } if (conn->entry) { g_list_free (conn->entry); } g_free (conn); } static void rspamd_redis_pool_elt_dtor (gpointer p) { GList *cur; struct rspamd_redis_pool_elt *elt = p; struct rspamd_redis_pool_connection *c; for (cur = elt->active->head; cur != NULL; cur = g_list_next (cur)) { c = cur->data; c->entry = NULL; REF_RELEASE (c); } for (cur = elt->inactive->head; cur != NULL; cur = g_list_next (cur)) { c = cur->data; c->entry = NULL; REF_RELEASE (c); } g_queue_free (elt->active); g_queue_free (elt->inactive); g_free (elt); } static void rspamd_redis_on_quit (redisAsyncContext *c, gpointer r, gpointer priv) { struct rspamd_redis_pool_connection *conn = (struct rspamd_redis_pool_connection *)priv; msg_debug_rpool ("quit command reply for the connection %p, refcount: %d", conn->ctx, conn->ref.refcount); /* * The connection will be freed by hiredis itself as we are here merely after * quit command has succeeded and we have timer being set already. * The problem is that when this callback is called, our connection is likely * dead, so probably even on_disconnect callback has been already called... * * Hence, the connection might already be freed, so even (conn) pointer may be * inaccessible. * * TODO: Use refcounts to prevent this stuff to happen, the problem is how * to handle Redis timeout on `quit` command in fact... The good thing is that * it will not likely happen. */ } static void rspamd_redis_conn_timeout (EV_P_ ev_timer *w, int revents) { struct rspamd_redis_pool_connection *conn = (struct rspamd_redis_pool_connection *)w->data; g_assert (conn->state != RSPAMD_REDIS_POOL_CONN_ACTIVE); if (conn->state == RSPAMD_REDIS_POOL_CONN_INACTIVE) { msg_debug_rpool ("scheduled soft removal of connection %p, refcount: %d", conn->ctx, conn->ref.refcount); /* Prevent reusing */ if (conn->entry) { g_queue_unlink (conn->elt->inactive, conn->entry); conn->entry = NULL; } conn->state = RSPAMD_REDIS_POOL_CONN_FINALISING; ev_timer_again (EV_A_ w); redisAsyncCommand (conn->ctx, rspamd_redis_on_quit, conn, "QUIT"); } else { /* Finalising by timeout */ ev_timer_stop (EV_A_ w); msg_debug_rpool ("final removal of connection %p, refcount: %d", conn->ctx, conn->ref.refcount); REF_RELEASE (conn); } } static void rspamd_redis_pool_schedule_timeout (struct rspamd_redis_pool_connection *conn) { gdouble real_timeout; guint active_elts; active_elts = g_queue_get_length (conn->elt->active); if (active_elts > conn->elt->pool->max_conns) { real_timeout = conn->elt->pool->timeout / 2.0; real_timeout = rspamd_time_jitter (real_timeout, real_timeout / 4.0); } else { real_timeout = conn->elt->pool->timeout; real_timeout = rspamd_time_jitter (real_timeout, real_timeout / 2.0); } msg_debug_rpool ("scheduled connection %p cleanup in %.1f seconds", conn->ctx, real_timeout); conn->timeout.data = conn; ev_timer_init (&conn->timeout, rspamd_redis_conn_timeout, real_timeout, real_timeout / 2.0); ev_timer_start (conn->elt->pool->event_loop, &conn->timeout); } static void rspamd_redis_pool_on_disconnect (const struct redisAsyncContext *ac, int status) { struct rspamd_redis_pool_connection *conn = ac->data; /* * Here, we know that redis itself will free this connection * so, we need to do something very clever about it */ if (conn->state != RSPAMD_REDIS_POOL_CONN_ACTIVE) { /* Do nothing for active connections as it is already handled somewhere */ if (conn->ctx) { msg_debug_rpool ("inactive connection terminated: %s, refs: %d", conn->ctx->errstr, conn->ref.refcount); } REF_RELEASE (conn); } } static struct rspamd_redis_pool_connection * rspamd_redis_pool_new_connection (struct rspamd_redis_pool *pool, struct rspamd_redis_pool_elt *elt, const char *db, const char *password, const char *ip, gint port) { struct rspamd_redis_pool_connection *conn; struct redisAsyncContext *ctx; if (*ip == '/' || *ip == '.') { ctx = redisAsyncConnectUnix (ip); } else { ctx = redisAsyncConnect (ip, port); } if (ctx) { if (ctx->err != REDIS_OK) { msg_err ("cannot connect to redis %s (port %d): %s", ip, port, ctx->errstr); redisAsyncFree (ctx); return NULL; } else { conn = g_malloc0 (sizeof (*conn)); conn->entry = g_list_prepend (NULL, conn); conn->elt = elt; conn->state = RSPAMD_REDIS_POOL_CONN_ACTIVE; g_hash_table_insert (elt->pool->elts_by_ctx, ctx, conn); g_queue_push_head_link (elt->active, conn->entry); conn->ctx = ctx; ctx->data = conn; rspamd_random_hex (conn->tag, sizeof (conn->tag)); REF_INIT_RETAIN (conn, rspamd_redis_pool_conn_dtor); msg_debug_rpool ("created new connection to %s:%d: %p", ip, port, ctx); redisLibevAttach (pool->event_loop, ctx); redisAsyncSetDisconnectCallback (ctx, rspamd_redis_pool_on_disconnect); if (password) { redisAsyncCommand (ctx, NULL, NULL, "AUTH %s", password); } if (db) { redisAsyncCommand (ctx, NULL, NULL, "SELECT %s", db); } } return conn; } return NULL; } static struct rspamd_redis_pool_elt * rspamd_redis_pool_new_elt (struct rspamd_redis_pool *pool) { struct rspamd_redis_pool_elt *elt; elt = g_malloc0 (sizeof (*elt)); elt->active = g_queue_new (); elt->inactive = g_queue_new (); elt->pool = pool; return elt; } struct rspamd_redis_pool * rspamd_redis_pool_init (void) { struct rspamd_redis_pool *pool; pool = g_malloc0 (sizeof (*pool)); pool->elts_by_key = g_hash_table_new_full (g_int64_hash, g_int64_equal, NULL, rspamd_redis_pool_elt_dtor); pool->elts_by_ctx = g_hash_table_new (g_direct_hash, g_direct_equal); return pool; } void rspamd_redis_pool_config (struct rspamd_redis_pool *pool, struct rspamd_config *cfg, struct ev_loop *ev_base) { g_assert (pool != NULL); pool->event_loop = ev_base; pool->cfg = cfg; pool->timeout = default_timeout; pool->max_conns = default_max_conns; } struct redisAsyncContext* rspamd_redis_pool_connect (struct rspamd_redis_pool *pool, const gchar *db, const gchar *password, const char *ip, int port) { guint64 key; struct rspamd_redis_pool_elt *elt; GList *conn_entry; struct rspamd_redis_pool_connection *conn; g_assert (pool != NULL); g_assert (pool->event_loop != NULL); g_assert (ip != NULL); key = rspamd_redis_pool_get_key (db, password, ip, port); elt = g_hash_table_lookup (pool->elts_by_key, &key); if (elt) { if (g_queue_get_length (elt->inactive) > 0) { conn_entry = g_queue_pop_head_link (elt->inactive); conn = conn_entry->data; g_assert (conn->state != RSPAMD_REDIS_POOL_CONN_ACTIVE); if (conn->ctx->err == REDIS_OK) { /* Also check SO_ERROR */ gint err; socklen_t len = sizeof (gint); if (getsockopt (conn->ctx->c.fd, SOL_SOCKET, SO_ERROR, (void *) &err, &len) == -1) { err = errno; } if (err != 0) { g_list_free (conn->entry); conn->entry = NULL; REF_RELEASE (conn); conn = rspamd_redis_pool_new_connection (pool, elt, db, password, ip, port); } else { ev_timer_stop (elt->pool->event_loop, &conn->timeout); conn->state = RSPAMD_REDIS_POOL_CONN_ACTIVE; g_queue_push_tail_link (elt->active, conn_entry); msg_debug_rpool ("reused existing connection to %s:%d: %p", ip, port, conn->ctx); } } else { g_list_free (conn->entry); conn->entry = NULL; REF_RELEASE (conn); conn = rspamd_redis_pool_new_connection (pool, elt, db, password, ip, port); } } else { /* Need to create connection */ conn = rspamd_redis_pool_new_connection (pool, elt, db, password, ip, port); } } else { /* Need to create a pool */ elt = rspamd_redis_pool_new_elt (pool); elt->key = key; g_hash_table_insert (pool->elts_by_key, &elt->key, elt); conn = rspamd_redis_pool_new_connection (pool, elt, db, password, ip, port); } if (!conn) { return NULL; } REF_RETAIN (conn); return conn->ctx; } void rspamd_redis_pool_release_connection (struct rspamd_redis_pool *pool, struct redisAsyncContext *ctx, enum rspamd_redis_pool_release_type how) { struct rspamd_redis_pool_connection *conn; g_assert (pool != NULL); g_assert (ctx != NULL); conn = g_hash_table_lookup (pool->elts_by_ctx, ctx); if (conn != NULL) { g_assert (conn->state == RSPAMD_REDIS_POOL_CONN_ACTIVE); if (ctx->err != REDIS_OK) { /* We need to terminate connection forcefully */ msg_debug_rpool ("closed connection %p due to an error", conn->ctx); REF_RELEASE (conn); } else { if (how == RSPAMD_REDIS_RELEASE_DEFAULT) { /* Ensure that there are no callbacks attached to this conn */ if (ctx->replies.head == NULL) { /* Just move it to the inactive queue */ g_queue_unlink (conn->elt->active, conn->entry); g_queue_push_head_link (conn->elt->inactive, conn->entry); conn->state = RSPAMD_REDIS_POOL_CONN_INACTIVE; rspamd_redis_pool_schedule_timeout (conn); msg_debug_rpool ("mark connection %p inactive", conn->ctx); } else { msg_debug_rpool ("closed connection %p due to callbacks left", conn->ctx); REF_RELEASE (conn); } } else { if (how == RSPAMD_REDIS_RELEASE_FATAL) { msg_debug_rpool ("closed connection %p due to an fatal termination", conn->ctx); } else { msg_debug_rpool ("closed connection %p due to explicit termination", conn->ctx); } REF_RELEASE (conn); } } REF_RELEASE (conn); } else { g_assert_not_reached (); } } void rspamd_redis_pool_destroy (struct rspamd_redis_pool *pool) { struct rspamd_redis_pool_elt *elt; GHashTableIter it; gpointer k, v; g_assert (pool != NULL); g_hash_table_iter_init (&it, pool->elts_by_key); while (g_hash_table_iter_next (&it, &k, &v)) { elt = v; rspamd_redis_pool_elt_dtor (elt); g_hash_table_iter_steal (&it); } g_hash_table_unref (pool->elts_by_ctx); g_hash_table_unref (pool->elts_by_key); g_free (pool); } const gchar* rspamd_redis_type_to_string (int type) { const gchar *ret = "unknown"; switch (type) { case REDIS_REPLY_STRING: ret = "string"; break; case REDIS_REPLY_ARRAY: ret = "array"; break; case REDIS_REPLY_INTEGER: ret = "int"; break; case REDIS_REPLY_STATUS: ret = "status"; break; case REDIS_REPLY_NIL: ret = "nil"; break; case REDIS_REPLY_ERROR: ret = "error"; break; default: break; } return ret; }