From 1cb842be45ed0b12c4f4a228532604175408b591 Mon Sep 17 00:00:00 2001 From: resec Date: Tue, 19 Nov 2019 17:09:05 +0800 Subject: eliminate redis smembers --- src/libstat/backends/redis_backend.c | 80 +++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 29 deletions(-) (limited to 'src/libstat') diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c index 08edf1a4f..26f33b0a1 100644 --- a/src/libstat/backends/redis_backend.c +++ b/src/libstat/backends/redis_backend.c @@ -98,6 +98,7 @@ struct rspamd_redis_stat_cbdata { redisAsyncContext *redis; ucl_object_t *cur; GPtrArray *cur_keys; + guint struct upstream *selected; guint inflight; gboolean wanna_die; @@ -846,7 +847,7 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) { struct rspamd_redis_stat_elt *redis_elt = (struct rspamd_redis_stat_elt *)priv; struct rspamd_redis_stat_cbdata *cbdata; - redisReply *reply = r, *elt; + redisReply *reply = r, *more, **elts, *elt; gchar **pk, *k; guint i, processed = 0; @@ -860,10 +861,13 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) if (c->err == 0 && r != NULL) { if (reply->type == REDIS_REPLY_ARRAY) { - g_ptr_array_set_size (cbdata->cur_keys, reply->elements); + more = r.element[0] + elts = r.element[1] + + g_ptr_array_set_size (cbdata->cur_keys, elts->elements); - for (i = 0; i < reply->elements; i ++) { - elt = reply->element[i]; + for (i = 0; i < elts->elements; i ++) { + elt = elts->element[i]; if (elt->type == REDIS_REPLY_STRING) { pk = (gchar **)&g_ptr_array_index (cbdata->cur_keys, i); @@ -912,30 +916,48 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) } } - /* Set up the required keys */ - ucl_object_insert_key (cbdata->cur, - ucl_object_typed_new (UCL_INT), "revision", 0, false); - ucl_object_insert_key (cbdata->cur, - ucl_object_typed_new (UCL_INT), "used", 0, false); - ucl_object_insert_key (cbdata->cur, - ucl_object_typed_new (UCL_INT), "total", 0, false); - ucl_object_insert_key (cbdata->cur, - ucl_object_typed_new (UCL_INT), "size", 0, false); - ucl_object_insert_key (cbdata->cur, - ucl_object_fromstring (cbdata->elt->ctx->stcf->symbol), - "symbol", 0, false); - ucl_object_insert_key (cbdata->cur, ucl_object_fromstring ("redis"), - "type", 0, false); - ucl_object_insert_key (cbdata->cur, ucl_object_fromint (0), - "languages", 0, false); - ucl_object_insert_key (cbdata->cur, ucl_object_fromint (processed), - "users", 0, false); - - rspamd_upstream_ok (cbdata->selected); - - if (cbdata->inflight == 0) { - rspamd_redis_async_cbdata_cleanup (cbdata); - redis_elt->cbdata = NULL; + if (more != NULL && more.integer) { + /* Cleanup the cbdata->cur_keys and re-allowcate */ + for (i = 0; i < cbdata->cur_keys->len; i ++) { + k = g_ptr_array_index (cbdata->cur_keys, i); + g_free (k); + } + + g_ptr_array_free (cbdata->cur_keys, TRUE); + + cbdata->cur_keys = g_ptr_array_new (); + + /* Get more keys */ + redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, redis_elt, + "SSCAN %s_keys %d COUNT 1000", + ctx->stcf->symbol, more.integer); + } + else { + /* Set up the required keys */ + ucl_object_insert_key (cbdata->cur, + ucl_object_typed_new (UCL_INT), "revision", 0, false); + ucl_object_insert_key (cbdata->cur, + ucl_object_typed_new (UCL_INT), "used", 0, false); + ucl_object_insert_key (cbdata->cur, + ucl_object_typed_new (UCL_INT), "total", 0, false); + ucl_object_insert_key (cbdata->cur, + ucl_object_typed_new (UCL_INT), "size", 0, false); + ucl_object_insert_key (cbdata->cur, + ucl_object_fromstring (cbdata->elt->ctx->stcf->symbol), + "symbol", 0, false); + ucl_object_insert_key (cbdata->cur, ucl_object_fromstring ("redis"), + "type", 0, false); + ucl_object_insert_key (cbdata->cur, ucl_object_fromint (0), + "languages", 0, false); + ucl_object_insert_key (cbdata->cur, ucl_object_fromint (processed), + "users", 0, false); + + rspamd_upstream_ok (cbdata->selected); + + if (cbdata->inflight == 0) { + rspamd_redis_async_cbdata_cleanup (cbdata); + redis_elt->cbdata = NULL; + } } } else { @@ -1029,7 +1051,7 @@ rspamd_redis_async_stat_cb (struct rspamd_stat_async_elt *elt, gpointer d) /* Get keys in redis that match our symbol */ rspamd_redis_maybe_auth (ctx, cbdata->redis); redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, redis_elt, - "SMEMBERS %s_keys", + "SSCAN %s_keys 0 COUNT 1000", ctx->stcf->symbol); } -- cgit v1.2.3 From d98f4ef54a9fa3d8f4cf5e224f8427063b264444 Mon Sep 17 00:00:00 2001 From: resec Date: Tue, 19 Nov 2019 09:31:39 +0000 Subject: eliminate smembers: fix --- src/libstat/backends/redis_backend.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'src/libstat') diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c index 26f33b0a1..eab238532 100644 --- a/src/libstat/backends/redis_backend.c +++ b/src/libstat/backends/redis_backend.c @@ -98,7 +98,6 @@ struct rspamd_redis_stat_cbdata { redisAsyncContext *redis; ucl_object_t *cur; GPtrArray *cur_keys; - guint struct upstream *selected; guint inflight; gboolean wanna_die; @@ -847,7 +846,7 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) { struct rspamd_redis_stat_elt *redis_elt = (struct rspamd_redis_stat_elt *)priv; struct rspamd_redis_stat_cbdata *cbdata; - redisReply *reply = r, *more, **elts, *elt; + redisReply *reply = r, *more, *elts, *elt; gchar **pk, *k; guint i, processed = 0; @@ -861,8 +860,8 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) if (c->err == 0 && r != NULL) { if (reply->type == REDIS_REPLY_ARRAY) { - more = r.element[0] - elts = r.element[1] + more = reply->element[0]; + elts = reply->element[1]; g_ptr_array_set_size (cbdata->cur_keys, elts->elements); @@ -916,7 +915,7 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) } } - if (more != NULL && more.integer) { + if (more != NULL && more->integer) { /* Cleanup the cbdata->cur_keys and re-allowcate */ for (i = 0; i < cbdata->cur_keys->len; i ++) { k = g_ptr_array_index (cbdata->cur_keys, i); @@ -930,7 +929,7 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) /* Get more keys */ redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, redis_elt, "SSCAN %s_keys %d COUNT 1000", - ctx->stcf->symbol, more.integer); + cbdata->elt->ctx->stcf->symbol, more->integer); } else { /* Set up the required keys */ -- cgit v1.2.3 From 811f3d7c675d12860231d269bc728047f9ba80ff Mon Sep 17 00:00:00 2001 From: resec Date: Tue, 19 Nov 2019 11:30:21 +0000 Subject: eliminate smembers: fix again --- src/libstat/backends/redis_backend.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'src/libstat') diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c index eab238532..d4ac6a30e 100644 --- a/src/libstat/backends/redis_backend.c +++ b/src/libstat/backends/redis_backend.c @@ -846,9 +846,10 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) { struct rspamd_redis_stat_elt *redis_elt = (struct rspamd_redis_stat_elt *)priv; struct rspamd_redis_stat_cbdata *cbdata; - redisReply *reply = r, *more, *elts, *elt; + redisReply *reply = r, *more_elt, *elts, *elt; gchar **pk, *k; guint i, processed = 0; + gboolean more = false; cbdata = redis_elt->cbdata; @@ -860,9 +861,13 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) if (c->err == 0 && r != NULL) { if (reply->type == REDIS_REPLY_ARRAY) { - more = reply->element[0]; + more_elt = reply->element[0]; elts = reply->element[1]; + if (more_elt != NULL && more_elt->str != NULL && strcmp (more_elt->str, "0") != 0) { + more = true; + } + g_ptr_array_set_size (cbdata->cur_keys, elts->elements); for (i = 0; i < elts->elements; i ++) { @@ -915,7 +920,7 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) } } - if (more != NULL && more->integer) { + if (more) { /* Cleanup the cbdata->cur_keys and re-allowcate */ for (i = 0; i < cbdata->cur_keys->len; i ++) { k = g_ptr_array_index (cbdata->cur_keys, i); @@ -928,8 +933,10 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) /* Get more keys */ redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, redis_elt, - "SSCAN %s_keys %d COUNT 1000", - cbdata->elt->ctx->stcf->symbol, more->integer); + "SSCAN %s_keys %s COUNT 1000", + cbdata->elt->ctx->stcf->symbol, more_elt->str); + + cbdata->inflight += 1; } else { /* Set up the required keys */ -- cgit v1.2.3 From 004eb44a8652214635ffeb8b08e56b4347a5aa60 Mon Sep 17 00:00:00 2001 From: resec Date: Tue, 19 Nov 2019 22:18:41 +0800 Subject: eliminate smembers: remove redundent cleanup --- src/libstat/backends/redis_backend.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) (limited to 'src/libstat') diff --git a/src/libstat/backends/redis_backend.c b/src/libstat/backends/redis_backend.c index d4ac6a30e..ec65a133f 100644 --- a/src/libstat/backends/redis_backend.c +++ b/src/libstat/backends/redis_backend.c @@ -921,17 +921,7 @@ rspamd_redis_stat_keys (redisAsyncContext *c, gpointer r, gpointer priv) } if (more) { - /* Cleanup the cbdata->cur_keys and re-allowcate */ - for (i = 0; i < cbdata->cur_keys->len; i ++) { - k = g_ptr_array_index (cbdata->cur_keys, i); - g_free (k); - } - - g_ptr_array_free (cbdata->cur_keys, TRUE); - - cbdata->cur_keys = g_ptr_array_new (); - - /* Get more keys */ + /* Get more stat keys */ redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, redis_elt, "SSCAN %s_keys %s COUNT 1000", cbdata->elt->ctx->stcf->symbol, more_elt->str); -- cgit v1.2.3