[Feature] Add multiple base32 alphabets for decoding

9 files changed, 139 insertions, 52 deletions

diff --git a/src/controller.c b/src/controller.c
index 31c353e34..cfea10971 100644
--- a/src/controller.c
+++ b/src/controller.c
@@ -325,7 +325,7 @@ check_uncached:
 	if (salt != NULL && hash != NULL) {
 
 		/* decode salt */
-		salt_decoded = rspamd_decode_base32 (salt, salt_len, &salt_len);
+		salt_decoded = rspamd_decode_base32 (salt, salt_len, &salt_len, RSPAMD_BASE32_DEFAULT);
 
 		if (salt_decoded == NULL || salt_len != pbkdf->salt_len) {
 			/* We have some unknown salt here */
@@ -334,7 +334,7 @@ check_uncached:
 			return FALSE;
 		}
 
-		key_decoded = rspamd_decode_base32 (hash, key_len, &key_len);
+		key_decoded = rspamd_decode_base32 (hash, key_len, &key_len, RSPAMD_BASE32_DEFAULT);
 
 		if (key_decoded == NULL || key_len != pbkdf->key_len) {
 			/* We have some unknown salt here */
diff --git a/src/libcryptobox/keypair.c b/src/libcryptobox/keypair.c
index 49daba06c..24bd93c0b 100644
--- a/src/libcryptobox/keypair.c
+++ b/src/libcryptobox/keypair.c
@@ -336,7 +336,7 @@ rspamd_pubkey_from_base32 (const gchar *b32,
 		len = strlen (b32);
 	}
 
-	decoded = rspamd_decode_base32 (b32, len, &dlen);
+	decoded = rspamd_decode_base32 (b32, len, &dlen, RSPAMD_BASE32_DEFAULT);
 
 	if (decoded == NULL) {
 		return NULL;
@@ -736,7 +736,7 @@ rspamd_keypair_from_ucl (const ucl_object_t *obj)
 		dec_len = rspamd_decode_hex_buf (str, ucl_len, target, len);
 	}
 	else {
-		dec_len = rspamd_decode_base32_buf (str, ucl_len, target, len);
+		dec_len = rspamd_decode_base32_buf (str, ucl_len, target, len, RSPAMD_BASE32_DEFAULT);
 	}
 
 	if (dec_len != (gint)len) {
@@ -752,7 +752,7 @@ rspamd_keypair_from_ucl (const ucl_object_t *obj)
 		dec_len = rspamd_decode_hex_buf (str, ucl_len, target, len);
 	}
 	else {
-		dec_len = rspamd_decode_base32_buf (str, ucl_len, target, len);
+		dec_len = rspamd_decode_base32_buf (str, ucl_len, target, len, RSPAMD_BASE32_DEFAULT);
 	}
 
 	if (dec_len != (gint)len) {
diff --git a/src/libserver/http/http_connection.c b/src/libserver/http/http_connection.c
index 28a13f7ba..4e880f216 100644
--- a/src/libserver/http/http_connection.c
+++ b/src/libserver/http/http_connection.c
@@ -155,7 +155,7 @@ rspamd_http_parse_key (rspamd_ftok_t *data, struct rspamd_http_connection *conn,
 		eq_pos = memchr (data->begin, '=', data->len);
 		if (eq_pos != NULL) {
 			decoded_id = rspamd_decode_base32 (data->begin, eq_pos - data->begin,
-					&id_len);
+					&id_len, RSPAMD_BASE32_DEFAULT);
 
 			if (decoded_id != NULL && id_len >= RSPAMD_KEYPAIR_SHORT_ID_LEN) {
 				pk = rspamd_pubkey_from_base32 (eq_pos + 1,
diff --git a/src/libstat/backends/sqlite3_backend.c b/src/libstat/backends/sqlite3_backend.c
index 1ddfa6f17..6b1cca173 100644
--- a/src/libstat/backends/sqlite3_backend.c
+++ b/src/libstat/backends/sqlite3_backend.c
@@ -1053,7 +1053,7 @@ rspamd_sqlite3_load_tokenizer_config (gpointer runtime,
 	}
 	else {
 		/* Need to decode */
-		copied_conf = rspamd_decode_base32 (tk_conf, sz, len);
+		copied_conf = rspamd_decode_base32 (tk_conf, sz, len, RSPAMD_BASE32_DEFAULT);
 		g_free (tk_conf);
 		rspamd_mempool_add_destructor (rt->task->task_pool, g_free, copied_conf);
 	}
diff --git a/src/libstat/tokenizers/osb.c b/src/libstat/tokenizers/osb.c
index 0b53f8af9..a8007ec0f 100644
--- a/src/libstat/tokenizers/osb.c
+++ b/src/libstat/tokenizers/osb.c
@@ -114,7 +114,7 @@ rspamd_tokenizer_osb_config_from_ucl (rspamd_mempool_t * pool,
 
 			if (elt != NULL && ucl_object_type (elt) == UCL_STRING) {
 				key = rspamd_decode_base32 (ucl_object_tostring (elt),
-						0, &keylen);
+						0, &keylen, RSPAMD_BASE32_DEFAULT);
 				if (keylen < sizeof (rspamd_sipkey_t)) {
 					msg_warn ("siphash key is too short: %z", keylen);
 					g_free (key);
diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c
index aaa3a0084..3ea725738 100644
--- a/src/libutil/str_util.c
+++ b/src/libutil/str_util.c
@@ -642,54 +642,139 @@ rspamd_encode_base32 (const guchar *in, gsize inlen, enum rspamd_base32_type typ
 	return NULL;
 }
 
-static const guchar b32_dec[] = {
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0x12, 0xff, 0x19, 0x1a, 0x1b, 0x1e, 0x1d,
-	0x07, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0x18, 0x01, 0x0c, 0x03, 0x08, 0x05, 0x06,
-	0x1c, 0x15, 0x09, 0x0a, 0xff, 0x0b, 0x02, 0x10,
-	0x0d, 0x0e, 0x04, 0x16, 0x11, 0x13, 0xff, 0x14,
-	0x0f, 0x00, 0x17, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0x18, 0x01, 0x0c, 0x03, 0x08, 0x05, 0x06,
-	0x1c, 0x15, 0x09, 0x0a, 0xff, 0x0b, 0x02, 0x10,
-	0x0d, 0x0e, 0x04, 0x16, 0x11, 0x13, 0xff, 0x14,
-	0x0f, 0x00, 0x17, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+static const guchar b32_dec_zbase[] = {
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0x12, 0xff, 0x19, 0x1a, 0x1b, 0x1e, 0x1d,
+		0x07, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0x18, 0x01, 0x0c, 0x03, 0x08, 0x05, 0x06,
+		0x1c, 0x15, 0x09, 0x0a, 0xff, 0x0b, 0x02, 0x10,
+		0x0d, 0x0e, 0x04, 0x16, 0x11, 0x13, 0xff, 0x14,
+		0x0f, 0x00, 0x17, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
 };
+static const guchar b32_dec_bleach[] = {
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0x0f, 0xff, 0x0a, 0x11, 0x15, 0x14, 0x1a, 0x1e,
+		0x07, 0x05, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0x1d, 0xff, 0x18, 0x0d, 0x19, 0x09, 0x08,
+		0x17, 0xff, 0x12, 0x16, 0x1f, 0x1b, 0x13, 0xff,
+		0x01, 0x00, 0x03, 0x10, 0x0b, 0x1c, 0x0c, 0x0e,
+		0x06, 0x04, 0x02, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+static const guchar b32_dec_rfc[] = {
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
+		0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
+		0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
+		0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+};
+
 
 gint
-rspamd_decode_base32_buf (const gchar *in, gsize inlen,
-		guchar *out, gsize outlen)
+rspamd_decode_base32_buf (const gchar *in, gsize inlen, guchar *out, gsize outlen,
+		enum rspamd_base32_type type)
 {
 	guchar *o, *end, decoded;
 	guchar c;
 	guint acc = 0U;
 	guint processed_bits = 0;
 	gsize i;
+	const guchar *b32_dec;
 
 	end = out + outlen;
 	o = out;
 
+	switch (type) {
+	case RSPAMD_BASE32_DEFAULT:
+		b32_dec = b32_dec_zbase;
+		break;
+	case RSPAMD_BASE32_BLEACH:
+		b32_dec = b32_dec_bleach;
+		break;
+	case RSPAMD_BASE32_RFC:
+		b32_dec = b32_dec_rfc;
+		break;
+	default:
+		g_assert_not_reached ();
+		abort ();
+	}
+
 	for (i = 0; i < inlen; i ++) {
 		c = (guchar)in[i];
 
@@ -718,8 +803,9 @@ rspamd_decode_base32_buf (const gchar *in, gsize inlen,
 	return (o - out);
 }
 
-guchar*
-rspamd_decode_base32 (const gchar *in, gsize inlen, gsize *outlen)
+guchar *
+rspamd_decode_base32 (const gchar *in, gsize inlen, gsize *outlen,
+		enum rspamd_base32_type type)
 {
 	guchar *res;
 
@@ -728,7 +814,8 @@ rspamd_decode_base32 (const gchar *in, gsize inlen, gsize *outlen)
 
 	res = g_malloc (allocated_len);
 
-	olen = rspamd_decode_base32_buf (in, inlen, res, allocated_len - 1);
+	olen = rspamd_decode_base32_buf (in, inlen, res, allocated_len - 1,
+			type);
 
 	if (olen >= 0) {
 		res[olen] = '\0';
diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h
index 1d2a66841..3a794bcc0 100644
--- a/src/libutil/str_util.h
+++ b/src/libutil/str_util.h
@@ -180,7 +180,7 @@ gchar *rspamd_encode_base32 (const guchar *in, gsize inlen,
  * @param inlen input length
  * @return freshly allocated base32 decoded value or NULL if input is invalid
  */
-guchar *rspamd_decode_base32 (const gchar *in, gsize inlen, gsize *outlen);
+guchar *rspamd_decode_base32 (const gchar *in, gsize inlen, gsize *outlen, enum rspamd_base32_type type);
 
 /**
  * Encode string using base32 encoding
@@ -201,8 +201,8 @@ gint rspamd_encode_base32_buf (const guchar *in, gsize inlen, gchar *out,
  * @param outlen output buf len
  * @return decoded len if in is valid base32 and `outlen` is enough to encode `inlen`
  */
-gint rspamd_decode_base32_buf (const gchar *in, gsize inlen,
-							   guchar *out, gsize outlen);
+gint rspamd_decode_base32_buf (const gchar *in, gsize inlen, guchar *out,
+		gsize outlen, enum rspamd_base32_type type);
 
 /**
  * Encode string using hex encoding
diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c
index e18bc3efb..053cce864 100644
--- a/src/lua/lua_util.c
+++ b/src/lua/lua_util.c
@@ -1217,7 +1217,7 @@ lua_util_decode_base32 (lua_State *L)
 	if (s != NULL) {
 		t = lua_newuserdata (L, sizeof (*t));
 		rspamd_lua_setclass (L, "rspamd{text}", -1);
-		t->start = rspamd_decode_base32 (s, inlen, &outlen);
+		t->start = rspamd_decode_base32 (s, inlen, &outlen, RSPAMD_BASE32_DEFAULT);
 		t->len = outlen;
 		t->flags = RSPAMD_TEXT_FLAG_OWN;
 	}
diff --git a/src/rspamadm/pw.c b/src/rspamadm/pw.c
index 7fe9d4a33..c8e59cbc4 100644
--- a/src/rspamadm/pw.c
+++ b/src/rspamadm/pw.c
@@ -257,7 +257,7 @@ rspamadm_pw_check (void)
 	if (salt != NULL && hash != NULL) {
 
 		/* decode salt */
-		salt_decoded = rspamd_decode_base32 (salt, salt_len, &salt_len);
+		salt_decoded = rspamd_decode_base32 (salt, salt_len, &salt_len, RSPAMD_BASE32_DEFAULT);
 
 		if (salt_decoded == NULL || salt_len != pbkdf->salt_len) {
 			/* We have some unknown salt here */
@@ -266,7 +266,7 @@ rspamadm_pw_check (void)
 			exit (EXIT_FAILURE);
 		}
 
-		key_decoded = rspamd_decode_base32 (hash, key_len, &key_len);
+		key_decoded = rspamd_decode_base32 (hash, key_len, &key_len, RSPAMD_BASE32_DEFAULT);
 
 		if (key_decoded == NULL || key_len != pbkdf->key_len) {
 			/* We have some unknown salt here */