1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996 |
- /*-
- * Copyright 2018 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- #include "map_helpers.h"
- #include "map_private.h"
- #include "khash.h"
- #include "radix.h"
- #include "rspamd.h"
- #include "cryptobox.h"
- #include "mempool_vars_internal.h"
- #include "contrib/fastutf8/fastutf8.h"
- #include "contrib/cdb/cdb.h"
-
- #ifdef WITH_HYPERSCAN
- #include "hs.h"
- #endif
- #ifndef WITH_PCRE2
- #include <pcre.h>
- #else
- #include <pcre2.h>
- #endif
-
-
- static const guint64 map_hash_seed = 0xdeadbabeULL;
- static const gchar * const hash_fill = "1";
-
- struct rspamd_map_helper_value {
- gsize hits;
- gconstpointer key;
- gchar value[]; /* Null terminated */
- };
-
- #define rspamd_map_ftok_hash(t) (rspamd_icase_hash((t).begin, (t).len, rspamd_hash_seed ()))
- #define rspamd_map_ftok_equal(a, b) ((a).len == (b).len && rspamd_lc_cmp((a).begin, (b).begin, (a).len) == 0)
-
- KHASH_INIT (rspamd_map_hash, rspamd_ftok_t,
- struct rspamd_map_helper_value *, true,
- rspamd_map_ftok_hash, rspamd_map_ftok_equal);
-
- struct rspamd_radix_map_helper {
- rspamd_mempool_t *pool;
- khash_t(rspamd_map_hash) *htb;
- radix_compressed_t *trie;
- struct rspamd_map *map;
- rspamd_cryptobox_fast_hash_state_t hst;
- };
-
- struct rspamd_hash_map_helper {
- rspamd_mempool_t *pool;
- khash_t(rspamd_map_hash) *htb;
- struct rspamd_map *map;
- rspamd_cryptobox_fast_hash_state_t hst;
- };
-
- struct rspamd_cdb_map_helper {
- GQueue cdbs;
- struct rspamd_map *map;
- rspamd_cryptobox_fast_hash_state_t hst;
- gsize total_size;
- };
-
- struct rspamd_regexp_map_helper {
- rspamd_cryptobox_hash_state_t hst;
- guchar re_digest[rspamd_cryptobox_HASHBYTES];
- rspamd_mempool_t *pool;
- struct rspamd_map *map;
- GPtrArray *regexps;
- GPtrArray *values;
- khash_t(rspamd_map_hash) *htb;
- enum rspamd_regexp_map_flags map_flags;
- #ifdef WITH_HYPERSCAN
- hs_database_t *hs_db;
- hs_scratch_t *hs_scratch;
- gchar **patterns;
- gint *flags;
- gint *ids;
- #endif
- };
-
- /**
- * FSM for parsing lists
- */
-
- #define MAP_STORE_KEY do { \
- while (g_ascii_isspace (*c) && p > c) { c ++; } \
- key = g_malloc (p - c + 1); \
- rspamd_strlcpy (key, c, p - c + 1); \
- stripped_key = g_strstrip (key); \
- } while (0)
-
- #define MAP_STORE_VALUE do { \
- while (g_ascii_isspace (*c) && p > c) { c ++; } \
- value = g_malloc (p - c + 1); \
- rspamd_strlcpy (value, c, p - c + 1); \
- stripped_value = g_strstrip (value); \
- } while (0)
-
- gchar *
- rspamd_parse_kv_list (
- gchar * chunk,
- gint len,
- struct map_cb_data *data,
- rspamd_map_insert_func func,
- const gchar *default_value,
- gboolean final)
- {
- enum {
- map_skip_spaces_before_key = 0,
- map_read_key,
- map_read_key_quoted,
- map_read_key_slashed,
- map_skip_spaces_after_key,
- map_backslash_quoted,
- map_backslash_slashed,
- map_read_key_after_slash,
- map_read_value,
- map_read_comment_start,
- map_skip_comment,
- map_read_eol,
- };
-
- gchar *c, *p, *key = NULL, *value = NULL, *stripped_key, *stripped_value, *end;
- struct rspamd_map *map = data->map;
- guint line_number = 0;
-
- p = chunk;
- c = p;
- end = p + len;
-
- while (p < end) {
- switch (data->state) {
- case map_skip_spaces_before_key:
- if (g_ascii_isspace (*p)) {
- p ++;
- }
- else {
- if (*p == '"') {
- p++;
- c = p;
- data->state = map_read_key_quoted;
- }
- else if (*p == '/') {
- /* Note that c is on '/' here as '/' is a part of key */
- c = p;
- p++;
- data->state = map_read_key_slashed;
- }
- else {
- c = p;
- data->state = map_read_key;
- }
- }
- break;
- case map_read_key:
- /* read key */
- /* Check here comments, eol and end of buffer */
- if (*p == '#' && (p == c || *(p - 1) != '\\')) {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func (data->cur_data, stripped_key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s; line: %d",
- stripped_key, default_value, line_number);
- g_free (key);
- }
-
- key = NULL;
- data->state = map_read_comment_start;
- }
- else if (*p == '\r' || *p == '\n') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func (data->cur_data, stripped_key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s; line: %d",
- stripped_key, default_value, line_number);
- g_free (key);
- }
-
- data->state = map_read_eol;
- key = NULL;
- }
- else if (g_ascii_isspace (*p)) {
- if (p - c > 0) {
- MAP_STORE_KEY;
- data->state = map_skip_spaces_after_key;
- }
- else {
- msg_err_map ("empty or invalid key found on line %d", line_number);
- data->state = map_skip_comment;
- }
- }
- else {
- p++;
- }
- break;
- case map_read_key_quoted:
- if (*p == '\\') {
- data->state = map_backslash_quoted;
- p ++;
- }
- else if (*p == '"') {
- /* Allow empty keys in this case */
- if (p - c >= 0) {
- MAP_STORE_KEY;
- data->state = map_skip_spaces_after_key;
- }
- else {
- g_assert_not_reached ();
- }
- p ++;
- }
- else {
- p ++;
- }
- break;
- case map_read_key_slashed:
- if (*p == '\\') {
- data->state = map_backslash_slashed;
- p ++;
- }
- else if (*p == '/') {
- /* Allow empty keys in this case */
- if (p - c >= 0) {
- data->state = map_read_key_after_slash;
- }
- else {
- g_assert_not_reached ();
- }
- }
- else {
- p ++;
- }
- break;
- case map_read_key_after_slash:
- /*
- * This state is equal to reading of key but '/' is not
- * treated specially
- */
- if (*p == '#') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func (data->cur_data, stripped_key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s; line: %d",
- stripped_key, default_value, line_number);
- g_free (key);
- key = NULL;
- }
-
- data->state = map_read_comment_start;
- }
- else if (*p == '\r' || *p == '\n') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func (data->cur_data, stripped_key, default_value);
-
- msg_debug_map ("insert key only pair: %s -> %s; line: %d",
- stripped_key, default_value, line_number);
- g_free (key);
- key = NULL;
- }
-
- data->state = map_read_eol;
- key = NULL;
- }
- else if (g_ascii_isspace (*p)) {
- if (p - c > 0) {
- MAP_STORE_KEY;
- data->state = map_skip_spaces_after_key;
- }
- else {
- msg_err_map ("empty or invalid key found on line %d", line_number);
- data->state = map_skip_comment;
- }
- }
- else {
- p ++;
- }
- break;
- case map_backslash_quoted:
- p ++;
- data->state = map_read_key_quoted;
- break;
- case map_backslash_slashed:
- p ++;
- data->state = map_read_key_slashed;
- break;
- case map_skip_spaces_after_key:
- if (*p == ' ' || *p == '\t') {
- p ++;
- }
- else {
- c = p;
- data->state = map_read_value;
- }
- break;
- case map_read_value:
- if (key == NULL) {
- /* Ignore line */
- msg_err_map ("empty or invalid key found on line %d", line_number);
- data->state = map_skip_comment;
- }
- else {
- if (*p == '#') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_VALUE;
- func (data->cur_data, stripped_key, stripped_value);
- msg_debug_map ("insert key value pair: %s -> %s; line: %d",
- stripped_key, stripped_value, line_number);
- g_free (key);
- g_free (value);
- key = NULL;
- value = NULL;
- } else {
- func (data->cur_data, stripped_key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s; line: %d",
- stripped_key, default_value, line_number);
- g_free (key);
- key = NULL;
- }
-
- data->state = map_read_comment_start;
- } else if (*p == '\r' || *p == '\n') {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_VALUE;
- func (data->cur_data, stripped_key, stripped_value);
- msg_debug_map ("insert key value pair: %s -> %s",
- stripped_key, stripped_value);
- g_free (key);
- g_free (value);
- key = NULL;
- value = NULL;
- } else {
- func (data->cur_data, stripped_key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s",
- stripped_key, default_value);
- g_free (key);
- key = NULL;
- }
-
- data->state = map_read_eol;
- key = NULL;
- }
- else {
- p++;
- }
- }
- break;
- case map_read_comment_start:
- if (*p == '#') {
- data->state = map_skip_comment;
- p ++;
- key = NULL;
- value = NULL;
- }
- else {
- g_assert_not_reached ();
- }
- break;
- case map_skip_comment:
- if (*p == '\r' || *p == '\n') {
- data->state = map_read_eol;
- }
- else {
- p ++;
- }
- break;
- case map_read_eol:
- /* Skip \r\n and whitespaces */
- if (*p == '\r' || *p == '\n') {
- if (*p == '\n') {
- /* We don't care about \r only line separators, they are too rare */
- line_number ++;
- }
- p++;
- }
- else {
- data->state = map_skip_spaces_before_key;
- }
- break;
- default:
- g_assert_not_reached ();
- break;
- }
- }
-
- if (final) {
- /* Examine the state */
- switch (data->state) {
- case map_read_key:
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_KEY;
- func (data->cur_data, stripped_key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s",
- stripped_key, default_value);
- g_free (key);
- key = NULL;
- }
- break;
- case map_read_value:
- if (key == NULL) {
- /* Ignore line */
- msg_err_map ("empty or invalid key found on line %d", line_number);
- data->state = map_skip_comment;
- }
- else {
- if (p - c > 0) {
- /* Store a single key */
- MAP_STORE_VALUE;
- func (data->cur_data, stripped_key, stripped_value);
- msg_debug_map ("insert key value pair: %s -> %s",
- stripped_key, stripped_value);
- g_free (key);
- g_free (value);
- key = NULL;
- value = NULL;
- } else {
- func (data->cur_data, stripped_key, default_value);
- msg_debug_map ("insert key only pair: %s -> %s",
- stripped_key, default_value);
- g_free (key);
- key = NULL;
- }
- }
- break;
- }
-
- data->state = map_skip_spaces_before_key;
- }
-
- return c;
- }
-
- /**
- * Radix tree helper function
- */
- void
- rspamd_map_helper_insert_radix (gpointer st, gconstpointer key, gconstpointer value)
- {
- struct rspamd_radix_map_helper *r = (struct rspamd_radix_map_helper *)st;
- struct rspamd_map_helper_value *val;
- gsize vlen;
- khiter_t k;
- gconstpointer nk;
- rspamd_ftok_t tok;
- gint res;
- struct rspamd_map *map;
-
- map = r->map;
- tok.begin = key;
- tok.len = strlen (key);
-
- k = kh_get (rspamd_map_hash, r->htb, tok);
-
- if (k == kh_end (r->htb)) {
- nk = rspamd_mempool_strdup (r->pool, key);
- tok.begin = nk;
- k = kh_put (rspamd_map_hash, r->htb, tok, &res);
- }
- else {
- val = kh_value (r->htb, k);
-
- if (strcmp (value, val->value) == 0) {
- /* Same element, skip */
- return;
- }
- else {
- msg_warn_map ("duplicate radix entry found for map %s: %s (old value: '%s', new: '%s')",
- map->name, key, val->value, value);
- }
-
- nk = kh_key (r->htb, k).begin;
- val->key = nk;
- kh_value (r->htb, k) = val;
-
- return; /* do not touch radix in case of exact duplicate */
- }
-
- vlen = strlen (value);
- val = rspamd_mempool_alloc0 (r->pool, sizeof (*val) +
- vlen + 1);
- memcpy (val->value, value, vlen);
-
- nk = kh_key (r->htb, k).begin;
- val->key = nk;
- kh_value (r->htb, k) = val;
- rspamd_radix_add_iplist (key, ",", r->trie, val, FALSE,
- r->map->name);
- rspamd_cryptobox_fast_hash_update (&r->hst, nk, tok.len);
- }
-
- void
- rspamd_map_helper_insert_radix_resolve (gpointer st, gconstpointer key, gconstpointer value)
- {
- struct rspamd_radix_map_helper *r = (struct rspamd_radix_map_helper *)st;
- struct rspamd_map_helper_value *val;
- gsize vlen;
- khiter_t k;
- gconstpointer nk;
- rspamd_ftok_t tok;
- gint res;
- struct rspamd_map *map;
-
- map = r->map;
- tok.begin = key;
- tok.len = strlen (key);
-
- k = kh_get (rspamd_map_hash, r->htb, tok);
-
- if (k == kh_end (r->htb)) {
- nk = rspamd_mempool_strdup (r->pool, key);
- tok.begin = nk;
- k = kh_put (rspamd_map_hash, r->htb, tok, &res);
- }
- else {
- val = kh_value (r->htb, k);
-
- if (strcmp (value, val->value) == 0) {
- /* Same element, skip */
- return;
- }
- else {
- msg_warn_map ("duplicate radix entry found for map %s: %s (old value: '%s', new: '%s')",
- map->name, key, val->value, value);
- }
-
- nk = kh_key (r->htb, k).begin;
- val->key = nk;
- kh_value (r->htb, k) = val;
-
- return; /* do not touch radix in case of exact duplicate */
- }
-
- vlen = strlen (value);
- val = rspamd_mempool_alloc0 (r->pool, sizeof (*val) +
- vlen + 1);
- memcpy (val->value, value, vlen);
- nk = kh_key (r->htb, k).begin;
- val->key = nk;
- kh_value (r->htb, k) = val;
- rspamd_radix_add_iplist (key, ",", r->trie, val, TRUE,
- r->map->name);
- rspamd_cryptobox_fast_hash_update (&r->hst, nk, tok.len);
- }
-
- void
- rspamd_map_helper_insert_hash (gpointer st, gconstpointer key, gconstpointer value)
- {
- struct rspamd_hash_map_helper *ht = st;
- struct rspamd_map_helper_value *val;
- khiter_t k;
- gconstpointer nk;
- gsize vlen;
- gint r;
- rspamd_ftok_t tok;
- struct rspamd_map *map;
-
- tok.begin = key;
- tok.len = strlen (key);
- map = ht->map;
-
- k = kh_get (rspamd_map_hash, ht->htb, tok);
-
- if (k == kh_end (ht->htb)) {
- nk = rspamd_mempool_strdup (ht->pool, key);
- tok.begin = nk;
- k = kh_put (rspamd_map_hash, ht->htb, tok, &r);
- }
- else {
- val = kh_value (ht->htb, k);
-
- if (strcmp (value, val->value) == 0) {
- /* Same element, skip */
- return;
- }
- else {
- msg_warn_map ("duplicate hash entry found for map %s: %s (old value: '%s', new: '%s')",
- map->name, key, val->value, value);
- }
- }
-
- /* Null termination due to alloc0 */
- vlen = strlen (value);
- val = rspamd_mempool_alloc0 (ht->pool, sizeof (*val) + vlen + 1);
- memcpy (val->value, value, vlen);
-
- tok = kh_key (ht->htb, k);
- nk = tok.begin;
- val->key = nk;
- kh_value (ht->htb, k) = val;
-
- rspamd_cryptobox_fast_hash_update (&ht->hst, nk, tok.len);
- }
-
- void
- rspamd_map_helper_insert_re (gpointer st, gconstpointer key, gconstpointer value)
- {
- struct rspamd_regexp_map_helper *re_map = st;
- struct rspamd_map *map;
- rspamd_regexp_t *re;
- gchar *escaped;
- GError *err = NULL;
- gint pcre_flags;
- gsize escaped_len;
- struct rspamd_map_helper_value *val;
- khiter_t k;
- rspamd_ftok_t tok;
- gconstpointer nk;
- gsize vlen;
- gint r;
-
- map = re_map->map;
-
- tok.begin = key;
- tok.len = strlen (key);
-
- k = kh_get (rspamd_map_hash, re_map->htb, tok);
-
- if (k == kh_end (re_map->htb)) {
- nk = rspamd_mempool_strdup (re_map->pool, key);
- tok.begin = nk;
- k = kh_put (rspamd_map_hash, re_map->htb, tok, &r);
- }
- else {
- val = kh_value (re_map->htb, k);
-
- /* Always warn about regexp duplicate as it's likely a bad mistake */
- msg_warn_map ("duplicate re entry found for map %s: %s (old value: '%s', new: '%s')",
- map->name, key, val->value, value);
-
- if (strcmp (val->value, value) == 0) {
- /* Same value, skip */
- return;
- }
-
- /* Replace value but do not touch regexp */
- nk = kh_key (re_map->htb, k).begin;
- val->key = nk;
- kh_value (re_map->htb, k) = val;
-
- return;
- }
-
- /* Check regexp stuff */
- if (re_map->map_flags & RSPAMD_REGEXP_MAP_FLAG_GLOB) {
- escaped = rspamd_str_regexp_escape (key, strlen (key), &escaped_len,
- RSPAMD_REGEXP_ESCAPE_GLOB|RSPAMD_REGEXP_ESCAPE_UTF);
- re = rspamd_regexp_new (escaped, NULL, &err);
- g_free (escaped);
- }
- else {
- re = rspamd_regexp_new (key, NULL, &err);
- }
-
- if (re == NULL) {
- msg_err_map ("cannot parse regexp %s: %e", key, err);
-
- if (err) {
- g_error_free (err);
- }
-
- return;
- }
-
- vlen = strlen (value);
- val = rspamd_mempool_alloc0 (re_map->pool, sizeof (*val) +
- vlen + 1);
- memcpy (val->value, value, vlen); /* Null terminated due to alloc0 previously */
- nk = kh_key (re_map->htb, k).begin;
- val->key = nk;
- kh_value (re_map->htb, k) = val;
- rspamd_cryptobox_hash_update (&re_map->hst, nk, tok.len);
-
- pcre_flags = rspamd_regexp_get_pcre_flags (re);
-
- #ifndef WITH_PCRE2
- if (pcre_flags & PCRE_FLAG(UTF8)) {
- re_map->map_flags |= RSPAMD_REGEXP_MAP_FLAG_UTF;
- }
- #else
- if (pcre_flags & PCRE_FLAG(UTF)) {
- re_map->map_flags |= RSPAMD_REGEXP_MAP_FLAG_UTF;
- }
- #endif
-
- g_ptr_array_add (re_map->regexps, re);
- g_ptr_array_add (re_map->values, val);
- }
-
- static void
- rspamd_map_helper_traverse_regexp (void *data,
- rspamd_map_traverse_cb cb,
- gpointer cbdata,
- gboolean reset_hits)
- {
- rspamd_ftok_t tok;
- struct rspamd_map_helper_value *val;
- struct rspamd_regexp_map_helper *re_map = data;
-
- kh_foreach (re_map->htb, tok, val, {
- if (!cb (tok.begin, val->value, val->hits, cbdata)) {
- break;
- }
-
- if (reset_hits) {
- val->hits = 0;
- }
- });
- }
-
- struct rspamd_hash_map_helper *
- rspamd_map_helper_new_hash (struct rspamd_map *map)
- {
- struct rspamd_hash_map_helper *htb;
- rspamd_mempool_t *pool;
-
- if (map) {
- pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
- map->tag, 0);
- }
- else {
- pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
- NULL, 0);
- }
-
- htb = rspamd_mempool_alloc0_type(pool, struct rspamd_hash_map_helper);
- htb->htb = kh_init (rspamd_map_hash);
- htb->pool = pool;
- htb->map = map;
- rspamd_cryptobox_fast_hash_init (&htb->hst, map_hash_seed);
-
- return htb;
- }
-
- void
- rspamd_map_helper_destroy_hash (struct rspamd_hash_map_helper *r)
- {
- if (r == NULL || r->pool == NULL) {
- return;
- }
-
- rspamd_mempool_t *pool = r->pool;
- kh_destroy (rspamd_map_hash, r->htb);
- memset (r, 0, sizeof (*r));
- rspamd_mempool_delete (pool);
- }
-
- static void
- rspamd_map_helper_traverse_hash (void *data,
- rspamd_map_traverse_cb cb,
- gpointer cbdata,
- gboolean reset_hits)
- {
- rspamd_ftok_t tok;
- struct rspamd_map_helper_value *val;
- struct rspamd_hash_map_helper *ht = data;
-
- kh_foreach (ht->htb, tok, val, {
- if (!cb (tok.begin, val->value, val->hits, cbdata)) {
- break;
- }
-
- if (reset_hits) {
- val->hits = 0;
- }
- });
- }
-
- struct rspamd_radix_map_helper *
- rspamd_map_helper_new_radix (struct rspamd_map *map)
- {
- struct rspamd_radix_map_helper *r;
- rspamd_mempool_t *pool;
- const gchar *name = "unnamed";
-
- if (map) {
- pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
- map->tag, 0);
- name = map->name;
- }
- else {
- pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
- NULL, 0);
- }
-
- r = rspamd_mempool_alloc0_type (pool, struct rspamd_radix_map_helper);
- r->trie = radix_create_compressed_with_pool (pool, name);
- r->htb = kh_init (rspamd_map_hash);
- r->pool = pool;
- r->map = map;
- rspamd_cryptobox_fast_hash_init (&r->hst, map_hash_seed);
-
- return r;
- }
-
- void
- rspamd_map_helper_destroy_radix (struct rspamd_radix_map_helper *r)
- {
- if (r == NULL || !r->pool) {
- return;
- }
-
- kh_destroy (rspamd_map_hash, r->htb);
- rspamd_mempool_t *pool = r->pool;
- memset (r, 0, sizeof (*r));
- rspamd_mempool_delete (pool);
- }
-
- static void
- rspamd_map_helper_traverse_radix (void *data,
- rspamd_map_traverse_cb cb,
- gpointer cbdata,
- gboolean reset_hits)
- {
- rspamd_ftok_t tok;
- struct rspamd_map_helper_value *val;
- struct rspamd_radix_map_helper *r = data;
-
- kh_foreach (r->htb, tok, val, {
- if (!cb (tok.begin, val->value, val->hits, cbdata)) {
- break;
- }
-
- if (reset_hits) {
- val->hits = 0;
- }
- });
- }
-
- struct rspamd_regexp_map_helper *
- rspamd_map_helper_new_regexp (struct rspamd_map *map,
- enum rspamd_regexp_map_flags flags)
- {
- struct rspamd_regexp_map_helper *re_map;
- rspamd_mempool_t *pool;
-
- pool = rspamd_mempool_new (rspamd_mempool_suggest_size (),
- map->tag, 0);
-
- re_map = rspamd_mempool_alloc0_type (pool, struct rspamd_regexp_map_helper);
- re_map->pool = pool;
- re_map->values = g_ptr_array_new ();
- re_map->regexps = g_ptr_array_new ();
- re_map->map = map;
- re_map->map_flags = flags;
- re_map->htb = kh_init (rspamd_map_hash);
- rspamd_cryptobox_hash_init (&re_map->hst, NULL, 0);
-
- return re_map;
- }
-
-
- void
- rspamd_map_helper_destroy_regexp (struct rspamd_regexp_map_helper *re_map)
- {
- rspamd_regexp_t *re;
- guint i;
-
- if (!re_map || !re_map->regexps) {
- return;
- }
-
- #ifdef WITH_HYPERSCAN
- if (re_map->hs_scratch) {
- hs_free_scratch (re_map->hs_scratch);
- }
- if (re_map->hs_db) {
- hs_free_database (re_map->hs_db);
- }
- if (re_map->patterns) {
- for (i = 0; i < re_map->regexps->len; i ++) {
- g_free (re_map->patterns[i]);
- }
-
- g_free (re_map->patterns);
- }
- if (re_map->flags) {
- g_free (re_map->flags);
- }
- if (re_map->ids) {
- g_free (re_map->ids);
- }
- #endif
-
- for (i = 0; i < re_map->regexps->len; i ++) {
- re = g_ptr_array_index (re_map->regexps, i);
- rspamd_regexp_unref (re);
- }
-
- g_ptr_array_free (re_map->regexps, TRUE);
- g_ptr_array_free (re_map->values, TRUE);
- kh_destroy (rspamd_map_hash, re_map->htb);
-
- rspamd_mempool_t *pool = re_map->pool;
- memset (re_map, 0, sizeof (*re_map));
- rspamd_mempool_delete (pool);
- }
-
- gchar *
- rspamd_kv_list_read (
- gchar * chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
- {
- if (data->cur_data == NULL) {
- data->cur_data = rspamd_map_helper_new_hash (data->map);
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- rspamd_map_helper_insert_hash,
- "",
- final);
- }
-
- void
- rspamd_kv_list_fin (struct map_cb_data *data, void **target)
- {
- struct rspamd_map *map = data->map;
- struct rspamd_hash_map_helper *htb;
-
- if (data->errored) {
- /* Clean up the current data and do not touch prev data */
- if (data->cur_data) {
- msg_info_map ("cleanup unfinished new data as error occurred for %s",
- map->name);
- htb = (struct rspamd_hash_map_helper *) data->cur_data;
- rspamd_map_helper_destroy_hash(htb);
- data->cur_data = NULL;
- }
- }
- else {
- if (data->cur_data) {
- htb = (struct rspamd_hash_map_helper *) data->cur_data;
- msg_info_map ("read hash of %d elements from %s", kh_size(htb->htb),
- map->name);
- data->map->traverse_function = rspamd_map_helper_traverse_hash;
- data->map->nelts = kh_size (htb->htb);
- data->map->digest = rspamd_cryptobox_fast_hash_final(&htb->hst);
- }
-
- if (target) {
- *target = data->cur_data;
- }
-
- if (data->prev_data) {
- htb = (struct rspamd_hash_map_helper *) data->prev_data;
- rspamd_map_helper_destroy_hash(htb);
- }
- }
- }
-
- void
- rspamd_kv_list_dtor (struct map_cb_data *data)
- {
- struct rspamd_hash_map_helper *htb;
-
- if (data->cur_data) {
- htb = (struct rspamd_hash_map_helper *)data->cur_data;
- rspamd_map_helper_destroy_hash (htb);
- }
- }
-
- gchar *
- rspamd_radix_read (
- gchar * chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_radix_map_helper *r;
- struct rspamd_map *map = data->map;
-
- if (data->cur_data == NULL) {
- r = rspamd_map_helper_new_radix (map);
- data->cur_data = r;
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- rspamd_map_helper_insert_radix,
- hash_fill,
- final);
- }
-
- void
- rspamd_radix_fin (struct map_cb_data *data, void **target)
- {
- struct rspamd_map *map = data->map;
- struct rspamd_radix_map_helper *r;
-
- if (data->errored) {
- /* Clean up the current data and do not touch prev data */
- if (data->cur_data) {
- msg_info_map ("cleanup unfinished new data as error occurred for %s",
- map->name);
- r = (struct rspamd_radix_map_helper *) data->cur_data;
- rspamd_map_helper_destroy_radix(r);
- data->cur_data = NULL;
- }
- }
- else {
- if (data->cur_data) {
- r = (struct rspamd_radix_map_helper *) data->cur_data;
- msg_info_map ("read radix trie of %z elements: %s",
- radix_get_size(r->trie), radix_get_info(r->trie));
- data->map->traverse_function = rspamd_map_helper_traverse_radix;
- data->map->nelts = kh_size (r->htb);
- data->map->digest = rspamd_cryptobox_fast_hash_final(&r->hst);
- }
-
- if (target) {
- *target = data->cur_data;
- }
-
- if (data->prev_data) {
- r = (struct rspamd_radix_map_helper *) data->prev_data;
- rspamd_map_helper_destroy_radix(r);
- }
- }
- }
-
- void
- rspamd_radix_dtor (struct map_cb_data *data)
- {
- struct rspamd_radix_map_helper *r;
-
- if (data->cur_data) {
- r = (struct rspamd_radix_map_helper *)data->cur_data;
- rspamd_map_helper_destroy_radix (r);
- }
- }
-
- #ifdef WITH_HYPERSCAN
- struct rspamd_re_maps_cache_dtor_cbdata {
- struct rspamd_config *cfg;
- GHashTable *valid_re_hashes;
- gchar *dirname;
- };
-
- static void
- rspamd_re_maps_cache_cleanup_dtor (gpointer ud)
- {
- struct rspamd_re_maps_cache_dtor_cbdata *cbd =
- (struct rspamd_re_maps_cache_dtor_cbdata *)ud;
- GPtrArray *cache_files;
- GError *err = NULL;
- struct rspamd_config *cfg;
-
- cfg = cbd->cfg;
-
- if (cfg->cur_worker != NULL) {
- /* Skip dtor, limit it to main process only */
- return;
- }
-
- cache_files = rspamd_glob_path (cbd->dirname, "*.hsmc", FALSE, &err);
-
- if (!cache_files) {
- msg_err_config ("cannot glob files in %s: %e", cbd->dirname, err);
- g_error_free (err);
- }
- else {
- const gchar *fname;
- guint i;
-
- PTR_ARRAY_FOREACH (cache_files, i, fname) {
- gchar *basename = g_path_get_basename (fname);
-
- if (g_hash_table_lookup (cbd->valid_re_hashes, basename) == NULL) {
- gchar *dir;
-
- dir = g_path_get_dirname (fname);
-
- /* Sanity check to avoid removal of something bad */
- if (strcmp (dir, cbd->dirname) != 0) {
- msg_err_config ("bogus file found: %s in %s, skip deleting",
- fname, dir);
- }
- else {
- if (unlink (fname) == -1) {
- msg_err_config ("cannot delete obsolete file %s in %s: %s",
- fname, dir, strerror (errno));
- }
- else {
- msg_info_config ("deleted obsolete file %s in %s",
- fname, dir);
- }
- }
-
- g_free (dir);
- }
- else {
- msg_debug_config ("valid re cache file %s", fname);
- }
-
- g_free (basename);
- }
-
- g_ptr_array_free (cache_files, TRUE);
- }
-
- g_hash_table_unref (cbd->valid_re_hashes);
- g_free (cbd->dirname);
- }
-
- static void
- rspamd_re_map_cache_update (const gchar *fname, struct rspamd_config *cfg)
- {
- GHashTable *valid_re_hashes;
-
- valid_re_hashes = rspamd_mempool_get_variable (cfg->cfg_pool,
- RSPAMD_MEMPOOL_RE_MAPS_CACHE);
-
- if (!valid_re_hashes) {
- valid_re_hashes = g_hash_table_new_full (g_str_hash, g_str_equal,
- g_free, NULL);
- rspamd_mempool_set_variable (cfg->cfg_pool,
- RSPAMD_MEMPOOL_RE_MAPS_CACHE,
- valid_re_hashes, (rspamd_mempool_destruct_t)g_hash_table_unref);
-
- /* We also add a cleanup dtor for all hashes */
- static struct rspamd_re_maps_cache_dtor_cbdata cbd;
-
- cbd.valid_re_hashes = g_hash_table_ref (valid_re_hashes);
- cbd.cfg = cfg;
- cbd.dirname = g_path_get_dirname (fname);
- rspamd_mempool_add_destructor (cfg->cfg_pool,
- rspamd_re_maps_cache_cleanup_dtor, &cbd);
- }
-
- g_hash_table_insert (valid_re_hashes, g_path_get_basename (fname), "1");
- }
-
- static gboolean
- rspamd_try_load_re_map_cache (struct rspamd_regexp_map_helper *re_map)
- {
- gchar fp[PATH_MAX];
- gpointer data;
- gsize len;
- struct rspamd_map *map;
-
- map = re_map->map;
-
- if (!map->cfg->hs_cache_dir) {
- return FALSE;
- }
-
- rspamd_snprintf (fp, sizeof (fp), "%s/%*xs.hsmc",
- map->cfg->hs_cache_dir,
- (gint)rspamd_cryptobox_HASHBYTES / 2, re_map->re_digest);
-
- if ((data = rspamd_file_xmap (fp, PROT_READ, &len, TRUE)) != NULL) {
- if (hs_deserialize_database (data, len, &re_map->hs_db) == HS_SUCCESS) {
- rspamd_re_map_cache_update (fp, map->cfg);
- munmap (data, len);
-
- msg_info_map ("loaded hypersan cache from %s (%Hz length) for %s",
- fp, len, map->name);
-
- return TRUE;
- }
-
- msg_info_map ("invalid hypersan cache in %s (%Hz length) for %s, removing file",
- fp, len, map->name);
- munmap (data, len);
- /* Remove stale file */
- (void)unlink (fp);
- }
-
- return FALSE;
- }
-
- static gboolean
- rspamd_try_save_re_map_cache (struct rspamd_regexp_map_helper *re_map)
- {
- gchar fp[PATH_MAX], np[PATH_MAX];
- gsize len;
- gint fd;
- char *bytes = NULL;
- struct rspamd_map *map;
-
- map = re_map->map;
-
- if (!map->cfg->hs_cache_dir) {
- return FALSE;
- }
-
- rspamd_snprintf (fp, sizeof (fp), "%s/%*xs.hsmc.tmp",
- re_map->map->cfg->hs_cache_dir,
- (gint)rspamd_cryptobox_HASHBYTES / 2, re_map->re_digest);
-
- if ((fd = rspamd_file_xopen (fp, O_WRONLY | O_CREAT | O_EXCL, 00644, 0)) != -1) {
- if (hs_serialize_database (re_map->hs_db, &bytes, &len) == HS_SUCCESS) {
- if (write (fd, bytes, len) == -1) {
- msg_warn_map ("cannot write hyperscan cache to %s: %s",
- fp, strerror (errno));
- unlink (fp);
- free (bytes);
- }
- else {
- free (bytes);
- fsync (fd);
-
- rspamd_snprintf (np, sizeof (np), "%s/%*xs.hsmc",
- re_map->map->cfg->hs_cache_dir,
- (gint)rspamd_cryptobox_HASHBYTES / 2, re_map->re_digest);
-
- if (rename (fp, np) == -1) {
- msg_warn_map ("cannot rename hyperscan cache from %s to %s: %s",
- fp, np, strerror (errno));
- unlink (fp);
- }
- else {
- msg_info_map ("written cached hyperscan data for %s to %s (%Hz length)",
- map->name, np, len);
-
- rspamd_re_map_cache_update (np, map->cfg);
- }
- }
- }
- else {
- msg_warn_map ("cannot serialize hyperscan cache to %s: %s",
- fp, strerror (errno));
- unlink (fp);
- }
-
-
- close (fd);
- }
-
- return FALSE;
- }
-
- static gboolean
- rspamd_re_map_cache_cleanup_old (struct rspamd_regexp_map_helper *old_re_map)
- {
- gchar fp[PATH_MAX];
- struct rspamd_map *map;
-
- map = old_re_map->map;
-
- if (!map->cfg->hs_cache_dir) {
- return FALSE;
- }
-
- rspamd_snprintf (fp, sizeof (fp), "%s/%*xs.hsmc",
- map->cfg->hs_cache_dir,
- (gint)rspamd_cryptobox_HASHBYTES / 2, old_re_map->re_digest);
-
- msg_info_map ("unlink stale cache file for %s: %s", map->name, fp);
-
- if (unlink (fp) == -1) {
- msg_warn_map ("cannot unlink stale cache file for %s (%s): %s",
- map->name, fp, strerror (errno));
- return FALSE;
- }
-
- GHashTable *valid_re_hashes;
-
- valid_re_hashes = rspamd_mempool_get_variable (map->cfg->cfg_pool,
- RSPAMD_MEMPOOL_RE_MAPS_CACHE);
-
- if (valid_re_hashes) {
- g_hash_table_remove (valid_re_hashes, fp);
- }
-
- return TRUE;
- }
-
- #endif
-
- static void
- rspamd_re_map_finalize (struct rspamd_regexp_map_helper *re_map)
- {
- #ifdef WITH_HYPERSCAN
- guint i;
- hs_platform_info_t plt;
- hs_compile_error_t *err;
- struct rspamd_map *map;
- rspamd_regexp_t *re;
- gint pcre_flags;
-
- map = re_map->map;
-
- #if !defined(__aarch64__) && !defined(__powerpc64__)
- if (!(map->cfg->libs_ctx->crypto_ctx->cpu_config & CPUID_SSSE3)) {
- msg_info_map ("disable hyperscan for map %s, ssse3 instructions are not supported by CPU",
- map->name);
- return;
- }
- #endif
-
- if (hs_populate_platform (&plt) != HS_SUCCESS) {
- msg_err_map ("cannot populate hyperscan platform");
- return;
- }
-
- re_map->patterns = g_new (gchar *, re_map->regexps->len);
- re_map->flags = g_new (gint, re_map->regexps->len);
- re_map->ids = g_new (gint, re_map->regexps->len);
-
- for (i = 0; i < re_map->regexps->len; i ++) {
- const gchar *pat;
- gchar *escaped;
- gint pat_flags;
-
- re = g_ptr_array_index (re_map->regexps, i);
- pcre_flags = rspamd_regexp_get_pcre_flags (re);
- pat = rspamd_regexp_get_pattern (re);
- pat_flags = rspamd_regexp_get_flags (re);
-
- if (pat_flags & RSPAMD_REGEXP_FLAG_UTF) {
- escaped = rspamd_str_regexp_escape (pat, strlen (pat), NULL,
- RSPAMD_REGEXP_ESCAPE_RE|RSPAMD_REGEXP_ESCAPE_UTF);
- re_map->flags[i] |= HS_FLAG_UTF8;
- }
- else {
- escaped = rspamd_str_regexp_escape (pat, strlen (pat), NULL,
- RSPAMD_REGEXP_ESCAPE_RE);
- }
-
- re_map->patterns[i] = escaped;
- re_map->flags[i] = HS_FLAG_SINGLEMATCH;
-
- #ifndef WITH_PCRE2
- if (pcre_flags & PCRE_FLAG(UTF8)) {
- re_map->flags[i] |= HS_FLAG_UTF8;
- }
- #else
- if (pcre_flags & PCRE_FLAG(UTF)) {
- re_map->flags[i] |= HS_FLAG_UTF8;
- }
- #endif
- if (pcre_flags & PCRE_FLAG(CASELESS)) {
- re_map->flags[i] |= HS_FLAG_CASELESS;
- }
- if (pcre_flags & PCRE_FLAG(MULTILINE)) {
- re_map->flags[i] |= HS_FLAG_MULTILINE;
- }
- if (pcre_flags & PCRE_FLAG(DOTALL)) {
- re_map->flags[i] |= HS_FLAG_DOTALL;
- }
- if (rspamd_regexp_get_maxhits (re) == 1) {
- re_map->flags[i] |= HS_FLAG_SINGLEMATCH;
- }
-
- re_map->ids[i] = i;
- }
-
- if (re_map->regexps->len > 0 && re_map->patterns) {
-
- if (!rspamd_try_load_re_map_cache (re_map)) {
- gdouble ts1 = rspamd_get_ticks (FALSE);
-
- if (hs_compile_multi ((const gchar **) re_map->patterns,
- re_map->flags,
- re_map->ids,
- re_map->regexps->len,
- HS_MODE_BLOCK,
- &plt,
- &re_map->hs_db,
- &err) != HS_SUCCESS) {
-
- msg_err_map ("cannot create tree of regexp when processing '%s': %s",
- err->expression >= 0 ?
- re_map->patterns[err->expression] :
- "unknown regexp", err->message);
- re_map->hs_db = NULL;
- hs_free_compile_error (err);
-
- return;
- }
-
- ts1 = (rspamd_get_ticks (FALSE) - ts1) * 1000.0;
- msg_info_map ("hyperscan compiled %d regular expressions from %s in %.1f ms",
- re_map->regexps->len, re_map->map->name, ts1);
- rspamd_try_save_re_map_cache (re_map);
- }
- else {
- msg_info_map ("hyperscan read %d cached regular expressions from %s",
- re_map->regexps->len, re_map->map->name);
- }
-
- if (hs_alloc_scratch (re_map->hs_db, &re_map->hs_scratch) != HS_SUCCESS) {
- msg_err_map ("cannot allocate scratch space for hyperscan");
- hs_free_database (re_map->hs_db);
- re_map->hs_db = NULL;
- }
- }
- else {
- msg_err_map ("regexp map is empty");
- }
- #endif
- }
-
- gchar *
- rspamd_regexp_list_read_single (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_regexp_map_helper *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_map_helper_new_regexp (data->map, 0);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- rspamd_map_helper_insert_re,
- hash_fill,
- final);
- }
-
- gchar *
- rspamd_glob_list_read_single (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_regexp_map_helper *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_map_helper_new_regexp (data->map, RSPAMD_REGEXP_MAP_FLAG_GLOB);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- rspamd_map_helper_insert_re,
- hash_fill,
- final);
- }
-
- gchar *
- rspamd_regexp_list_read_multiple (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_regexp_map_helper *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_map_helper_new_regexp (data->map,
- RSPAMD_REGEXP_MAP_FLAG_MULTIPLE);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- rspamd_map_helper_insert_re,
- hash_fill,
- final);
- }
-
- gchar *
- rspamd_glob_list_read_multiple (
- gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_regexp_map_helper *re_map;
-
- if (data->cur_data == NULL) {
- re_map = rspamd_map_helper_new_regexp (data->map,
- RSPAMD_REGEXP_MAP_FLAG_GLOB|RSPAMD_REGEXP_MAP_FLAG_MULTIPLE);
- data->cur_data = re_map;
- }
-
- return rspamd_parse_kv_list (
- chunk,
- len,
- data,
- rspamd_map_helper_insert_re,
- hash_fill,
- final);
- }
-
-
- void
- rspamd_regexp_list_fin (struct map_cb_data *data, void **target)
- {
- struct rspamd_regexp_map_helper *re_map = NULL, *old_re_map;
- struct rspamd_map *map = data->map;
-
- if (data->errored) {
- /* Clean up the current data and do not touch prev data */
- if (data->cur_data) {
- msg_info_map ("cleanup unfinished new data as error occurred for %s",
- map->name);
- re_map = (struct rspamd_regexp_map_helper *)data->cur_data;
- rspamd_map_helper_destroy_regexp (re_map);
- data->cur_data = NULL;
- }
- }
- else {
- if (data->cur_data) {
- re_map = data->cur_data;
- rspamd_cryptobox_hash_final(&re_map->hst, re_map->re_digest);
- memcpy(&data->map->digest, re_map->re_digest, sizeof(data->map->digest));
- rspamd_re_map_finalize(re_map);
- msg_info_map ("read regexp list of %ud elements",
- re_map->regexps->len);
- data->map->traverse_function = rspamd_map_helper_traverse_regexp;
- data->map->nelts = kh_size (re_map->htb);
- }
-
- if (target) {
- *target = data->cur_data;
- }
-
- if (data->prev_data) {
- old_re_map = data->prev_data;
-
- #ifdef WITH_HYPERSCAN
- if (re_map && memcmp(re_map->re_digest, old_re_map->re_digest,
- sizeof(re_map->re_digest)) != 0) {
- /* Cleanup old stuff */
- rspamd_re_map_cache_cleanup_old(old_re_map);
- }
- #endif
-
- rspamd_map_helper_destroy_regexp(old_re_map);
- }
- }
- }
- void
- rspamd_regexp_list_dtor (struct map_cb_data *data)
- {
- if (data->cur_data) {
- rspamd_map_helper_destroy_regexp (data->cur_data);
- }
- }
-
- #ifdef WITH_HYPERSCAN
- static int
- rspamd_match_hs_single_handler (unsigned int id, unsigned long long from,
- unsigned long long to,
- unsigned int flags, void *context)
- {
- guint *i = context;
- /* Always return non-zero as we need a single match here */
-
- *i = id;
-
- return 1;
- }
- #endif
-
- gconstpointer
- rspamd_match_regexp_map_single (struct rspamd_regexp_map_helper *map,
- const gchar *in, gsize len)
- {
- guint i;
- rspamd_regexp_t *re;
- gint res = 0;
- gpointer ret = NULL;
- struct rspamd_map_helper_value *val;
- gboolean validated = FALSE;
-
- g_assert (in != NULL);
-
- if (map == NULL || len == 0 || map->regexps == NULL) {
- return NULL;
- }
-
- if (map->map_flags & RSPAMD_REGEXP_MAP_FLAG_UTF) {
- if (rspamd_fast_utf8_validate (in, len) == 0) {
- validated = TRUE;
- }
- }
- else {
- validated = TRUE;
- }
-
- #ifdef WITH_HYPERSCAN
- if (map->hs_db && map->hs_scratch) {
-
- if (validated) {
-
- res = hs_scan (map->hs_db, in, len, 0, map->hs_scratch,
- rspamd_match_hs_single_handler, (void *)&i);
-
- if (res == HS_SCAN_TERMINATED) {
- res = 1;
- val = g_ptr_array_index (map->values, i);
-
- ret = val->value;
- val->hits ++;
- }
-
- return ret;
- }
- }
- #endif
-
- if (!res) {
- /* PCRE version */
- for (i = 0; i < map->regexps->len; i ++) {
- re = g_ptr_array_index (map->regexps, i);
-
- if (rspamd_regexp_search (re, in, len, NULL, NULL, !validated, NULL)) {
- val = g_ptr_array_index (map->values, i);
-
- ret = val->value;
- val->hits ++;
- break;
- }
- }
- }
-
- return ret;
- }
-
- #ifdef WITH_HYPERSCAN
- struct rspamd_multiple_cbdata {
- GPtrArray *ar;
- struct rspamd_regexp_map_helper *map;
- };
-
- static int
- rspamd_match_hs_multiple_handler (unsigned int id, unsigned long long from,
- unsigned long long to,
- unsigned int flags, void *context)
- {
- struct rspamd_multiple_cbdata *cbd = context;
- struct rspamd_map_helper_value *val;
-
-
- if (id < cbd->map->values->len) {
- val = g_ptr_array_index (cbd->map->values, id);
- val->hits ++;
- g_ptr_array_add (cbd->ar, val->value);
- }
-
- /* Always return zero as we need all matches here */
- return 0;
- }
- #endif
-
- GPtrArray*
- rspamd_match_regexp_map_all (struct rspamd_regexp_map_helper *map,
- const gchar *in, gsize len)
- {
- guint i;
- rspamd_regexp_t *re;
- GPtrArray *ret;
- gint res = 0;
- gboolean validated = FALSE;
- struct rspamd_map_helper_value *val;
-
- if (map == NULL || map->regexps == NULL || len == 0) {
- return NULL;
- }
-
- g_assert (in != NULL);
-
- if (map->map_flags & RSPAMD_REGEXP_MAP_FLAG_UTF) {
- if (rspamd_fast_utf8_validate (in, len) == 0) {
- validated = TRUE;
- }
- }
- else {
- validated = TRUE;
- }
-
- ret = g_ptr_array_new ();
-
- #ifdef WITH_HYPERSCAN
- if (map->hs_db && map->hs_scratch) {
-
- if (validated) {
- struct rspamd_multiple_cbdata cbd;
-
- cbd.ar = ret;
- cbd.map = map;
-
- if (hs_scan (map->hs_db, in, len, 0, map->hs_scratch,
- rspamd_match_hs_multiple_handler, &cbd) == HS_SUCCESS) {
- res = 1;
- }
- }
- }
- #endif
-
- if (!res) {
- /* PCRE version */
- for (i = 0; i < map->regexps->len; i ++) {
- re = g_ptr_array_index (map->regexps, i);
-
- if (rspamd_regexp_search (re, in, len, NULL, NULL,
- !validated, NULL)) {
- val = g_ptr_array_index (map->values, i);
- val->hits ++;
- g_ptr_array_add (ret, val->value);
- }
- }
- }
-
- if (ret->len > 0) {
- return ret;
- }
-
- g_ptr_array_free (ret, TRUE);
-
- return NULL;
- }
-
- gconstpointer
- rspamd_match_hash_map (struct rspamd_hash_map_helper *map, const gchar *in,
- gsize len)
- {
- khiter_t k;
- struct rspamd_map_helper_value *val;
- rspamd_ftok_t tok;
-
- if (map == NULL || map->htb == NULL) {
- return NULL;
- }
-
- tok.begin = in;
- tok.len = len;
-
- k = kh_get (rspamd_map_hash, map->htb, tok);
-
- if (k != kh_end (map->htb)) {
- val = kh_value (map->htb, k);
- val->hits ++;
-
- return val->value;
- }
-
- return NULL;
- }
-
- gconstpointer
- rspamd_match_radix_map (struct rspamd_radix_map_helper *map,
- const guchar *in, gsize inlen)
- {
- struct rspamd_map_helper_value *val;
-
- if (map == NULL || map->trie == NULL) {
- return NULL;
- }
-
- val = (struct rspamd_map_helper_value *)radix_find_compressed (map->trie,
- in, inlen);
-
- if (val != (gconstpointer)RADIX_NO_VALUE) {
- val->hits ++;
-
- return val->value;
- }
-
- return NULL;
- }
-
- gconstpointer
- rspamd_match_radix_map_addr (struct rspamd_radix_map_helper *map,
- const rspamd_inet_addr_t *addr)
- {
- struct rspamd_map_helper_value *val;
-
- if (map == NULL || map->trie == NULL) {
- return NULL;
- }
-
- val = (struct rspamd_map_helper_value *)radix_find_compressed_addr (map->trie, addr);
-
- if (val != (gconstpointer)RADIX_NO_VALUE) {
- val->hits ++;
-
- return val->value;
- }
-
- return NULL;
- }
-
-
- /*
- * CBD stuff
- */
-
- struct rspamd_cdb_map_helper *
- rspamd_map_helper_new_cdb (struct rspamd_map *map)
- {
- struct rspamd_cdb_map_helper *n;
-
- n = g_malloc0 (sizeof (*n));
- n->cdbs = (GQueue)G_QUEUE_INIT;
- n->map = map;
-
- rspamd_cryptobox_fast_hash_init (&n->hst, map_hash_seed);
-
- return n;
- }
-
- void
- rspamd_map_helper_destroy_cdb (struct rspamd_cdb_map_helper *c)
- {
- if (c == NULL) {
- return;
- }
-
- GList *cur = c->cdbs.head;
-
- while (cur) {
- struct cdb *cdb = (struct cdb *)cur->data;
-
- cdb_free (cdb);
- g_free (cdb->filename);
- close (cdb->cdb_fd);
- g_free (cdb);
-
- cur = g_list_next (cur);
- }
-
- g_queue_clear (&c->cdbs);
-
- g_free (c);
- }
-
- gchar *
- rspamd_cdb_list_read (gchar *chunk,
- gint len,
- struct map_cb_data *data,
- gboolean final)
- {
- struct rspamd_cdb_map_helper *cdb_data;
- struct cdb *found = NULL;
- struct rspamd_map *map = data->map;
-
- g_assert (map->no_file_read);
-
- if (data->cur_data == NULL) {
- cdb_data = rspamd_map_helper_new_cdb (data->map);
- data->cur_data = cdb_data;
- }
- else {
- cdb_data = (struct rspamd_cdb_map_helper *)data->cur_data;
- }
-
- GList *cur = cdb_data->cdbs.head;
-
- while (cur) {
- struct cdb *elt = (struct cdb *)cur->data;
-
- if (strcmp (elt->filename, chunk) == 0) {
- found = elt;
- break;
- }
-
- cur = g_list_next (cur);
- }
-
- if (found == NULL) {
- /* New cdb */
- gint fd;
- struct cdb *cdb;
-
- fd = rspamd_file_xopen (chunk, O_RDONLY, 0, TRUE);
-
- if (fd == -1) {
- msg_err_map ("cannot open cdb map from %s: %s", chunk, strerror (errno));
-
- return NULL;
- }
-
- cdb = g_malloc0 (sizeof (struct cdb));
-
- if (cdb_init (cdb, fd) == -1) {
- g_free (cdb);
- msg_err_map ("cannot init cdb map from %s: %s", chunk, strerror (errno));
-
- return NULL;
- }
-
- cdb->filename = g_strdup (chunk);
- g_queue_push_tail (&cdb_data->cdbs, cdb);
- cdb_data->total_size += cdb->cdb_fsize;
- rspamd_cryptobox_fast_hash_update (&cdb_data->hst, chunk, len);
- }
-
- return chunk + len;
- }
-
- void
- rspamd_cdb_list_fin (struct map_cb_data *data, void **target)
- {
- struct rspamd_map *map = data->map;
- struct rspamd_cdb_map_helper *cdb_data;
-
- if (data->errored) {
- /* Clean up the current data and do not touch prev data */
- if (data->cur_data) {
- msg_info_map ("cleanup unfinished new data as error occurred for %s",
- map->name);
- cdb_data = (struct rspamd_cdb_map_helper *) data->cur_data;
- rspamd_map_helper_destroy_cdb(cdb_data);
- data->cur_data = NULL;
- }
- }
- else {
- if (data->cur_data) {
- cdb_data = (struct rspamd_cdb_map_helper *) data->cur_data;
- msg_info_map ("read cdb of %Hz size", cdb_data->total_size);
- data->map->traverse_function = NULL;
- data->map->nelts = 0;
- data->map->digest = rspamd_cryptobox_fast_hash_final(&cdb_data->hst);
- }
-
- if (target) {
- *target = data->cur_data;
- }
-
- if (data->prev_data) {
- cdb_data = (struct rspamd_cdb_map_helper *) data->prev_data;
- rspamd_map_helper_destroy_cdb(cdb_data);
- }
- }
- }
- void
- rspamd_cdb_list_dtor (struct map_cb_data *data)
- {
- if (data->cur_data) {
- rspamd_map_helper_destroy_cdb (data->cur_data);
- }
- }
-
- gconstpointer
- rspamd_match_cdb_map (struct rspamd_cdb_map_helper *map,
- const gchar *in, gsize inlen)
- {
- if (map == NULL || map->cdbs.head == NULL) {
- return NULL;
- }
-
- GList *cur = map->cdbs.head;
- static rspamd_ftok_t found;
-
- while (cur) {
- struct cdb *cdb = (struct cdb *)cur->data;
-
- if (cdb_find (cdb, in, inlen) > 0) {
- /* Extract and push value to lua as string */
- unsigned vlen;
- gconstpointer vpos;
-
- vpos = cdb->cdb_mem + cdb_datapos (cdb);
- vlen = cdb_datalen (cdb);
- found.len = vlen;
- found.begin = vpos;
-
- return &found; /* Do not reuse! */
- }
-
- cur = g_list_next (cur);
- }
-
- return NULL;
- }
|