1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835 |
- /*-
- * Copyright 2016 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- #include "config.h"
- #include "util.h"
- #include "rspamd.h"
- #include "message.h"
- #include "rspamd_symcache.h"
- #include "cfg_file.h"
- #include "lua/lua_common.h"
- #include "unix-std.h"
- #include "contrib/t1ha/t1ha.h"
- #include "libserver/worker_util.h"
- #include "khash.h"
- #include "utlist.h"
- #include <math.h>
-
- #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
- # include <stdalign.h>
- #endif
-
- #define msg_err_cache(...) rspamd_default_log_function (G_LOG_LEVEL_CRITICAL, \
- cache->static_pool->tag.tagname, cache->cfg->checksum, \
- G_STRFUNC, \
- __VA_ARGS__)
- #define msg_warn_cache(...) rspamd_default_log_function (G_LOG_LEVEL_WARNING, \
- cache->static_pool->tag.tagname, cache->cfg->checksum, \
- G_STRFUNC, \
- __VA_ARGS__)
- #define msg_info_cache(...) rspamd_default_log_function (G_LOG_LEVEL_INFO, \
- cache->static_pool->tag.tagname, cache->cfg->checksum, \
- G_STRFUNC, \
- __VA_ARGS__)
- #define msg_debug_cache(...) rspamd_conditional_debug_fast (NULL, NULL, \
- rspamd_symcache_log_id, "symcache", cache->cfg->checksum, \
- G_STRFUNC, \
- __VA_ARGS__)
- #define msg_debug_cache_task(...) rspamd_conditional_debug_fast (NULL, NULL, \
- rspamd_symcache_log_id, "symcache", task->task_pool->tag.uid, \
- G_STRFUNC, \
- __VA_ARGS__)
-
- INIT_LOG_MODULE(symcache)
-
- #define CHECK_START_BIT(checkpoint, dyn_item) \
- ((dyn_item)->started)
- #define SET_START_BIT(checkpoint, dyn_item) \
- (dyn_item)->started = 1
- #define CLR_START_BIT(checkpoint, dyn_item) \
- (dyn_item)->started = 0
-
- #define CHECK_FINISH_BIT(checkpoint, dyn_item) \
- ((dyn_item)->finished)
- #define SET_FINISH_BIT(checkpoint, dyn_item) \
- (dyn_item)->finished = 1
- #define CLR_FINISH_BIT(checkpoint, dyn_item) \
- (dyn_item)->finished = 0
- static const guchar rspamd_symcache_magic[8] = {'r', 's', 'c', 2, 0, 0, 0, 0 };
-
- struct rspamd_symcache_header {
- guchar magic[8];
- guint nitems;
- guchar checksum[64];
- guchar unused[128];
- };
-
- struct symcache_order {
- GPtrArray *d;
- guint id;
- ref_entry_t ref;
- };
-
- /*
- * This structure is optimised to store ids list:
- * - If the first element is -1 then use dynamic part, else use static part
- */
- struct rspamd_symcache_id_list {
- union {
- guint32 st[4];
- struct {
- guint32 e; /* First element */
- guint16 len;
- guint16 allocated;
- guint *n;
- } dyn;
- };
- };
-
- struct rspamd_symcache_condition {
- gint cb;
- struct rspamd_symcache_condition *prev, *next;
- };
-
- struct rspamd_symcache_item {
- /* This block is likely shared */
- struct rspamd_symcache_item_stat *st;
-
- guint64 last_count;
- struct rspamd_counter_data *cd;
- gchar *symbol;
- const gchar *type_descr;
- gint type;
-
- /* Callback data */
- union {
- struct {
- symbol_func_t func;
- gpointer user_data;
- struct rspamd_symcache_condition *conditions;
- } normal;
- struct {
- gint parent;
- struct rspamd_symcache_item *parent_item;
- } virtual;
- } specific;
-
- /* Condition of execution */
- gboolean enabled;
- /* Used for async stuff checks */
- gboolean is_filter;
- gboolean is_virtual;
-
- /* Priority */
- gint priority;
- /* Topological order */
- guint order;
- gint id;
- gint frequency_peaks;
- /* Settings ids */
- struct rspamd_symcache_id_list allowed_ids;
- /* Allows execution but not symbols insertion */
- struct rspamd_symcache_id_list exec_only_ids;
- struct rspamd_symcache_id_list forbidden_ids;
-
- /* Dependencies */
- GPtrArray *deps;
- GPtrArray *rdeps;
-
- /* Container */
- GPtrArray *container;
- };
-
- struct rspamd_symcache {
- /* Hash table for fast access */
- GHashTable *items_by_symbol;
- GPtrArray *items_by_id;
- struct symcache_order *items_by_order;
- GPtrArray *connfilters;
- GPtrArray *prefilters;
- GPtrArray *filters;
- GPtrArray *postfilters;
- GPtrArray *composites;
- GPtrArray *idempotent;
- GPtrArray *virtual;
- GList *delayed_deps;
- GList *delayed_conditions;
- rspamd_mempool_t *static_pool;
- guint64 cksum;
- gdouble total_weight;
- guint used_items;
- guint stats_symbols_count;
- guint64 total_hits;
- guint id;
- struct rspamd_config *cfg;
- gdouble reload_time;
- gdouble last_profile;
- gint peak_cb;
- };
-
- struct rspamd_symcache_dynamic_item {
- guint16 start_msec; /* Relative to task time */
- unsigned started:1;
- unsigned finished:1;
- /* unsigned pad:14; */
- guint32 async_events;
- };
-
-
-
- struct cache_dependency {
- struct rspamd_symcache_item *item; /* Real dependency */
- gchar *sym; /* Symbolic dep name */
- gint id; /* Real from */
- gint vid; /* Virtual from */
- };
-
- struct delayed_cache_dependency {
- gchar *from;
- gchar *to;
- };
-
- struct delayed_cache_condition {
- gchar *sym;
- gint cbref;
- lua_State *L;
- };
-
- struct cache_savepoint {
- guint version;
- guint items_inflight;
- gboolean profile;
- gboolean has_slow;
- gdouble profile_start;
-
- struct rspamd_scan_result *rs;
- gdouble lim;
-
- struct rspamd_symcache_item *cur_item;
- struct symcache_order *order;
- struct rspamd_symcache_dynamic_item dynamic_items[];
- };
-
- struct rspamd_cache_refresh_cbdata {
- gdouble last_resort;
- ev_timer resort_ev;
- struct rspamd_symcache *cache;
- struct rspamd_worker *w;
- struct ev_loop *event_loop;
- };
-
- /* At least once per minute */
- #define PROFILE_MAX_TIME (60.0)
- /* For messages larger than 2Mb enable profiling */
- #define PROFILE_MESSAGE_SIZE_THRESHOLD (1024 * 1024 * 2)
- /* Enable profile at least once per this amount of messages processed */
- #define PROFILE_PROBABILITY (0.01)
-
- /* weight, frequency, time */
- #define TIME_ALPHA (1.0)
- #define WEIGHT_ALPHA (0.1)
- #define FREQ_ALPHA (0.01)
- #define SCORE_FUN(w, f, t) (((w) > 0 ? (w) : WEIGHT_ALPHA) \
- * ((f) > 0 ? (f) : FREQ_ALPHA) \
- / (t > TIME_ALPHA ? t : TIME_ALPHA))
-
- static gboolean rspamd_symcache_check_symbol (struct rspamd_task *task,
- struct rspamd_symcache *cache,
- struct rspamd_symcache_item *item,
- struct cache_savepoint *checkpoint);
- static gboolean rspamd_symcache_check_deps (struct rspamd_task *task,
- struct rspamd_symcache *cache,
- struct rspamd_symcache_item *item,
- struct cache_savepoint *checkpoint,
- guint recursion,
- gboolean check_only);
- static void rspamd_symcache_disable_symbol_checkpoint (struct rspamd_task *task,
- struct rspamd_symcache *cache, const gchar *symbol);
- static void rspamd_symcache_enable_symbol_checkpoint (struct rspamd_task *task,
- struct rspamd_symcache *cache, const gchar *symbol);
-
- static void
- rspamd_symcache_order_dtor (gpointer p)
- {
- struct symcache_order *ord = p;
-
- g_ptr_array_free (ord->d, TRUE);
- g_free (ord);
- }
-
- static void
- rspamd_symcache_order_unref (gpointer p)
- {
- struct symcache_order *ord = p;
-
- REF_RELEASE (ord);
- }
-
- static gint
- rspamd_id_cmp (const void * a, const void * b)
- {
- return (*(guint32*)a - *(guint32*)b);
- }
-
- static struct symcache_order *
- rspamd_symcache_order_new (struct rspamd_symcache *cache,
- gsize nelts)
- {
- struct symcache_order *ord;
-
- ord = g_malloc0 (sizeof (*ord));
- ord->d = g_ptr_array_sized_new (nelts);
- ord->id = cache->id;
- REF_INIT_RETAIN (ord, rspamd_symcache_order_dtor);
-
- return ord;
- }
-
- static inline struct rspamd_symcache_dynamic_item*
- rspamd_symcache_get_dynamic (struct cache_savepoint *checkpoint,
- struct rspamd_symcache_item *item)
- {
- return &checkpoint->dynamic_items[item->id];
- }
-
- static inline struct rspamd_symcache_item *
- rspamd_symcache_find_filter (struct rspamd_symcache *cache,
- const gchar *name,
- bool resolve_parent)
- {
- struct rspamd_symcache_item *item;
-
- g_assert (cache != NULL);
-
- if (name == NULL) {
- return NULL;
- }
-
- item = g_hash_table_lookup (cache->items_by_symbol, name);
-
- if (item != NULL) {
-
- if (resolve_parent && item->is_virtual && !(item->type & SYMBOL_TYPE_GHOST)) {
- item =item->specific.virtual.parent_item;
- }
-
- return item;
- }
-
- return NULL;
- }
-
- const gchar *
- rspamd_symcache_get_parent (struct rspamd_symcache *cache,
- const gchar *symbol)
- {
- struct rspamd_symcache_item *item, *parent;
-
- g_assert (cache != NULL);
-
- if (symbol == NULL) {
- return NULL;
- }
-
- item = g_hash_table_lookup (cache->items_by_symbol, symbol);
-
- if (item != NULL) {
-
- if (item->is_virtual && !(item->type & SYMBOL_TYPE_GHOST)) {
- parent = item->specific.virtual.parent_item;
-
- if (!parent) {
- item->specific.virtual.parent_item = g_ptr_array_index (cache->items_by_id,
- item->specific.virtual.parent);
- parent = item->specific.virtual.parent_item;
- }
-
- item = parent;
- }
-
- return item->symbol;
- }
-
- return NULL;
- }
-
- static gint
- postfilters_cmp (const void *p1, const void *p2, gpointer ud)
- {
- const struct rspamd_symcache_item *i1 = *(struct rspamd_symcache_item **)p1,
- *i2 = *(struct rspamd_symcache_item **)p2;
- double w1, w2;
-
- w1 = i1->priority;
- w2 = i2->priority;
-
- if (w1 > w2) {
- return 1;
- }
- else if (w1 < w2) {
- return -1;
- }
-
- return 0;
- }
-
- static gint
- prefilters_cmp (const void *p1, const void *p2, gpointer ud)
- {
- const struct rspamd_symcache_item *i1 = *(struct rspamd_symcache_item **)p1,
- *i2 = *(struct rspamd_symcache_item **)p2;
- double w1, w2;
-
- w1 = i1->priority;
- w2 = i2->priority;
-
- if (w1 < w2) {
- return 1;
- }
- else if (w1 > w2) {
- return -1;
- }
-
- return 0;
- }
-
- #define TSORT_MARK_PERM(it) (it)->order |= (1u << 31)
- #define TSORT_MARK_TEMP(it) (it)->order |= (1u << 30)
- #define TSORT_IS_MARKED_PERM(it) ((it)->order & (1u << 31))
- #define TSORT_IS_MARKED_TEMP(it) ((it)->order & (1u << 30))
- #define TSORT_UNMASK(it) ((it)->order & ~((1u << 31) | (1u << 30)))
-
- static gint
- cache_logic_cmp (const void *p1, const void *p2, gpointer ud)
- {
- const struct rspamd_symcache_item *i1 = *(struct rspamd_symcache_item **)p1,
- *i2 = *(struct rspamd_symcache_item **)p2;
- struct rspamd_symcache *cache = ud;
- double w1, w2;
- double weight1, weight2;
- double f1 = 0, f2 = 0, t1, t2, avg_freq, avg_weight;
- guint o1 = TSORT_UNMASK (i1), o2 = TSORT_UNMASK (i2);
-
-
- if (o1 == o2) {
- /* Heurstic */
- if (i1->priority == i2->priority) {
- avg_freq = ((gdouble) cache->total_hits / cache->used_items);
- avg_weight = (cache->total_weight / cache->used_items);
- f1 = (double) i1->st->total_hits / avg_freq;
- f2 = (double) i2->st->total_hits / avg_freq;
- weight1 = fabs (i1->st->weight) / avg_weight;
- weight2 = fabs (i2->st->weight) / avg_weight;
- t1 = i1->st->avg_time;
- t2 = i2->st->avg_time;
- w1 = SCORE_FUN (weight1, f1, t1);
- w2 = SCORE_FUN (weight2, f2, t2);
- } else {
- /* Strict sorting */
- w1 = abs (i1->priority);
- w2 = abs (i2->priority);
- }
- }
- else {
- w1 = o1;
- w2 = o2;
- }
-
- if (w2 > w1) {
- return 1;
- }
- else if (w2 < w1) {
- return -1;
- }
-
- return 0;
- }
-
- static void
- rspamd_symcache_tsort_visit (struct rspamd_symcache *cache,
- struct rspamd_symcache_item *it,
- guint cur_order)
- {
- struct cache_dependency *dep;
- guint i;
-
- if (TSORT_IS_MARKED_PERM (it)) {
- if (cur_order > TSORT_UNMASK (it)) {
- /* Need to recalculate the whole chain */
- it->order = cur_order; /* That also removes all masking */
- }
- else {
- /* We are fine, stop DFS */
- return;
- }
- }
- else if (TSORT_IS_MARKED_TEMP (it)) {
- msg_err_cache ("cyclic dependencies found when checking '%s'!",
- it->symbol);
- return;
- }
-
- TSORT_MARK_TEMP (it);
- msg_debug_cache ("visiting node: %s (%d)", it->symbol, cur_order);
-
- PTR_ARRAY_FOREACH (it->deps, i, dep) {
- msg_debug_cache ("visiting dep: %s (%d)", dep->item->symbol, cur_order + 1);
- rspamd_symcache_tsort_visit (cache, dep->item, cur_order + 1);
- }
-
- it->order = cur_order;
-
- TSORT_MARK_PERM (it);
- }
-
- static void
- rspamd_symcache_resort (struct rspamd_symcache *cache)
- {
- struct symcache_order *ord;
- guint i;
- guint64 total_hits = 0;
- struct rspamd_symcache_item *it;
-
- ord = rspamd_symcache_order_new (cache, cache->filters->len);
-
- for (i = 0; i < cache->filters->len; i ++) {
- it = g_ptr_array_index (cache->filters, i);
- total_hits += it->st->total_hits;
- it->order = 0;
- g_ptr_array_add (ord->d, it);
- }
-
- /* Topological sort, intended to be O(N) but my implementation
- * is not linear (semi-linear usually) as I want to make it as
- * simple as possible.
- * On each stage it does DFS for unseen nodes. In theory, that
- * can be more complicated than linear - O(N^2) for specially
- * crafted data. But I don't care.
- */
- PTR_ARRAY_FOREACH (ord->d, i, it) {
- if (it->order == 0) {
- rspamd_symcache_tsort_visit (cache, it, 1);
- }
- }
-
- /*
- * Now we have all sorted and can do some heuristical sort, keeping
- * topological order invariant
- */
- g_ptr_array_sort_with_data (ord->d, cache_logic_cmp, cache);
- cache->total_hits = total_hits;
-
- if (cache->items_by_order) {
- REF_RELEASE (cache->items_by_order);
- }
-
- cache->items_by_order = ord;
- }
-
- static void
- rspamd_symcache_propagate_dep (struct rspamd_symcache *cache,
- struct rspamd_symcache_item *it,
- struct rspamd_symcache_item *dit)
- {
- const guint *ids;
- guint nids = 0;
-
- msg_debug_cache ("check id propagation for dependency %s from %s",
- it->symbol, dit->symbol);
- ids = rspamd_symcache_get_allowed_settings_ids (cache, dit->symbol, &nids);
-
- /* TODO: merge? */
- if (nids > 0) {
- msg_info_cache ("propagate allowed ids from %s to %s",
- dit->symbol, it->symbol);
-
- rspamd_symcache_set_allowed_settings_ids (cache, it->symbol, ids,
- nids);
- }
-
- ids = rspamd_symcache_get_forbidden_settings_ids (cache, dit->symbol, &nids);
-
- if (nids > 0) {
- msg_info_cache ("propagate forbidden ids from %s to %s",
- dit->symbol, it->symbol);
-
- rspamd_symcache_set_forbidden_settings_ids (cache, it->symbol, ids,
- nids);
- }
- }
-
- static void
- rspamd_symcache_process_dep (struct rspamd_symcache *cache,
- struct rspamd_symcache_item *it,
- struct cache_dependency *dep)
- {
- struct rspamd_symcache_item *dit = NULL, *vdit = NULL;
- struct cache_dependency *rdep;
-
- if (dep->id >= 0) {
- msg_debug_cache ("process real dependency %s on %s", it->symbol, dep->sym);
- dit = rspamd_symcache_find_filter (cache, dep->sym, true);
- }
-
- if (dep->vid >= 0) {
- /* Case of the virtual symbol that depends on another (maybe virtual) symbol */
- vdit = rspamd_symcache_find_filter (cache, dep->sym, false);
-
- if (!vdit) {
- if (dit) {
- msg_err_cache ("cannot add dependency from %s on %s: no dependency symbol registered",
- dep->sym, dit->symbol);
- }
- }
- else {
- msg_debug_cache ("process virtual dependency %s(%d) on %s(%d)", it->symbol,
- dep->vid, vdit->symbol, vdit->id);
- }
- }
- else {
- vdit = dit;
- }
-
- if (dit != NULL) {
- if (!dit->is_filter) {
- /*
- * Check sanity:
- * - filters -> prefilter dependency is OK and always satisfied
- * - postfilter -> (filter, prefilter) dep is ok
- * - idempotent -> (any) dep is OK
- *
- * Otherwise, emit error
- * However, even if everything is fine this dep is useless ¯\_(ツ)_/¯
- */
- gboolean ok_dep = FALSE;
-
- if (it->is_filter) {
- if (dit->is_filter) {
- ok_dep = TRUE;
- }
- else if (dit->type & SYMBOL_TYPE_PREFILTER) {
- ok_dep = TRUE;
- }
- }
- else if (it->type & SYMBOL_TYPE_POSTFILTER) {
- if (dit->type & SYMBOL_TYPE_PREFILTER) {
- ok_dep = TRUE;
- }
- }
- else if (it->type & SYMBOL_TYPE_IDEMPOTENT) {
- if (dit->type & (SYMBOL_TYPE_PREFILTER|SYMBOL_TYPE_POSTFILTER)) {
- ok_dep = TRUE;
- }
- }
- else if (it->type & SYMBOL_TYPE_PREFILTER) {
- if (it->priority < dit->priority) {
- /* Also OK */
- ok_dep = TRUE;
- }
- }
-
- if (!ok_dep) {
- msg_err_cache ("cannot add dependency from %s on %s: invalid symbol types",
- dep->sym, dit->symbol);
-
- return;
- }
- }
- else {
- if (dit->id == it->id) {
- msg_err_cache ("cannot add dependency on self: %s -> %s "
- "(resolved to %s)",
- it->symbol, dep->sym, dit->symbol);
- } else {
- rdep = rspamd_mempool_alloc (cache->static_pool,
- sizeof (*rdep));
-
- rdep->sym = dep->sym;
- rdep->item = it;
- rdep->id = it->id;
- g_assert (dit->rdeps != NULL);
- g_ptr_array_add (dit->rdeps, rdep);
- dep->item = dit;
- dep->id = dit->id;
-
- msg_debug_cache ("add dependency from %d on %d", it->id,
- dit->id);
- }
- }
- }
- else if (dep->id >= 0) {
- msg_err_cache ("cannot find dependency on symbol %s for symbol %s",
- dep->sym, it->symbol);
-
- return;
- }
-
- if (vdit) {
- /* Use virtual symbol to propagate deps */
- rspamd_symcache_propagate_dep (cache, it, vdit);
- }
- }
-
- /* Sort items in logical order */
- static void
- rspamd_symcache_post_init (struct rspamd_symcache *cache)
- {
- struct rspamd_symcache_item *it, *vit;
- struct cache_dependency *dep;
- struct delayed_cache_dependency *ddep;
- struct delayed_cache_condition *dcond;
- GList *cur;
- gint i, j;
-
- cur = cache->delayed_deps;
- while (cur) {
- ddep = cur->data;
-
- vit = rspamd_symcache_find_filter (cache, ddep->from, false);
- it = rspamd_symcache_find_filter (cache, ddep->from, true);
-
- if (it == NULL || vit == NULL) {
- msg_err_cache ("cannot register delayed dependency between %s and %s: "
- "%s is missing", ddep->from, ddep->to, ddep->from);
- }
- else {
- msg_debug_cache ("delayed between %s(%d:%d) -> %s", ddep->from,
- it->id, vit->id, ddep->to);
- rspamd_symcache_add_dependency (cache, it->id, ddep->to, vit != it ?
- vit->id : -1);
- }
-
- cur = g_list_next (cur);
- }
-
- cur = cache->delayed_conditions;
- while (cur) {
- dcond = cur->data;
-
- it = rspamd_symcache_find_filter (cache, dcond->sym, true);
-
- if (it == NULL) {
- msg_err_cache (
- "cannot register delayed condition for %s",
- dcond->sym);
- luaL_unref (dcond->L, LUA_REGISTRYINDEX, dcond->cbref);
- }
- else {
- struct rspamd_symcache_condition *ncond = rspamd_mempool_alloc0 (cache->static_pool,
- sizeof (*ncond));
- ncond->cb = dcond->cbref;
- DL_APPEND (it->specific.normal.conditions, ncond);
- }
-
- cur = g_list_next (cur);
- }
-
- PTR_ARRAY_FOREACH (cache->items_by_id, i, it) {
-
- PTR_ARRAY_FOREACH (it->deps, j, dep) {
- rspamd_symcache_process_dep (cache, it, dep);
- }
-
- if (it->deps) {
- /* Reversed loop to make removal safe */
- for (j = it->deps->len - 1; j >= 0; j--) {
- dep = g_ptr_array_index (it->deps, j);
-
- if (dep->item == NULL) {
- /* Remove useless dep */
- g_ptr_array_remove_index (it->deps, j);
- }
- }
- }
- }
-
- /* Special case for virtual symbols */
- PTR_ARRAY_FOREACH (cache->virtual, i, it) {
-
- PTR_ARRAY_FOREACH (it->deps, j, dep) {
- rspamd_symcache_process_dep (cache, it, dep);
- }
- }
-
- g_ptr_array_sort_with_data (cache->connfilters, prefilters_cmp, cache);
- g_ptr_array_sort_with_data (cache->prefilters, prefilters_cmp, cache);
- g_ptr_array_sort_with_data (cache->postfilters, postfilters_cmp, cache);
- g_ptr_array_sort_with_data (cache->idempotent, postfilters_cmp, cache);
-
- rspamd_symcache_resort (cache);
- }
-
- static gboolean
- rspamd_symcache_load_items (struct rspamd_symcache *cache, const gchar *name)
- {
- struct rspamd_symcache_header *hdr;
- struct stat st;
- struct ucl_parser *parser;
- ucl_object_t *top;
- const ucl_object_t *cur, *elt;
- ucl_object_iter_t it;
- struct rspamd_symcache_item *item, *parent;
- const guchar *p;
- gint fd;
- gpointer map;
-
- fd = open (name, O_RDONLY);
-
- if (fd == -1) {
- msg_info_cache ("cannot open file %s, error %d, %s", name,
- errno, strerror (errno));
- return FALSE;
- }
-
- rspamd_file_lock (fd, FALSE);
-
- if (fstat (fd, &st) == -1) {
- rspamd_file_unlock (fd, FALSE);
- close (fd);
- msg_info_cache ("cannot stat file %s, error %d, %s", name,
- errno, strerror (errno));
- return FALSE;
- }
-
- if (st.st_size < (gint)sizeof (*hdr)) {
- rspamd_file_unlock (fd, FALSE);
- close (fd);
- errno = EINVAL;
- msg_info_cache ("cannot use file %s, error %d, %s", name,
- errno, strerror (errno));
- return FALSE;
- }
-
- map = mmap (NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
-
- if (map == MAP_FAILED) {
- rspamd_file_unlock (fd, FALSE);
- close (fd);
- msg_info_cache ("cannot mmap file %s, error %d, %s", name,
- errno, strerror (errno));
- return FALSE;
- }
-
- hdr = map;
-
- if (memcmp (hdr->magic, rspamd_symcache_magic,
- sizeof (rspamd_symcache_magic)) != 0) {
- msg_info_cache ("cannot use file %s, bad magic", name);
- munmap (map, st.st_size);
- rspamd_file_unlock (fd, FALSE);
- close (fd);
-
- return FALSE;
- }
-
- parser = ucl_parser_new (0);
- p = (const guchar *)(hdr + 1);
-
- if (!ucl_parser_add_chunk (parser, p, st.st_size - sizeof (*hdr))) {
- msg_info_cache ("cannot use file %s, cannot parse: %s", name,
- ucl_parser_get_error (parser));
- munmap (map, st.st_size);
- ucl_parser_free (parser);
- rspamd_file_unlock (fd, FALSE);
- close (fd);
-
- return FALSE;
- }
-
- top = ucl_parser_get_object (parser);
- munmap (map, st.st_size);
- rspamd_file_unlock (fd, FALSE);
- close (fd);
- ucl_parser_free (parser);
-
- if (top == NULL || ucl_object_type (top) != UCL_OBJECT) {
- msg_info_cache ("cannot use file %s, bad object", name);
- ucl_object_unref (top);
- return FALSE;
- }
-
- it = ucl_object_iterate_new (top);
-
- while ((cur = ucl_object_iterate_safe (it, true))) {
- item = g_hash_table_lookup (cache->items_by_symbol, ucl_object_key (cur));
-
- if (item) {
- /* Copy saved info */
- /*
- * XXX: don't save or load weight, it should be obtained from the
- * metric
- */
- #if 0
- elt = ucl_object_lookup (cur, "weight");
-
- if (elt) {
- w = ucl_object_todouble (elt);
- if (w != 0) {
- item->weight = w;
- }
- }
- #endif
- elt = ucl_object_lookup (cur, "time");
- if (elt) {
- item->st->avg_time = ucl_object_todouble (elt);
- }
-
- elt = ucl_object_lookup (cur, "count");
- if (elt) {
- item->st->total_hits = ucl_object_toint (elt);
- item->last_count = item->st->total_hits;
- }
-
- elt = ucl_object_lookup (cur, "frequency");
- if (elt && ucl_object_type (elt) == UCL_OBJECT) {
- const ucl_object_t *freq_elt;
-
- freq_elt = ucl_object_lookup (elt, "avg");
-
- if (freq_elt) {
- item->st->avg_frequency = ucl_object_todouble (freq_elt);
- }
- freq_elt = ucl_object_lookup (elt, "stddev");
-
- if (freq_elt) {
- item->st->stddev_frequency = ucl_object_todouble (freq_elt);
- }
- }
-
- if (item->is_virtual && !(item->type & SYMBOL_TYPE_GHOST)) {
- g_assert (item->specific.virtual.parent < (gint)cache->items_by_id->len);
- parent = g_ptr_array_index (cache->items_by_id,
- item->specific.virtual.parent);
- item->specific.virtual.parent_item = parent;
-
- if (parent->st->weight < item->st->weight) {
- parent->st->weight = item->st->weight;
- }
-
- /*
- * We maintain avg_time for virtual symbols equal to the
- * parent item avg_time
- */
- item->st->avg_time = parent->st->avg_time;
- }
-
- cache->total_weight += fabs (item->st->weight);
- cache->total_hits += item->st->total_hits;
- }
- }
-
- ucl_object_iterate_free (it);
- ucl_object_unref (top);
-
- return TRUE;
- }
-
- #define ROUND_DOUBLE(x) (floor((x) * 100.0) / 100.0)
-
- static gboolean
- rspamd_symcache_save_items (struct rspamd_symcache *cache, const gchar *name)
- {
- struct rspamd_symcache_header hdr;
- ucl_object_t *top, *elt, *freq;
- GHashTableIter it;
- struct rspamd_symcache_item *item;
- struct ucl_emitter_functions *efunc;
- gpointer k, v;
- gint fd;
- FILE *fp;
- bool ret;
- gchar path[PATH_MAX];
-
- rspamd_snprintf (path, sizeof (path), "%s.new", name);
-
- fd = open (path, O_CREAT | O_WRONLY | O_EXCL, 00644);
-
- if (fd == -1) {
- if (errno == EEXIST) {
- /* Some other process is already writing data, give up silently */
- return TRUE;
- }
-
- msg_err_cache ("cannot open file %s, error %d, %s", path,
- errno, strerror (errno));
- return FALSE;
- }
-
- rspamd_file_lock (fd, FALSE);
- fp = fdopen (fd, "w");
-
- memset (&hdr, 0, sizeof (hdr));
- memcpy (hdr.magic, rspamd_symcache_magic,
- sizeof (rspamd_symcache_magic));
-
- if (fwrite (&hdr, sizeof (hdr), 1, fp) == -1) {
- msg_err_cache ("cannot write to file %s, error %d, %s", path,
- errno, strerror (errno));
- rspamd_file_unlock (fd, FALSE);
- fclose (fp);
-
- return FALSE;
- }
-
- top = ucl_object_typed_new (UCL_OBJECT);
- g_hash_table_iter_init (&it, cache->items_by_symbol);
-
- while (g_hash_table_iter_next (&it, &k, &v)) {
- item = v;
- elt = ucl_object_typed_new (UCL_OBJECT);
- ucl_object_insert_key (elt,
- ucl_object_fromdouble (ROUND_DOUBLE (item->st->weight)),
- "weight", 0, false);
- ucl_object_insert_key (elt,
- ucl_object_fromdouble (ROUND_DOUBLE (item->st->time_counter.mean)),
- "time", 0, false);
- ucl_object_insert_key (elt, ucl_object_fromint (item->st->total_hits),
- "count", 0, false);
-
- freq = ucl_object_typed_new (UCL_OBJECT);
- ucl_object_insert_key (freq,
- ucl_object_fromdouble (ROUND_DOUBLE (item->st->frequency_counter.mean)),
- "avg", 0, false);
- ucl_object_insert_key (freq,
- ucl_object_fromdouble (ROUND_DOUBLE (item->st->frequency_counter.stddev)),
- "stddev", 0, false);
- ucl_object_insert_key (elt, freq, "frequency", 0, false);
-
- ucl_object_insert_key (top, elt, k, 0, false);
- }
-
- efunc = ucl_object_emit_file_funcs (fp);
- ret = ucl_object_emit_full (top, UCL_EMIT_JSON_COMPACT, efunc, NULL);
- ucl_object_emit_funcs_free (efunc);
- ucl_object_unref (top);
- rspamd_file_unlock (fd, FALSE);
- fclose (fp);
-
- if (rename (path, name) == -1) {
- msg_err_cache ("cannot rename %s -> %s, error %d, %s", path, name,
- errno, strerror (errno));
- (void)unlink (path);
- ret = FALSE;
- }
-
- return ret;
- }
-
- #undef ROUND_DOUBLE
-
- gint
- rspamd_symcache_add_symbol (struct rspamd_symcache *cache,
- const gchar *name,
- gint priority,
- symbol_func_t func,
- gpointer user_data,
- enum rspamd_symbol_type type,
- gint parent)
- {
- struct rspamd_symcache_item *item = NULL;
- const gchar *type_str = "normal";
-
- g_assert (cache != NULL);
-
- if (name == NULL && !(type & SYMBOL_TYPE_CALLBACK)) {
- msg_warn_cache ("no name for non-callback symbol!");
- }
- else if ((type & SYMBOL_TYPE_VIRTUAL & (~SYMBOL_TYPE_GHOST)) && parent == -1) {
- msg_warn_cache ("no parent symbol is associated with virtual symbol %s",
- name);
- }
-
- if (name != NULL && !(type & SYMBOL_TYPE_CALLBACK)) {
- struct rspamd_symcache_item *existing;
-
- if (strcspn (name, " \t\n\r") != strlen (name)) {
- msg_warn_cache ("bogus characters in symbol name: \"%s\"",
- name);
- }
-
- if ((existing = g_hash_table_lookup (cache->items_by_symbol, name)) != NULL) {
-
- if (existing->type & SYMBOL_TYPE_GHOST) {
- /*
- * Complicated part:
- * - we need to remove the existing ghost symbol
- * - we need to cleanup containers:
- * - symbols hash
- * - specific array
- * - items_by_it
- * - decrement used_items
- */
- msg_info_cache ("duplicate ghost symbol %s is removed", name);
-
- if (existing->container) {
- g_ptr_array_remove (existing->container, existing);
- }
-
- g_ptr_array_remove (cache->items_by_id, existing->container);
- cache->used_items --;
- g_hash_table_remove (cache->items_by_symbol, name);
- /*
- * Here can be memory leak, but we assume that ghost symbols
- * are also virtual
- */
- }
- else {
- msg_err_cache ("skip duplicate symbol registration for %s", name);
- return -1;
- }
- }
- }
-
- if (type & (SYMBOL_TYPE_CLASSIFIER|SYMBOL_TYPE_CALLBACK|
- SYMBOL_TYPE_PREFILTER|SYMBOL_TYPE_POSTFILTER|
- SYMBOL_TYPE_IDEMPOTENT|SYMBOL_TYPE_GHOST)) {
- type |= SYMBOL_TYPE_NOSTAT;
- }
-
- item = rspamd_mempool_alloc0 (cache->static_pool,
- sizeof (struct rspamd_symcache_item));
- item->st = rspamd_mempool_alloc0_shared (cache->static_pool,
- sizeof (*item->st));
- item->enabled = TRUE;
-
- /*
- * We do not share cd to skip locking, instead we'll just calculate it on
- * save or accumulate
- */
- item->cd = rspamd_mempool_alloc0 (cache->static_pool,
- sizeof (struct rspamd_counter_data));
- item->priority = priority;
- item->type = type;
-
- if ((type & SYMBOL_TYPE_FINE) && item->priority == 0) {
- /* Make priority for negative weighted symbols */
- item->priority = 1;
- }
-
- if (func) {
- /* Non-virtual symbol */
- g_assert (parent == -1);
-
- if (item->type & SYMBOL_TYPE_PREFILTER) {
- type_str = "prefilter";
- g_ptr_array_add (cache->prefilters, item);
- item->container = cache->prefilters;
- }
- else if (item->type & SYMBOL_TYPE_IDEMPOTENT) {
- type_str = "idempotent";
- g_ptr_array_add (cache->idempotent, item);
- item->container = cache->idempotent;
- }
- else if (item->type & SYMBOL_TYPE_POSTFILTER) {
- type_str = "postfilter";
- g_ptr_array_add (cache->postfilters, item);
- item->container = cache->postfilters;
- }
- else if (item->type & SYMBOL_TYPE_CONNFILTER) {
- type_str = "connfilter";
- g_ptr_array_add (cache->connfilters, item);
- item->container = cache->connfilters;
- }
- else {
- item->is_filter = TRUE;
- g_ptr_array_add (cache->filters, item);
- item->container = cache->filters;
- }
-
- item->id = cache->items_by_id->len;
- g_ptr_array_add (cache->items_by_id, item);
-
- item->specific.normal.func = func;
- item->specific.normal.user_data = user_data;
- item->specific.normal.conditions = NULL;
- }
- else {
- /*
- * Three possibilities here when no function is specified:
- * - virtual symbol (beware of ghosts!)
- * - classifier symbol
- * - composite symbol
- */
- if (item->type & SYMBOL_TYPE_COMPOSITE) {
- item->specific.normal.conditions = NULL;
- item->specific.normal.user_data = user_data;
- g_assert (user_data != NULL);
- g_ptr_array_add (cache->composites, item);
-
- item->id = cache->items_by_id->len;
- g_ptr_array_add (cache->items_by_id, item);
- item->container = cache->composites;
- type_str = "composite";
- }
- else if (item->type & SYMBOL_TYPE_CLASSIFIER) {
- /* Treat it as normal symbol to allow enable/disable */
- item->id = cache->items_by_id->len;
- g_ptr_array_add (cache->items_by_id, item);
-
- item->is_filter = TRUE;
- item->specific.normal.func = NULL;
- item->specific.normal.user_data = NULL;
- item->specific.normal.conditions = NULL;
- type_str = "classifier";
- }
- else {
- item->is_virtual = TRUE;
- item->specific.virtual.parent = parent;
- item->specific.virtual.parent_item =
- g_ptr_array_index (cache->items_by_id, parent);
- item->id = cache->virtual->len;
- g_ptr_array_add (cache->virtual, item);
- item->container = cache->virtual;
- /* Not added to items_by_id, handled by parent */
- type_str = "virtual";
- }
- }
-
- cache->used_items ++;
- cache->id ++;
-
- if (!(item->type &
- (SYMBOL_TYPE_IDEMPOTENT|SYMBOL_TYPE_NOSTAT|SYMBOL_TYPE_CLASSIFIER))) {
- if (name != NULL) {
- cache->cksum = t1ha (name, strlen (name),
- cache->cksum);
- } else {
- cache->cksum = t1ha (&item->id, sizeof (item->id),
- cache->cksum);
- }
-
- cache->stats_symbols_count ++;
- }
-
- if (name != NULL) {
- item->symbol = rspamd_mempool_strdup (cache->static_pool, name);
- msg_debug_cache ("used items: %d, added symbol: %s, %d; symbol type: %s",
- cache->used_items, name, item->id, type_str);
- } else {
- g_assert (func != NULL);
- msg_debug_cache ("used items: %d, added unnamed symbol: %d; symbol type: %s",
- cache->used_items, item->id, type_str);
- }
-
- item->deps = g_ptr_array_new ();
- item->rdeps = g_ptr_array_new ();
- item->type_descr = type_str;
- rspamd_mempool_add_destructor (cache->static_pool,
- rspamd_ptr_array_free_hard, item->deps);
- rspamd_mempool_add_destructor (cache->static_pool,
- rspamd_ptr_array_free_hard, item->rdeps);
-
- if (name != NULL) {
- g_hash_table_insert (cache->items_by_symbol, item->symbol, item);
- }
-
- return item->id;
- }
-
- void
- rspamd_symcache_set_peak_callback (struct rspamd_symcache *cache,
- gint cbref)
- {
- g_assert (cache != NULL);
-
- if (cache->peak_cb != -1) {
- luaL_unref (cache->cfg->lua_state, LUA_REGISTRYINDEX,
- cache->peak_cb);
- }
-
- cache->peak_cb = cbref;
- msg_info_cache ("registered peak callback");
- }
-
- gboolean
- rspamd_symcache_add_condition_delayed (struct rspamd_symcache *cache,
- const gchar *sym, lua_State *L, gint cbref)
- {
- struct delayed_cache_condition *ncond;
-
- g_assert (cache != NULL);
- g_assert (sym != NULL);
-
- ncond = g_malloc0 (sizeof (*ncond));
- ncond->sym = g_strdup (sym);
- ncond->cbref = cbref;
- ncond->L = L;
- cache->id ++;
-
- cache->delayed_conditions = g_list_prepend (cache->delayed_conditions, ncond);
-
- return TRUE;
- }
-
- void
- rspamd_symcache_save (struct rspamd_symcache *cache)
- {
- if (cache != NULL) {
-
- if (cache->cfg->cache_filename) {
- /* Try to sync values to the disk */
- if (!rspamd_symcache_save_items (cache,
- cache->cfg->cache_filename)) {
- msg_err_cache ("cannot save cache data to %s: %s",
- cache->cfg->cache_filename, strerror (errno));
- }
- }
- }
- }
-
- void
- rspamd_symcache_destroy (struct rspamd_symcache *cache)
- {
- GList *cur;
- struct delayed_cache_dependency *ddep;
- struct delayed_cache_condition *dcond;
-
- if (cache != NULL) {
- if (cache->delayed_deps) {
- cur = cache->delayed_deps;
-
- while (cur) {
- ddep = cur->data;
- g_free (ddep->from);
- g_free (ddep->to);
- g_free (ddep);
- cur = g_list_next (cur);
- }
-
- g_list_free (cache->delayed_deps);
- }
-
- if (cache->delayed_conditions) {
- cur = cache->delayed_conditions;
-
- while (cur) {
- dcond = cur->data;
- g_free (dcond->sym);
- g_free (dcond);
- cur = g_list_next (cur);
- }
-
- g_list_free (cache->delayed_conditions);
- }
-
- g_hash_table_destroy (cache->items_by_symbol);
- g_ptr_array_free (cache->items_by_id, TRUE);
- rspamd_mempool_delete (cache->static_pool);
- g_ptr_array_free (cache->connfilters, TRUE);
- g_ptr_array_free (cache->prefilters, TRUE);
- g_ptr_array_free (cache->filters, TRUE);
- g_ptr_array_free (cache->postfilters, TRUE);
- g_ptr_array_free (cache->idempotent, TRUE);
- g_ptr_array_free (cache->composites, TRUE);
- g_ptr_array_free (cache->virtual, TRUE);
- REF_RELEASE (cache->items_by_order);
-
- if (cache->peak_cb != -1) {
- luaL_unref (cache->cfg->lua_state, LUA_REGISTRYINDEX, cache->peak_cb);
- }
-
- g_free (cache);
- }
- }
-
- struct rspamd_symcache*
- rspamd_symcache_new (struct rspamd_config *cfg)
- {
- struct rspamd_symcache *cache;
-
- cache = g_malloc0 (sizeof (struct rspamd_symcache));
- cache->static_pool =
- rspamd_mempool_new (rspamd_mempool_suggest_size (), "symcache", 0);
- cache->items_by_symbol = g_hash_table_new (rspamd_str_hash,
- rspamd_str_equal);
- cache->items_by_id = g_ptr_array_new ();
- cache->connfilters = g_ptr_array_new ();
- cache->prefilters = g_ptr_array_new ();
- cache->filters = g_ptr_array_new ();
- cache->postfilters = g_ptr_array_new ();
- cache->idempotent = g_ptr_array_new ();
- cache->composites = g_ptr_array_new ();
- cache->virtual = g_ptr_array_new ();
- cache->reload_time = cfg->cache_reload_time;
- cache->total_hits = 1;
- cache->total_weight = 1.0;
- cache->cfg = cfg;
- cache->cksum = 0xdeadbabe;
- cache->peak_cb = -1;
- cache->id = (guint)rspamd_random_uint64_fast ();
-
- return cache;
- }
-
- static void
- rspamd_symcache_metric_connect_cb (gpointer k, gpointer v, gpointer ud)
- {
- struct rspamd_symcache *cache = (struct rspamd_symcache *)ud;
- const gchar *sym = k;
- struct rspamd_symbol *s = (struct rspamd_symbol *)v;
- gdouble weight;
- struct rspamd_symcache_item *item;
-
- weight = *s->weight_ptr;
- item = g_hash_table_lookup (cache->items_by_symbol, sym);
-
- if (item) {
- item->st->weight = weight;
- s->cache_item = item;
- }
- }
-
- gboolean
- rspamd_symcache_init (struct rspamd_symcache *cache)
- {
- gboolean res = TRUE;
-
- g_assert (cache != NULL);
-
- cache->reload_time = cache->cfg->cache_reload_time;
-
- if (cache->cfg->cache_filename != NULL) {
- res = rspamd_symcache_load_items (cache, cache->cfg->cache_filename);
- }
-
- rspamd_symcache_post_init (cache);
-
- /* Connect metric symbols with symcache symbols */
- if (cache->cfg->symbols) {
- g_hash_table_foreach (cache->cfg->symbols,
- rspamd_symcache_metric_connect_cb,
- cache);
- }
-
- return res;
- }
-
-
- static void
- rspamd_symcache_validate_cb (gpointer k, gpointer v, gpointer ud)
- {
- struct rspamd_symcache_item *item = v, *parent;
- struct rspamd_config *cfg;
- struct rspamd_symcache *cache = (struct rspamd_symcache *)ud;
- struct rspamd_symbol *s;
- gboolean skipped, ghost;
- gint p1, p2;
-
- ghost = item->st->weight == 0 ? TRUE : FALSE;
- cfg = cache->cfg;
-
- /* Check whether this item is skipped */
- skipped = !ghost;
- g_assert (cfg != NULL);
-
- if ((item->type &
- (SYMBOL_TYPE_NORMAL|SYMBOL_TYPE_VIRTUAL|SYMBOL_TYPE_COMPOSITE|SYMBOL_TYPE_CLASSIFIER))
- && g_hash_table_lookup (cfg->symbols, item->symbol) == NULL) {
-
- if (cfg->unknown_weight != 0) {
-
- skipped = FALSE;
- item->st->weight = cfg->unknown_weight;
- s = rspamd_mempool_alloc0 (cache->static_pool,
- sizeof (*s));
- s->name = item->symbol;
- s->weight_ptr = &item->st->weight;
- g_hash_table_insert (cfg->symbols, item->symbol, s);
-
- msg_info_cache ("adding unknown symbol %s", item->symbol);
- ghost = FALSE;
- }
- else {
- skipped = TRUE;
- }
- }
- else {
- skipped = FALSE;
- }
-
- if (!ghost && skipped) {
- if (!(item->type & SYMBOL_TYPE_SKIPPED)) {
- item->type |= SYMBOL_TYPE_SKIPPED;
- msg_warn_cache ("symbol %s has no score registered, skip its check",
- item->symbol);
- }
- }
-
- if (ghost) {
- msg_debug_cache ("symbol %s is registered as ghost symbol, it won't be inserted "
- "to any metric", item->symbol);
- }
-
- if (item->st->weight < 0 && item->priority == 0) {
- item->priority ++;
- }
-
- if (item->is_virtual) {
- if (!(item->type & SYMBOL_TYPE_GHOST)) {
- g_assert (item->specific.virtual.parent != -1);
- g_assert (item->specific.virtual.parent < (gint) cache->items_by_id->len);
- parent = g_ptr_array_index (cache->items_by_id,
- item->specific.virtual.parent);
- item->specific.virtual.parent_item = parent;
-
- if (fabs (parent->st->weight) < fabs (item->st->weight)) {
- parent->st->weight = item->st->weight;
- }
-
- p1 = abs (item->priority);
- p2 = abs (parent->priority);
-
- if (p1 != p2) {
- parent->priority = MAX (p1, p2);
- item->priority = parent->priority;
- }
- }
- }
-
- cache->total_weight += fabs (item->st->weight);
- }
-
- gboolean
- rspamd_symcache_validate (struct rspamd_symcache *cache,
- struct rspamd_config *cfg,
- gboolean strict)
- {
- struct rspamd_symcache_item *item;
- GHashTableIter it;
- gpointer k, v;
- struct rspamd_symbol *sym_def;
- gboolean ignore_symbol = FALSE, ret = TRUE;
-
- if (cache == NULL) {
- msg_err ("empty cache is invalid");
- return FALSE;
- }
-
-
- g_hash_table_foreach (cache->items_by_symbol,
- rspamd_symcache_validate_cb,
- cache);
- /* Now check each metric item and find corresponding symbol in a cache */
- g_hash_table_iter_init (&it, cfg->symbols);
-
- while (g_hash_table_iter_next (&it, &k, &v)) {
- ignore_symbol = FALSE;
- sym_def = v;
-
- if (sym_def && (sym_def->flags &
- (RSPAMD_SYMBOL_FLAG_IGNORE_METRIC|RSPAMD_SYMBOL_FLAG_DISABLED))) {
- ignore_symbol = TRUE;
- }
-
- if (!ignore_symbol) {
- item = g_hash_table_lookup (cache->items_by_symbol, k);
-
- if (item == NULL) {
- msg_warn_cache (
- "symbol '%s' has its score defined but there is no "
- "corresponding rule registered",
- k);
- if (strict) {
- ret = FALSE;
- }
- }
- }
- else if (sym_def->flags & RSPAMD_SYMBOL_FLAG_DISABLED) {
- item = g_hash_table_lookup (cache->items_by_symbol, k);
-
- if (item) {
- item->enabled = FALSE;
- }
- }
- }
-
- return ret;
- }
-
- /* Return true if metric has score that is more than spam score for it */
- static gboolean
- rspamd_symcache_metric_limit (struct rspamd_task *task,
- struct cache_savepoint *cp)
- {
- struct rspamd_scan_result *res;
- double ms;
-
- if (task->flags & RSPAMD_TASK_FLAG_PASS_ALL) {
- return FALSE;
- }
-
- if (cp->lim == 0.0) {
- res = task->result;
-
- if (res) {
- ms = rspamd_task_get_required_score (task, res);
-
- if (!isnan (ms) && cp->lim < ms) {
- cp->rs = res;
- cp->lim = ms;
- }
- }
- }
-
- if (cp->rs) {
-
- if (cp->rs->score > cp->lim) {
- return TRUE;
- }
- }
- else {
- /* No reject score define, always check all rules */
- cp->lim = -1;
- }
-
- return FALSE;
- }
-
- static inline gboolean
- rspamd_symcache_check_id_list (const struct rspamd_symcache_id_list *ls, guint32 id)
- {
- guint i;
-
- if (ls->dyn.e == -1) {
- guint *res = bsearch (&id, ls->dyn.n, ls->dyn.len, sizeof (guint32),
- rspamd_id_cmp);
-
- if (res) {
- return TRUE;
- }
- }
- else {
- for (i = 0; i < G_N_ELEMENTS (ls->st); i ++) {
- if (ls->st[i] == id) {
- return TRUE;
- }
- else if (ls->st[i] == 0) {
- return FALSE;
- }
- }
- }
-
- return FALSE;
- }
-
- gboolean
- rspamd_symcache_is_item_allowed (struct rspamd_task *task,
- struct rspamd_symcache_item *item,
- gboolean exec_only)
- {
- const gchar *what = "execution";
-
- if (!exec_only) {
- what = "symbol insertion";
- }
-
- /* Static checks */
- if (!item->enabled ||
- (RSPAMD_TASK_IS_EMPTY (task) && !(item->type & SYMBOL_TYPE_EMPTY)) ||
- (item->type & SYMBOL_TYPE_MIME_ONLY && !RSPAMD_TASK_IS_MIME(task))) {
-
- if (!item->enabled) {
- msg_debug_cache_task ("skipping %s of %s as it is permanently disabled; symbol type=%s",
- what, item->symbol, item->type_descr);
-
- return FALSE;
- }
- else {
- /*
- * Exclude virtual symbols
- */
- if (exec_only) {
- msg_debug_cache_task ("skipping check of %s as it cannot be "
- "executed for this task type; symbol type=%s",
- item->symbol, item->type_descr);
-
- return FALSE;
- }
- }
- }
-
- /* Settings checks */
- if (task->settings_elt != 0) {
- guint32 id = task->settings_elt->id;
-
- if (item->forbidden_ids.st[0] != 0 &&
- rspamd_symcache_check_id_list (&item->forbidden_ids,
- id)) {
- msg_debug_cache_task ("deny %s of %s as it is forbidden for "
- "settings id %ud; symbol type=%s",
- what,
- item->symbol,
- id,
- item->type_descr);
-
- return FALSE;
- }
-
- if (!(item->type & SYMBOL_TYPE_EXPLICIT_DISABLE)) {
- if (item->allowed_ids.st[0] == 0 ||
- !rspamd_symcache_check_id_list (&item->allowed_ids,
- id)) {
-
- if (task->settings_elt->policy == RSPAMD_SETTINGS_POLICY_IMPLICIT_ALLOW) {
- msg_debug_cache_task ("allow execution of %s settings id %ud "
- "allows implicit execution of the symbols;"
- "symbol type=%s",
- item->symbol,
- id,
- item->type_descr);
-
- return TRUE;
- }
-
- if (exec_only) {
- /*
- * Special case if any of our virtual children are enabled
- */
- if (rspamd_symcache_check_id_list (&item->exec_only_ids, id)) {
- return TRUE;
- }
- }
-
- msg_debug_cache_task ("deny %s of %s as it is not listed "
- "as allowed for settings id %ud; symbol type=%s",
- what,
- item->symbol,
- id,
- item->type_descr);
- return FALSE;
- }
- }
- else {
- msg_debug_cache_task ("allow %s of %s for "
- "settings id %ud as it can be only disabled explicitly;"
- " symbol type=%s",
- what,
- item->symbol,
- id,
- item->type_descr);
- }
- }
- else if (item->type & SYMBOL_TYPE_EXPLICIT_ENABLE) {
- msg_debug_cache_task ("deny %s of %s as it must be explicitly enabled; symbol type=%s",
- what,
- item->symbol,
- item->type_descr);
- return FALSE;
- }
-
- /* Allow all symbols with no settings id */
- return TRUE;
- }
-
- static gboolean
- rspamd_symcache_check_symbol (struct rspamd_task *task,
- struct rspamd_symcache *cache,
- struct rspamd_symcache_item *item,
- struct cache_savepoint *checkpoint)
- {
- struct rspamd_task **ptask;
- lua_State *L;
- gboolean check = TRUE;
- struct rspamd_symcache_dynamic_item *dyn_item =
- rspamd_symcache_get_dynamic (checkpoint, item);
-
- if (item->type & (SYMBOL_TYPE_CLASSIFIER|SYMBOL_TYPE_COMPOSITE)) {
- /* Classifiers are special :( */
- return TRUE;
- }
-
- if (rspamd_session_blocked (task->s)) {
- /*
- * We cannot add new events as session is either destroyed or
- * being cleaned up.
- */
- return TRUE;
- }
-
- g_assert (!item->is_virtual);
- g_assert (item->specific.normal.func != NULL);
- if (CHECK_START_BIT (checkpoint, dyn_item)) {
- /*
- * This can actually happen when deps span over different layers
- */
- return CHECK_FINISH_BIT (checkpoint, dyn_item);
- }
-
- /* Check has been started */
- SET_START_BIT (checkpoint, dyn_item);
-
- if (!rspamd_symcache_is_item_allowed (task, item, TRUE)) {
- check = FALSE;
- }
- else if (item->specific.normal.conditions) {
- struct rspamd_symcache_condition *cur_cond;
-
- DL_FOREACH (item->specific.normal.conditions, cur_cond) {
- /* We also executes condition callback to check if we need this symbol */
- L = task->cfg->lua_state;
- lua_rawgeti (L, LUA_REGISTRYINDEX, cur_cond->cb);
- ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
- rspamd_lua_setclass (L, "rspamd{task}", -1);
- *ptask = task;
-
- if (lua_pcall (L, 1, 1, 0) != 0) {
- msg_info_task ("call to condition for %s failed: %s",
- item->symbol, lua_tostring (L, -1));
- lua_pop (L, 1);
- }
- else {
- check = lua_toboolean (L, -1);
- lua_pop (L, 1);
- }
-
- if (!check) {
- break;
- }
- }
-
- if (!check) {
- msg_debug_cache_task ("skipping check of %s as its start condition is false; "
- "symbol type = %s",
- item->symbol, item->type_descr);
- }
- }
-
- if (check) {
- msg_debug_cache_task ("execute %s, %d; symbol type = %s", item->symbol,
- item->id, item->type_descr);
-
- if (checkpoint->profile) {
- ev_now_update_if_cheap (task->event_loop);
- dyn_item->start_msec = (ev_now (task->event_loop) -
- checkpoint->profile_start) * 1e3;
- }
-
- dyn_item->async_events = 0;
- checkpoint->cur_item = item;
- checkpoint->items_inflight ++;
- /* Callback now must finalize itself */
- item->specific.normal.func (task, item, item->specific.normal.user_data);
- checkpoint->cur_item = NULL;
-
- if (checkpoint->items_inflight == 0) {
-
- return TRUE;
- }
-
- if (dyn_item->async_events == 0 && !CHECK_FINISH_BIT (checkpoint, dyn_item)) {
- msg_err_cache ("critical error: item %s has no async events pending, "
- "but it is not finalised", item->symbol);
- g_assert_not_reached ();
- }
-
- return FALSE;
- }
- else {
- SET_FINISH_BIT (checkpoint, dyn_item);
- }
-
- return TRUE;
- }
-
- static gboolean
- rspamd_symcache_check_deps (struct rspamd_task *task,
- struct rspamd_symcache *cache,
- struct rspamd_symcache_item *item,
- struct cache_savepoint *checkpoint,
- guint recursion,
- gboolean check_only)
- {
- struct cache_dependency *dep;
- guint i;
- gboolean ret = TRUE;
- static const guint max_recursion = 20;
- struct rspamd_symcache_dynamic_item *dyn_item;
-
- if (recursion > max_recursion) {
- msg_err_task ("cyclic dependencies: maximum check level %ud exceed when "
- "checking dependencies for %s", max_recursion, item->symbol);
-
- return TRUE;
- }
-
- if (item->deps != NULL && item->deps->len > 0) {
- for (i = 0; i < item->deps->len; i ++) {
- dep = g_ptr_array_index (item->deps, i);
-
- if (dep->item == NULL) {
- /* Assume invalid deps as done */
- msg_debug_cache_task ("symbol %d(%s) has invalid dependencies on %d(%s)",
- item->id, item->symbol, dep->id, dep->sym);
- continue;
- }
-
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, dep->item);
-
- if (!CHECK_FINISH_BIT (checkpoint, dyn_item)) {
- if (!CHECK_START_BIT (checkpoint, dyn_item)) {
- /* Not started */
- if (!check_only) {
- if (!rspamd_symcache_check_deps (task, cache,
- dep->item,
- checkpoint,
- recursion + 1,
- check_only)) {
-
- ret = FALSE;
- msg_debug_cache_task ("delayed dependency %d(%s) for "
- "symbol %d(%s)",
- dep->id, dep->sym, item->id, item->symbol);
- }
- else if (!rspamd_symcache_check_symbol (task, cache,
- dep->item,
- checkpoint)) {
- /* Now started, but has events pending */
- ret = FALSE;
- msg_debug_cache_task ("started check of %d(%s) symbol "
- "as dep for "
- "%d(%s)",
- dep->id, dep->sym, item->id, item->symbol);
- }
- else {
- msg_debug_cache_task ("dependency %d(%s) for symbol %d(%s) is "
- "already processed",
- dep->id, dep->sym, item->id, item->symbol);
- }
- }
- else {
- msg_debug_cache_task ("dependency %d(%s) for symbol %d(%s) "
- "cannot be started now",
- dep->id, dep->sym,
- item->id, item->symbol);
- ret = FALSE;
- }
- }
- else {
- /* Started but not finished */
- msg_debug_cache_task ("dependency %d(%s) for symbol %d(%s) is "
- "still executing",
- dep->id, dep->sym,
- item->id, item->symbol);
- ret = FALSE;
- }
- }
- else {
- msg_debug_cache_task ("dependency %d(%s) for symbol %d(%s) is already "
- "checked",
- dep->id, dep->sym,
- item->id, item->symbol);
- }
- }
- }
-
- return ret;
- }
-
- static struct cache_savepoint *
- rspamd_symcache_make_checkpoint (struct rspamd_task *task,
- struct rspamd_symcache *cache)
- {
- struct cache_savepoint *checkpoint;
-
- if (cache->items_by_order->id != cache->id) {
- /*
- * Cache has been modified, need to resort it
- */
- msg_info_cache ("symbols cache has been modified since last check:"
- " old id: %ud, new id: %ud",
- cache->items_by_order->id, cache->id);
- rspamd_symcache_resort (cache);
- }
-
- checkpoint = rspamd_mempool_alloc0 (task->task_pool,
- sizeof (*checkpoint) +
- sizeof (struct rspamd_symcache_dynamic_item) * cache->items_by_id->len);
-
- g_assert (cache->items_by_order != NULL);
- checkpoint->version = cache->items_by_order->d->len;
- checkpoint->order = cache->items_by_order;
- REF_RETAIN (checkpoint->order);
- rspamd_mempool_add_destructor (task->task_pool,
- rspamd_symcache_order_unref, checkpoint->order);
-
- /* Calculate profile probability */
- ev_now_update_if_cheap (task->event_loop);
- ev_tstamp now = ev_now (task->event_loop);
- checkpoint->profile_start = now;
-
- if ((cache->last_profile == 0.0 || now > cache->last_profile + PROFILE_MAX_TIME) ||
- (task->msg.len >= PROFILE_MESSAGE_SIZE_THRESHOLD) ||
- (rspamd_random_double_fast () >= (1 - PROFILE_PROBABILITY))) {
- msg_debug_cache_task ("enable profiling of symbols for task");
- checkpoint->profile = TRUE;
- cache->last_profile = now;
- }
-
- task->checkpoint = checkpoint;
-
- return checkpoint;
- }
-
- gboolean
- rspamd_symcache_process_settings (struct rspamd_task *task,
- struct rspamd_symcache *cache)
- {
- const ucl_object_t *wl, *cur, *disabled, *enabled;
- struct rspamd_symbols_group *gr;
- GHashTableIter gr_it;
- ucl_object_iter_t it = NULL;
- gboolean already_disabled = FALSE;
- gpointer k, v;
-
- wl = ucl_object_lookup (task->settings, "whitelist");
-
- if (wl != NULL) {
- msg_info_task ("task is whitelisted");
- task->flags |= RSPAMD_TASK_FLAG_SKIP;
- return TRUE;
- }
-
- enabled = ucl_object_lookup (task->settings, "symbols_enabled");
-
- if (enabled) {
- /* Disable all symbols but selected */
- rspamd_symcache_disable_all_symbols (task, cache,
- SYMBOL_TYPE_EXPLICIT_DISABLE);
- already_disabled = TRUE;
- it = NULL;
-
- while ((cur = ucl_iterate_object (enabled, &it, true)) != NULL) {
- rspamd_symcache_enable_symbol_checkpoint (task, cache,
- ucl_object_tostring (cur));
- }
- }
-
- /* Enable groups of symbols */
- enabled = ucl_object_lookup (task->settings, "groups_enabled");
-
- if (enabled) {
- it = NULL;
-
- if (!already_disabled) {
- rspamd_symcache_disable_all_symbols (task, cache,
- SYMBOL_TYPE_EXPLICIT_DISABLE);
- }
-
- while ((cur = ucl_iterate_object (enabled, &it, true)) != NULL) {
- if (ucl_object_type (cur) == UCL_STRING) {
- gr = g_hash_table_lookup (task->cfg->groups,
- ucl_object_tostring (cur));
-
- if (gr) {
- g_hash_table_iter_init (&gr_it, gr->symbols);
-
- while (g_hash_table_iter_next (&gr_it, &k, &v)) {
- rspamd_symcache_enable_symbol_checkpoint (task, cache, k);
- }
- }
- }
- }
- }
-
- disabled = ucl_object_lookup (task->settings, "symbols_disabled");
-
- if (disabled) {
- it = NULL;
-
- while ((cur = ucl_iterate_object (disabled, &it, true)) != NULL) {
- rspamd_symcache_disable_symbol_checkpoint (task, cache,
- ucl_object_tostring (cur));
- }
- }
-
- /* Disable groups of symbols */
- disabled = ucl_object_lookup (task->settings, "groups_disabled");
-
- if (disabled) {
- it = NULL;
-
- while ((cur = ucl_iterate_object (disabled, &it, true)) != NULL) {
- if (ucl_object_type (cur) == UCL_STRING) {
- gr = g_hash_table_lookup (task->cfg->groups,
- ucl_object_tostring (cur));
-
- if (gr) {
- g_hash_table_iter_init (&gr_it, gr->symbols);
-
- while (g_hash_table_iter_next (&gr_it, &k, &v)) {
- rspamd_symcache_disable_symbol_checkpoint (task, cache, k);
- }
- }
- }
- }
- }
-
- return FALSE;
- }
-
- gboolean
- rspamd_symcache_process_symbols (struct rspamd_task *task,
- struct rspamd_symcache *cache,
- gint stage)
- {
- struct rspamd_symcache_item *item = NULL;
- struct rspamd_symcache_dynamic_item *dyn_item;
- struct cache_savepoint *checkpoint;
- gint i;
- gboolean all_done = TRUE;
- gint saved_priority;
- guint start_events_pending;
-
- g_assert (cache != NULL);
-
- if (task->checkpoint == NULL) {
- checkpoint = rspamd_symcache_make_checkpoint (task, cache);
- task->checkpoint = checkpoint;
- }
- else {
- checkpoint = task->checkpoint;
- }
-
- msg_debug_cache_task ("symbols processing stage at pass: %d", stage);
- start_events_pending = rspamd_session_events_pending (task->s);
-
- switch (stage) {
- case RSPAMD_TASK_STAGE_CONNFILTERS:
- /* Check for connection filters */
- saved_priority = G_MININT;
- all_done = TRUE;
-
- for (i = 0; i < (gint) cache->connfilters->len; i++) {
- item = g_ptr_array_index (cache->connfilters, i);
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
-
- if (RSPAMD_TASK_IS_SKIPPED (task)) {
- return TRUE;
- }
-
- if (!CHECK_START_BIT (checkpoint, dyn_item) &&
- !CHECK_FINISH_BIT (checkpoint, dyn_item)) {
-
- if (checkpoint->has_slow) {
- /* Delay */
- checkpoint->has_slow = FALSE;
-
- return FALSE;
- }
- /* Check priorities */
- if (saved_priority == G_MININT) {
- saved_priority = item->priority;
- }
- else {
- if (item->priority < saved_priority &&
- rspamd_session_events_pending (task->s) > start_events_pending) {
- /*
- * Delay further checks as we have higher
- * priority filters to be processed
- */
- return FALSE;
- }
- }
-
- rspamd_symcache_check_symbol (task, cache, item,
- checkpoint);
- all_done = FALSE;
- }
- }
- break;
-
- case RSPAMD_TASK_STAGE_PRE_FILTERS:
- /* Check for prefilters */
- saved_priority = G_MININT;
- all_done = TRUE;
-
- for (i = 0; i < (gint) cache->prefilters->len; i++) {
- item = g_ptr_array_index (cache->prefilters, i);
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
-
- if (RSPAMD_TASK_IS_SKIPPED (task)) {
- return TRUE;
- }
-
- if (!CHECK_START_BIT (checkpoint, dyn_item) &&
- !CHECK_FINISH_BIT (checkpoint, dyn_item)) {
- /* Check priorities */
- if (checkpoint->has_slow) {
- /* Delay */
- checkpoint->has_slow = FALSE;
-
- return FALSE;
- }
-
- if (saved_priority == G_MININT) {
- saved_priority = item->priority;
- }
- else {
- if (item->priority < saved_priority &&
- rspamd_session_events_pending (task->s) > start_events_pending) {
- /*
- * Delay further checks as we have higher
- * priority filters to be processed
- */
- return FALSE;
- }
- }
-
- rspamd_symcache_check_symbol (task, cache, item,
- checkpoint);
- all_done = FALSE;
- }
- }
-
- break;
-
- case RSPAMD_TASK_STAGE_FILTERS:
- all_done = TRUE;
-
- for (i = 0; i < (gint) checkpoint->version; i++) {
- if (RSPAMD_TASK_IS_SKIPPED (task)) {
- return TRUE;
- }
-
- item = g_ptr_array_index (checkpoint->order->d, i);
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
-
- if (item->type & SYMBOL_TYPE_CLASSIFIER) {
- continue;
- }
-
- if (!CHECK_START_BIT (checkpoint, dyn_item)) {
- all_done = FALSE;
-
- if (!rspamd_symcache_check_deps (task, cache, item,
- checkpoint, 0, FALSE)) {
-
- msg_debug_cache_task ("blocked execution of %d(%s) unless deps are "
- "resolved",
- item->id, item->symbol);
-
- continue;
- }
-
- rspamd_symcache_check_symbol (task, cache, item,
- checkpoint);
-
- if (checkpoint->has_slow) {
- /* Delay */
- checkpoint->has_slow = FALSE;
-
- return FALSE;
- }
- }
-
- if (!(item->type & SYMBOL_TYPE_FINE)) {
- if (rspamd_symcache_metric_limit (task, checkpoint)) {
- msg_info_task ("task has already scored more than %.2f, so do "
- "not "
- "plan more checks",
- checkpoint->rs->score);
- all_done = TRUE;
- break;
- }
- }
- }
-
- break;
-
- case RSPAMD_TASK_STAGE_POST_FILTERS:
- /* Check for postfilters */
- saved_priority = G_MININT;
- all_done = TRUE;
-
- for (i = 0; i < (gint) cache->postfilters->len; i++) {
- if (RSPAMD_TASK_IS_SKIPPED (task)) {
- return TRUE;
- }
-
- item = g_ptr_array_index (cache->postfilters, i);
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
-
- if (!CHECK_START_BIT (checkpoint, dyn_item) &&
- !CHECK_FINISH_BIT (checkpoint, dyn_item)) {
- /* Check priorities */
- all_done = FALSE;
-
- if (checkpoint->has_slow) {
- /* Delay */
- checkpoint->has_slow = FALSE;
-
- return FALSE;
- }
-
- if (saved_priority == G_MININT) {
- saved_priority = item->priority;
- }
- else {
- if (item->priority > saved_priority &&
- rspamd_session_events_pending (task->s) > start_events_pending) {
- /*
- * Delay further checks as we have higher
- * priority filters to be processed
- */
-
- return FALSE;
- }
- }
-
- rspamd_symcache_check_symbol (task, cache, item,
- checkpoint);
- }
- }
-
- break;
-
- case RSPAMD_TASK_STAGE_IDEMPOTENT:
- /* Check for postfilters */
- saved_priority = G_MININT;
-
- for (i = 0; i < (gint) cache->idempotent->len; i++) {
- item = g_ptr_array_index (cache->idempotent, i);
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
-
- if (!CHECK_START_BIT (checkpoint, dyn_item) &&
- !CHECK_FINISH_BIT (checkpoint, dyn_item)) {
- /* Check priorities */
- if (checkpoint->has_slow) {
- /* Delay */
- checkpoint->has_slow = FALSE;
-
- return FALSE;
- }
-
- if (saved_priority == G_MININT) {
- saved_priority = item->priority;
- }
- else {
- if (item->priority > saved_priority &&
- rspamd_session_events_pending (task->s) > start_events_pending) {
- /*
- * Delay further checks as we have higher
- * priority filters to be processed
- */
- return FALSE;
- }
- }
- rspamd_symcache_check_symbol (task, cache, item,
- checkpoint);
- }
- }
- break;
- default:
- g_assert_not_reached ();
- }
-
- return all_done;
- }
-
- struct counters_cbdata {
- ucl_object_t *top;
- struct rspamd_symcache *cache;
- };
-
- /* Leave several digits */
- #define P10(X) (1e##X)
- #define ROUND_DOUBLE_DIGITS(x, dig) (floor((x) * P10(dig)) / P10(dig))
- #define ROUND_DOUBLE(x) ROUND_DOUBLE_DIGITS(x, 3)
-
- static void
- rspamd_symcache_counters_cb (gpointer k, gpointer v, gpointer ud)
- {
- struct counters_cbdata *cbd = ud;
- ucl_object_t *obj, *top;
- struct rspamd_symcache_item *item = v, *parent;
- const gchar *symbol = k;
-
- top = cbd->top;
-
- obj = ucl_object_typed_new (UCL_OBJECT);
- ucl_object_insert_key (obj, ucl_object_fromstring (symbol ? symbol : "unknown"),
- "symbol", 0, false);
-
- if (item->is_virtual) {
- if (!(item->type & SYMBOL_TYPE_GHOST)) {
- parent = g_ptr_array_index (cbd->cache->items_by_id,
- item->specific.virtual.parent);
- ucl_object_insert_key (obj,
- ucl_object_fromdouble (ROUND_DOUBLE (item->st->weight)),
- "weight", 0, false);
- ucl_object_insert_key (obj,
- ucl_object_fromdouble (ROUND_DOUBLE (parent->st->avg_frequency)),
- "frequency", 0, false);
- ucl_object_insert_key (obj,
- ucl_object_fromint (parent->st->total_hits),
- "hits", 0, false);
- ucl_object_insert_key (obj,
- ucl_object_fromdouble (ROUND_DOUBLE (parent->st->avg_time)),
- "time", 0, false);
- }
- else {
- ucl_object_insert_key (obj,
- ucl_object_fromdouble (ROUND_DOUBLE (item->st->weight)),
- "weight", 0, false);
- ucl_object_insert_key (obj,
- ucl_object_fromdouble (0.0),
- "frequency", 0, false);
- ucl_object_insert_key (obj,
- ucl_object_fromdouble (0.0),
- "hits", 0, false);
- ucl_object_insert_key (obj,
- ucl_object_fromdouble (0.0),
- "time", 0, false);
- }
- }
- else {
- ucl_object_insert_key (obj,
- ucl_object_fromdouble (ROUND_DOUBLE (item->st->weight)),
- "weight", 0, false);
- ucl_object_insert_key (obj,
- ucl_object_fromdouble (ROUND_DOUBLE (item->st->avg_frequency)),
- "frequency", 0, false);
- ucl_object_insert_key (obj,
- ucl_object_fromint (item->st->total_hits),
- "hits", 0, false);
- ucl_object_insert_key (obj,
- ucl_object_fromdouble (ROUND_DOUBLE (item->st->avg_time)),
- "time", 0, false);
- }
-
- ucl_array_append (top, obj);
- }
-
- #undef ROUND_DOUBLE
-
- ucl_object_t *
- rspamd_symcache_counters (struct rspamd_symcache *cache)
- {
- ucl_object_t *top;
- struct counters_cbdata cbd;
-
- g_assert (cache != NULL);
- top = ucl_object_typed_new (UCL_ARRAY);
- cbd.top = top;
- cbd.cache = cache;
- g_hash_table_foreach (cache->items_by_symbol,
- rspamd_symcache_counters_cb, &cbd);
-
- return top;
- }
-
- static void
- rspamd_symcache_call_peak_cb (struct ev_loop *ev_base,
- struct rspamd_symcache *cache,
- struct rspamd_symcache_item *item,
- gdouble cur_value,
- gdouble cur_err)
- {
- lua_State *L = cache->cfg->lua_state;
- struct ev_loop **pbase;
-
- lua_rawgeti (L, LUA_REGISTRYINDEX, cache->peak_cb);
- pbase = lua_newuserdata (L, sizeof (*pbase));
- *pbase = ev_base;
- rspamd_lua_setclass (L, "rspamd{ev_base}", -1);
- lua_pushstring (L, item->symbol);
- lua_pushnumber (L, item->st->avg_frequency);
- lua_pushnumber (L, sqrt (item->st->stddev_frequency));
- lua_pushnumber (L, cur_value);
- lua_pushnumber (L, cur_err);
-
- if (lua_pcall (L, 6, 0, 0) != 0) {
- msg_info_cache ("call to peak function for %s failed: %s",
- item->symbol, lua_tostring (L, -1));
- lua_pop (L, 1);
- }
- }
-
- static void
- rspamd_symcache_resort_cb (EV_P_ ev_timer *w, int revents)
- {
- gdouble tm;
- struct rspamd_cache_refresh_cbdata *cbdata =
- (struct rspamd_cache_refresh_cbdata *)w->data;
- struct rspamd_symcache *cache;
- struct rspamd_symcache_item *item;
- guint i;
- gdouble cur_ticks;
- static const double decay_rate = 0.25;
-
- cache = cbdata->cache;
- /* Plan new event */
- tm = rspamd_time_jitter (cache->reload_time, 0);
- cur_ticks = rspamd_get_ticks (FALSE);
- msg_debug_cache ("resort symbols cache, next reload in %.2f seconds", tm);
- g_assert (cache != NULL);
- cbdata->resort_ev.repeat = tm;
- ev_timer_again (EV_A_ w);
-
- if (rspamd_worker_is_primary_controller (cbdata->w)) {
- /* Gather stats from shared execution times */
- for (i = 0; i < cache->filters->len; i ++) {
- item = g_ptr_array_index (cache->filters, i);
- item->st->total_hits += item->st->hits;
- g_atomic_int_set (&item->st->hits, 0);
-
- if (item->last_count > 0 && cbdata->w->index == 0) {
- /* Calculate frequency */
- gdouble cur_err, cur_value;
-
- cur_value = (item->st->total_hits - item->last_count) /
- (cur_ticks - cbdata->last_resort);
- rspamd_set_counter_ema (&item->st->frequency_counter,
- cur_value, decay_rate);
- item->st->avg_frequency = item->st->frequency_counter.mean;
- item->st->stddev_frequency = item->st->frequency_counter.stddev;
-
- if (cur_value > 0) {
- msg_debug_cache ("frequency for %s is %.2f, avg: %.2f",
- item->symbol, cur_value, item->st->avg_frequency);
- }
-
- cur_err = (item->st->avg_frequency - cur_value);
- cur_err *= cur_err;
-
- /*
- * TODO: replace magic number
- */
- if (item->st->frequency_counter.number > 10 &&
- cur_err > sqrt (item->st->stddev_frequency) * 3) {
- item->frequency_peaks ++;
- msg_debug_cache ("peak found for %s is %.2f, avg: %.2f, "
- "stddev: %.2f, error: %.2f, peaks: %d",
- item->symbol, cur_value,
- item->st->avg_frequency,
- item->st->stddev_frequency,
- cur_err,
- item->frequency_peaks);
-
- if (cache->peak_cb != -1) {
- rspamd_symcache_call_peak_cb (cbdata->event_loop,
- cache, item,
- cur_value, cur_err);
- }
- }
- }
-
- item->last_count = item->st->total_hits;
-
- if (item->cd->number > 0) {
- if (item->type & (SYMBOL_TYPE_CALLBACK|SYMBOL_TYPE_NORMAL)) {
- item->st->avg_time = item->cd->mean;
- rspamd_set_counter_ema (&item->st->time_counter,
- item->st->avg_time, decay_rate);
- item->st->avg_time = item->st->time_counter.mean;
- memset (item->cd, 0, sizeof (*item->cd));
- }
- }
- }
-
- cbdata->last_resort = cur_ticks;
- /* We don't do actual sorting due to topological guarantees */
- }
- }
-
- static void
- rspamd_symcache_refresh_dtor (void *d)
- {
- struct rspamd_cache_refresh_cbdata *cbdata =
- (struct rspamd_cache_refresh_cbdata *)d;
-
- ev_timer_stop (cbdata->event_loop, &cbdata->resort_ev);
- }
-
- void
- rspamd_symcache_start_refresh (struct rspamd_symcache *cache,
- struct ev_loop *ev_base, struct rspamd_worker *w)
- {
- gdouble tm;
- struct rspamd_cache_refresh_cbdata *cbdata;
-
- cbdata = rspamd_mempool_alloc0 (cache->static_pool, sizeof (*cbdata));
- cbdata->last_resort = rspamd_get_ticks (TRUE);
- cbdata->event_loop = ev_base;
- cbdata->w = w;
- cbdata->cache = cache;
- tm = rspamd_time_jitter (cache->reload_time, 0);
- msg_debug_cache ("next reload in %.2f seconds", tm);
- g_assert (cache != NULL);
- cbdata->resort_ev.data = cbdata;
- ev_timer_init (&cbdata->resort_ev, rspamd_symcache_resort_cb,
- tm, tm);
- ev_timer_start (cbdata->event_loop, &cbdata->resort_ev);
- rspamd_mempool_add_destructor (cache->static_pool,
- rspamd_symcache_refresh_dtor, cbdata);
- }
-
- void
- rspamd_symcache_inc_frequency (struct rspamd_symcache *cache,
- struct rspamd_symcache_item *item)
- {
- if (item != NULL) {
- g_atomic_int_inc (&item->st->hits);
- }
- }
-
- void
- rspamd_symcache_add_dependency (struct rspamd_symcache *cache,
- gint id_from, const gchar *to,
- gint virtual_id_from)
- {
- struct rspamd_symcache_item *source, *vsource;
- struct cache_dependency *dep;
-
- g_assert (id_from >= 0 && id_from < (gint)cache->items_by_id->len);
-
- source = (struct rspamd_symcache_item *)g_ptr_array_index (cache->items_by_id, id_from);
- dep = rspamd_mempool_alloc (cache->static_pool, sizeof (*dep));
- dep->id = id_from;
- dep->sym = rspamd_mempool_strdup (cache->static_pool, to);
- /* Will be filled later */
- dep->item = NULL;
- dep->vid = -1;
- g_ptr_array_add (source->deps, dep);
-
- if (virtual_id_from >= 0) {
- g_assert (virtual_id_from < (gint)cache->virtual->len);
- /* We need that for settings id propagation */
- vsource = (struct rspamd_symcache_item *)
- g_ptr_array_index (cache->virtual, virtual_id_from);
- dep = rspamd_mempool_alloc (cache->static_pool, sizeof (*dep));
- dep->vid = virtual_id_from;
- dep->id = -1;
- dep->sym = rspamd_mempool_strdup (cache->static_pool, to);
- /* Will be filled later */
- dep->item = NULL;
- g_ptr_array_add (vsource->deps, dep);
- }
- }
-
- void
- rspamd_symcache_add_delayed_dependency (struct rspamd_symcache *cache,
- const gchar *from, const gchar *to)
- {
- struct delayed_cache_dependency *ddep;
-
- g_assert (from != NULL);
- g_assert (to != NULL);
-
- ddep = g_malloc0 (sizeof (*ddep));
- ddep->from = g_strdup (from);
- ddep->to = g_strdup (to);
-
- cache->delayed_deps = g_list_prepend (cache->delayed_deps, ddep);
- }
-
- gint
- rspamd_symcache_find_symbol (struct rspamd_symcache *cache, const gchar *name)
- {
- struct rspamd_symcache_item *item;
-
- g_assert (cache != NULL);
-
- if (name == NULL) {
- return -1;
- }
-
- item = g_hash_table_lookup (cache->items_by_symbol, name);
-
- if (item != NULL) {
- return item->id;
- }
-
- return -1;
- }
-
- gboolean
- rspamd_symcache_stat_symbol (struct rspamd_symcache *cache,
- const gchar *name,
- gdouble *frequency,
- gdouble *freq_stddev,
- gdouble *tm,
- guint *nhits)
- {
- struct rspamd_symcache_item *item;
-
- g_assert (cache != NULL);
-
- if (name == NULL) {
- return FALSE;
- }
-
- item = g_hash_table_lookup (cache->items_by_symbol, name);
-
- if (item != NULL) {
- *frequency = item->st->avg_frequency;
- *freq_stddev = sqrt (item->st->stddev_frequency);
- *tm = item->st->time_counter.mean;
-
- if (nhits) {
- *nhits = item->st->hits;
- }
-
- return TRUE;
- }
-
- return FALSE;
- }
-
- const gchar *
- rspamd_symcache_symbol_by_id (struct rspamd_symcache *cache,
- gint id)
- {
- struct rspamd_symcache_item *item;
-
- g_assert (cache != NULL);
-
- if (id < 0 || id >= (gint)cache->items_by_id->len) {
- return NULL;
- }
-
- item = g_ptr_array_index (cache->items_by_id, id);
-
- return item->symbol;
- }
-
- guint
- rspamd_symcache_stats_symbols_count (struct rspamd_symcache *cache)
- {
- g_assert (cache != NULL);
-
- return cache->stats_symbols_count;
- }
-
-
- void
- rspamd_symcache_disable_all_symbols (struct rspamd_task *task,
- struct rspamd_symcache *cache,
- guint skip_mask)
- {
- struct cache_savepoint *checkpoint;
- guint i;
- struct rspamd_symcache_item *item;
- struct rspamd_symcache_dynamic_item *dyn_item;
-
- if (task->checkpoint == NULL) {
- checkpoint = rspamd_symcache_make_checkpoint (task, cache);
- task->checkpoint = checkpoint;
- }
- else {
- checkpoint = task->checkpoint;
- }
-
- /* Enable for squeezed symbols */
- PTR_ARRAY_FOREACH (cache->items_by_id, i, item) {
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
-
- if (!(item->type & (skip_mask))) {
- SET_FINISH_BIT (checkpoint, dyn_item);
- SET_START_BIT (checkpoint, dyn_item);
- }
- }
- }
-
- static void
- rspamd_symcache_disable_symbol_checkpoint (struct rspamd_task *task,
- struct rspamd_symcache *cache, const gchar *symbol)
- {
- struct cache_savepoint *checkpoint;
- struct rspamd_symcache_item *item;
- struct rspamd_symcache_dynamic_item *dyn_item;
-
- if (task->checkpoint == NULL) {
- checkpoint = rspamd_symcache_make_checkpoint (task, cache);
- task->checkpoint = checkpoint;
- }
- else {
- checkpoint = task->checkpoint;
- }
-
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
- SET_FINISH_BIT (checkpoint, dyn_item);
- SET_START_BIT (checkpoint, dyn_item);
- msg_debug_cache_task ("disable execution of %s", symbol);
- }
- else {
- msg_info_task ("cannot disable %s: not found", symbol);
- }
- }
-
- static void
- rspamd_symcache_enable_symbol_checkpoint (struct rspamd_task *task,
- struct rspamd_symcache *cache, const gchar *symbol)
- {
- struct cache_savepoint *checkpoint;
- struct rspamd_symcache_item *item;
- struct rspamd_symcache_dynamic_item *dyn_item;
-
- if (task->checkpoint == NULL) {
- checkpoint = rspamd_symcache_make_checkpoint (task, cache);
- task->checkpoint = checkpoint;
- }
- else {
- checkpoint = task->checkpoint;
- }
-
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
- dyn_item->finished = 0;
- dyn_item->started = 0;
- msg_debug_cache_task ("enable execution of %s", symbol);
- }
- else {
- msg_info_task ("cannot enable %s: not found", symbol);
- }
- }
-
- struct rspamd_abstract_callback_data*
- rspamd_symcache_get_cbdata (struct rspamd_symcache *cache,
- const gchar *symbol)
- {
- struct rspamd_symcache_item *item;
-
- g_assert (cache != NULL);
- g_assert (symbol != NULL);
-
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
- return item->specific.normal.user_data;
- }
-
- return NULL;
- }
-
- gboolean
- rspamd_symcache_is_checked (struct rspamd_task *task,
- struct rspamd_symcache *cache, const gchar *symbol)
- {
- struct cache_savepoint *checkpoint;
- struct rspamd_symcache_item *item;
- struct rspamd_symcache_dynamic_item *dyn_item;
-
- g_assert (cache != NULL);
- g_assert (symbol != NULL);
-
- if (task->checkpoint == NULL) {
- checkpoint = rspamd_symcache_make_checkpoint (task, cache);
- task->checkpoint = checkpoint;
- }
- else {
- checkpoint = task->checkpoint;
- }
-
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
- return dyn_item->started;
- }
-
- return FALSE;
- }
-
- void
- rspamd_symcache_disable_symbol_perm (struct rspamd_symcache *cache,
- const gchar *symbol,
- gboolean resolve_parent)
- {
- struct rspamd_symcache_item *item;
-
- g_assert (cache != NULL);
- g_assert (symbol != NULL);
-
- item = rspamd_symcache_find_filter (cache, symbol, resolve_parent);
-
- if (item) {
- item->enabled = FALSE;
- }
- }
-
- void
- rspamd_symcache_enable_symbol_perm (struct rspamd_symcache *cache,
- const gchar *symbol)
- {
- struct rspamd_symcache_item *item;
-
- g_assert (cache != NULL);
- g_assert (symbol != NULL);
-
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
- item->enabled = TRUE;
- }
- }
-
- guint64
- rspamd_symcache_get_cksum (struct rspamd_symcache *cache)
- {
- g_assert (cache != NULL);
-
- return cache->cksum;
- }
-
-
- gboolean
- rspamd_symcache_is_symbol_enabled (struct rspamd_task *task,
- struct rspamd_symcache *cache,
- const gchar *symbol)
- {
- struct cache_savepoint *checkpoint;
- struct rspamd_symcache_item *item;
- struct rspamd_symcache_dynamic_item *dyn_item;
- lua_State *L;
- struct rspamd_task **ptask;
- gboolean ret = TRUE;
-
- g_assert (cache != NULL);
- g_assert (symbol != NULL);
-
- checkpoint = task->checkpoint;
-
-
- if (checkpoint) {
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
-
- if (!rspamd_symcache_is_item_allowed (task, item, TRUE)) {
- ret = FALSE;
- }
- else {
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
- if (CHECK_START_BIT (checkpoint, dyn_item)) {
- ret = FALSE;
- }
- else {
- if (item->specific.normal.conditions) {
- struct rspamd_symcache_condition *cur_cond;
-
- DL_FOREACH (item->specific.normal.conditions, cur_cond) {
- /*
- * We also executes condition callback to check
- * if we need this symbol
- */
- L = task->cfg->lua_state;
- lua_rawgeti (L, LUA_REGISTRYINDEX, cur_cond->cb);
- ptask = lua_newuserdata (L, sizeof (struct rspamd_task *));
- rspamd_lua_setclass (L, "rspamd{task}", -1);
- *ptask = task;
-
- if (lua_pcall (L, 1, 1, 0) != 0) {
- msg_info_task ("call to condition for %s failed: %s",
- item->symbol, lua_tostring (L, -1));
- lua_pop (L, 1);
- }
- else {
- ret = lua_toboolean (L, -1);
- lua_pop (L, 1);
- }
-
- if (!ret) {
- break;
- }
- }
- }
- }
- }
- }
- }
-
- return ret;
- }
-
-
- gboolean
- rspamd_symcache_enable_symbol (struct rspamd_task *task,
- struct rspamd_symcache *cache,
- const gchar *symbol)
- {
- struct cache_savepoint *checkpoint;
- struct rspamd_symcache_item *item;
- struct rspamd_symcache_dynamic_item *dyn_item;
- gboolean ret = FALSE;
-
- g_assert (cache != NULL);
- g_assert (symbol != NULL);
-
- checkpoint = task->checkpoint;
-
- if (checkpoint) {
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
-
- if (!CHECK_FINISH_BIT (checkpoint, dyn_item)) {
- ret = TRUE;
- CLR_START_BIT (checkpoint, dyn_item);
- CLR_FINISH_BIT (checkpoint, dyn_item);
- }
- else {
- msg_debug_task ("cannot enable symbol %s: already started", symbol);
- }
- }
- }
-
- return ret;
- }
-
-
- gboolean
- rspamd_symcache_disable_symbol (struct rspamd_task *task,
- struct rspamd_symcache *cache,
- const gchar *symbol)
- {
- struct cache_savepoint *checkpoint;
- struct rspamd_symcache_item *item;
- struct rspamd_symcache_dynamic_item *dyn_item;
- gboolean ret = FALSE;
-
- g_assert (cache != NULL);
- g_assert (symbol != NULL);
-
- checkpoint = task->checkpoint;
-
- if (checkpoint) {
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
-
- if (!CHECK_START_BIT (checkpoint, dyn_item)) {
- ret = TRUE;
- SET_START_BIT (checkpoint, dyn_item);
- SET_FINISH_BIT (checkpoint, dyn_item);
- }
- else {
- if (!CHECK_FINISH_BIT (checkpoint, dyn_item)) {
- msg_warn_task ("cannot disable symbol %s: already started",
- symbol);
- }
- }
- }
- }
-
- return ret;
- }
-
- void
- rspamd_symcache_foreach (struct rspamd_symcache *cache,
- void (*func) (struct rspamd_symcache_item *, gpointer),
- gpointer ud)
- {
- struct rspamd_symcache_item *item;
- GHashTableIter it;
- gpointer k, v;
-
- g_hash_table_iter_init (&it, cache->items_by_symbol);
-
- while (g_hash_table_iter_next (&it, &k, &v)) {
- item = (struct rspamd_symcache_item *)v;
- func (item, ud);
- }
- }
-
- struct rspamd_symcache_item *
- rspamd_symcache_get_cur_item (struct rspamd_task *task)
- {
- struct cache_savepoint *checkpoint = task->checkpoint;
-
- if (checkpoint == NULL) {
- return NULL;
- }
-
- return checkpoint->cur_item;
- }
-
- /**
- * Replaces the current item being processed.
- * Returns the current item being processed (if any)
- * @param task
- * @param item
- * @return
- */
- struct rspamd_symcache_item *
- rspamd_symcache_set_cur_item (struct rspamd_task *task,
- struct rspamd_symcache_item *item)
- {
- struct cache_savepoint *checkpoint = task->checkpoint;
- struct rspamd_symcache_item *ex;
-
- ex = checkpoint->cur_item;
- checkpoint->cur_item = item;
-
- return ex;
- }
-
- struct rspamd_symcache_delayed_cbdata {
- struct rspamd_symcache_item *item;
- struct rspamd_task *task;
- struct rspamd_async_event *event;
- struct ev_timer tm;
- };
-
- static void
- rspamd_symcache_delayed_item_fin (gpointer ud)
- {
- struct rspamd_symcache_delayed_cbdata *cbd =
- (struct rspamd_symcache_delayed_cbdata *)ud;
- struct rspamd_task *task;
- struct cache_savepoint *checkpoint;
-
- task = cbd->task;
- checkpoint = task->checkpoint;
- checkpoint->has_slow = FALSE;
- ev_timer_stop (task->event_loop, &cbd->tm);
- }
-
- static void
- rspamd_symcache_delayed_item_cb (EV_P_ ev_timer *w, int what)
- {
- struct rspamd_symcache_delayed_cbdata *cbd =
- (struct rspamd_symcache_delayed_cbdata *)w->data;
- struct rspamd_symcache_item *item;
- struct rspamd_task *task;
- struct cache_dependency *rdep;
- struct cache_savepoint *checkpoint;
- struct rspamd_symcache_dynamic_item *dyn_item;
- guint i;
-
- item = cbd->item;
- task = cbd->task;
- checkpoint = task->checkpoint;
- cbd->event = NULL;
-
- /* Timer will be stopped here */
- rspamd_session_remove_event (task->s,
- rspamd_symcache_delayed_item_fin, cbd);
-
- /* Process all reverse dependencies */
- PTR_ARRAY_FOREACH (item->rdeps, i, rdep) {
- if (rdep->item) {
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, rdep->item);
- if (!CHECK_START_BIT (checkpoint, dyn_item)) {
- msg_debug_cache_task ("check item %d(%s) rdep of %s ",
- rdep->item->id, rdep->item->symbol, item->symbol);
-
- if (!rspamd_symcache_check_deps (task, task->cfg->cache,
- rdep->item,
- checkpoint, 0, FALSE)) {
- msg_debug_cache_task ("blocked execution of %d(%s) rdep of %s "
- "unless deps are resolved",
- rdep->item->id, rdep->item->symbol, item->symbol);
- }
- else {
- rspamd_symcache_check_symbol (task, task->cfg->cache,
- rdep->item,
- checkpoint);
- }
- }
- }
- }
- }
-
- static void
- rspamd_delayed_timer_dtor (gpointer d)
- {
- struct rspamd_symcache_delayed_cbdata *cbd =
- (struct rspamd_symcache_delayed_cbdata *)d;
-
- if (cbd->event) {
- /* Event has not been executed */
- rspamd_session_remove_event (cbd->task->s,
- rspamd_symcache_delayed_item_fin, cbd);
- cbd->event = NULL;
- }
- }
-
- /**
- * Finalize the current async element potentially calling its deps
- */
- void
- rspamd_symcache_finalize_item (struct rspamd_task *task,
- struct rspamd_symcache_item *item)
- {
- struct cache_savepoint *checkpoint = task->checkpoint;
- struct cache_dependency *rdep;
- struct rspamd_symcache_dynamic_item *dyn_item;
- gdouble diff;
- guint i;
- gboolean enable_slow_timer = FALSE;
- const gdouble slow_diff_limit = 300;
-
- /* Sanity checks */
- g_assert (checkpoint->items_inflight > 0);
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
-
- if (dyn_item->async_events > 0) {
- /*
- * XXX: Race condition
- *
- * It is possible that some async event is still in flight, but we
- * already know its result, however, it is the responsibility of that
- * event to decrease async events count and call this function
- * one more time
- */
- msg_debug_cache_task ("postpone finalisation of %s(%d) as there are %d "
- "async events pendning",
- item->symbol, item->id, dyn_item->async_events);
-
- return;
- }
-
- msg_debug_cache_task ("process finalize for item %s(%d)", item->symbol, item->id);
- SET_FINISH_BIT (checkpoint, dyn_item);
- checkpoint->items_inflight --;
- checkpoint->cur_item = NULL;
-
- if (checkpoint->profile) {
- ev_now_update_if_cheap (task->event_loop);
- diff = ((ev_now (task->event_loop) - checkpoint->profile_start) * 1e3 -
- dyn_item->start_msec);
-
- if (diff > slow_diff_limit) {
-
- if (!checkpoint->has_slow) {
- checkpoint->has_slow = TRUE;
- enable_slow_timer = TRUE;
- msg_info_task ("slow rule: %s(%d): %.2f ms; enable slow timer delay",
- item->symbol, item->id,
- diff);
- }
- else {
- msg_info_task ("slow rule: %s(%d): %.2f ms",
- item->symbol, item->id,
- diff);
- }
- }
-
- if (G_UNLIKELY (RSPAMD_TASK_IS_PROFILING (task))) {
- rspamd_task_profile_set (task, item->symbol, diff);
- }
-
- if (rspamd_worker_is_scanner (task->worker)) {
- rspamd_set_counter (item->cd, diff);
- }
- }
-
- if (enable_slow_timer) {
- struct rspamd_symcache_delayed_cbdata *cbd =
- rspamd_mempool_alloc (task->task_pool,sizeof (*cbd));
- /* Add timer to allow something else to be executed */
- ev_timer *tm = &cbd->tm;
-
- cbd->event = rspamd_session_add_event (task->s,
- rspamd_symcache_delayed_item_fin, cbd,
- "symcache");
-
- /*
- * If no event could be added, then we are already in the destruction
- * phase. So the main issue is to deal with has slow here
- */
- if (cbd->event) {
- ev_timer_init (tm, rspamd_symcache_delayed_item_cb, 0.1, 0.0);
- ev_set_priority (tm, EV_MINPRI);
- rspamd_mempool_add_destructor (task->task_pool,
- rspamd_delayed_timer_dtor, cbd);
-
- cbd->task = task;
- cbd->item = item;
- tm->data = cbd;
- ev_timer_start (task->event_loop, tm);
- }
- else {
- /* Just reset as no timer is added */
- checkpoint->has_slow = FALSE;
- }
-
- return;
- }
-
- /* Process all reverse dependencies */
- PTR_ARRAY_FOREACH (item->rdeps, i, rdep) {
- if (rdep->item) {
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, rdep->item);
- if (!CHECK_START_BIT (checkpoint, dyn_item)) {
- msg_debug_cache_task ("check item %d(%s) rdep of %s ",
- rdep->item->id, rdep->item->symbol, item->symbol);
-
- if (!rspamd_symcache_check_deps (task, task->cfg->cache,
- rdep->item,
- checkpoint, 0, FALSE)) {
- msg_debug_cache_task ("blocked execution of %d(%s) rdep of %s "
- "unless deps are resolved",
- rdep->item->id, rdep->item->symbol, item->symbol);
- }
- else {
- rspamd_symcache_check_symbol (task, task->cfg->cache,
- rdep->item,
- checkpoint);
- }
- }
- }
- }
- }
-
- guint
- rspamd_symcache_item_async_inc_full (struct rspamd_task *task,
- struct rspamd_symcache_item *item,
- const gchar *subsystem,
- const gchar *loc)
- {
- struct rspamd_symcache_dynamic_item *dyn_item;
- struct cache_savepoint *checkpoint = task->checkpoint;
-
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
- msg_debug_cache_task ("increase async events counter for %s(%d) = %d + 1; "
- "subsystem %s (%s)",
- item->symbol, item->id, dyn_item->async_events, subsystem, loc);
- return ++dyn_item->async_events;
- }
-
- guint
- rspamd_symcache_item_async_dec_full (struct rspamd_task *task,
- struct rspamd_symcache_item *item,
- const gchar *subsystem,
- const gchar *loc)
- {
- struct rspamd_symcache_dynamic_item *dyn_item;
- struct cache_savepoint *checkpoint = task->checkpoint;
-
- dyn_item = rspamd_symcache_get_dynamic (checkpoint, item);
- msg_debug_cache_task ("decrease async events counter for %s(%d) = %d - 1; "
- "subsystem %s (%s)",
- item->symbol, item->id, dyn_item->async_events, subsystem, loc);
- g_assert (dyn_item->async_events > 0);
-
- return --dyn_item->async_events;
- }
-
- gboolean
- rspamd_symcache_item_async_dec_check_full (struct rspamd_task *task,
- struct rspamd_symcache_item *item,
- const gchar *subsystem,
- const gchar *loc)
- {
- if (rspamd_symcache_item_async_dec_full (task, item, subsystem, loc) == 0) {
- rspamd_symcache_finalize_item (task, item);
-
- return TRUE;
- }
-
- return FALSE;
- }
-
- gboolean
- rspamd_symcache_add_symbol_flags (struct rspamd_symcache *cache,
- const gchar *symbol,
- guint flags)
- {
- struct rspamd_symcache_item *item;
-
- g_assert (cache != NULL);
- g_assert (symbol != NULL);
-
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
- item->type |= flags;
-
- return TRUE;
- }
-
- return FALSE;
- }
-
- gboolean
- rspamd_symcache_set_symbol_flags (struct rspamd_symcache *cache,
- const gchar *symbol,
- guint flags)
- {
- struct rspamd_symcache_item *item;
-
- g_assert (cache != NULL);
- g_assert (symbol != NULL);
-
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
- item->type = flags;
-
- return TRUE;
- }
-
- return FALSE;
- }
-
- guint
- rspamd_symcache_get_symbol_flags (struct rspamd_symcache *cache,
- const gchar *symbol)
- {
- struct rspamd_symcache_item *item;
-
- g_assert (cache != NULL);
- g_assert (symbol != NULL);
-
- item = rspamd_symcache_find_filter (cache, symbol, true);
-
- if (item) {
- return item->type;
- }
-
- return 0;
- }
-
- void
- rspamd_symcache_composites_foreach (struct rspamd_task *task,
- struct rspamd_symcache *cache,
- GHFunc func,
- gpointer fd)
- {
- guint i;
- struct rspamd_symcache_item *item;
- struct rspamd_symcache_dynamic_item *dyn_item;
-
- if (task->checkpoint == NULL) {
- return;
- }
-
- PTR_ARRAY_FOREACH (cache->composites, i, item) {
- dyn_item = rspamd_symcache_get_dynamic (task->checkpoint, item);
-
- if (!CHECK_START_BIT (task->checkpoint, dyn_item)) {
- /* Cannot do it due to 2 passes */
- /* SET_START_BIT (task->checkpoint, dyn_item); */
- func (item->symbol, item->specific.normal.user_data, fd);
- SET_FINISH_BIT (task->checkpoint, dyn_item);
- }
- }
- }
-
- bool
- rspamd_symcache_set_allowed_settings_ids (struct rspamd_symcache *cache,
- const gchar *symbol,
- const guint32 *ids,
- guint nids)
- {
- struct rspamd_symcache_item *item;
-
- item = rspamd_symcache_find_filter (cache, symbol, false);
-
- if (item == NULL) {
- return false;
- }
-
- if (nids <= G_N_ELEMENTS (item->allowed_ids.st)) {
- /* Use static version */
- memset (&item->allowed_ids, 0, sizeof (item->allowed_ids));
- for (guint i = 0; i < nids; i++) {
- item->allowed_ids.st[i] = ids[i];
- }
- }
- else {
- /* Need to use a separate list */
- item->allowed_ids.dyn.e = -1; /* Flag */
- item->allowed_ids.dyn.n = rspamd_mempool_alloc (cache->static_pool,
- sizeof (guint32) * nids);
- item->allowed_ids.dyn.len = nids;
- item->allowed_ids.dyn.allocated = nids;
-
- for (guint i = 0; i < nids; i++) {
- item->allowed_ids.dyn.n[i] = ids[i];
- }
-
- /* Keep sorted */
- qsort (item->allowed_ids.dyn.n, nids, sizeof (guint32), rspamd_id_cmp);
- }
-
- return true;
- }
-
- bool
- rspamd_symcache_set_forbidden_settings_ids (struct rspamd_symcache *cache,
- const gchar *symbol,
- const guint32 *ids,
- guint nids)
- {
- struct rspamd_symcache_item *item;
-
- item = rspamd_symcache_find_filter (cache, symbol, false);
-
- if (item == NULL) {
- return false;
- }
-
- g_assert (nids < G_MAXUINT16);
-
- if (nids <= G_N_ELEMENTS (item->forbidden_ids.st)) {
- /* Use static version */
- memset (&item->forbidden_ids, 0, sizeof (item->forbidden_ids));
- for (guint i = 0; i < nids; i++) {
- item->forbidden_ids.st[i] = ids[i];
- }
- }
- else {
- /* Need to use a separate list */
- item->forbidden_ids.dyn.e = -1; /* Flag */
- item->forbidden_ids.dyn.n = rspamd_mempool_alloc (cache->static_pool,
- sizeof (guint32) * nids);
- item->forbidden_ids.dyn.len = nids;
- item->forbidden_ids.dyn.allocated = nids;
-
- for (guint i = 0; i < nids; i++) {
- item->forbidden_ids.dyn.n[i] = ids[i];
- }
-
- /* Keep sorted */
- qsort (item->forbidden_ids.dyn.n, nids, sizeof (guint32), rspamd_id_cmp);
- }
-
- return true;
- }
-
- const guint32*
- rspamd_symcache_get_allowed_settings_ids (struct rspamd_symcache *cache,
- const gchar *symbol,
- guint *nids)
- {
- struct rspamd_symcache_item *item;
- guint cnt = 0;
-
- item = rspamd_symcache_find_filter (cache, symbol, false);
-
- if (item == NULL) {
- return NULL;
- }
-
- if (item->allowed_ids.dyn.e == -1) {
- /* Dynamic list */
- *nids = item->allowed_ids.dyn.len;
-
- return item->allowed_ids.dyn.n;
- }
- else {
- while (item->allowed_ids.st[cnt] != 0 && cnt < G_N_ELEMENTS (item->allowed_ids.st)) {
- cnt ++;
- }
-
- *nids = cnt;
-
- return item->allowed_ids.st;
- }
- }
-
- const guint32*
- rspamd_symcache_get_forbidden_settings_ids (struct rspamd_symcache *cache,
- const gchar *symbol,
- guint *nids)
- {
- struct rspamd_symcache_item *item;
- guint cnt = 0;
-
- item = rspamd_symcache_find_filter (cache, symbol, false);
-
- if (item == NULL) {
- return NULL;
- }
-
- if (item->forbidden_ids.dyn.e == -1) {
- /* Dynamic list */
- *nids = item->allowed_ids.dyn.len;
-
- return item->allowed_ids.dyn.n;
- }
- else {
- while (item->forbidden_ids.st[cnt] != 0 && cnt < G_N_ELEMENTS (item->allowed_ids.st)) {
- cnt ++;
- }
-
- *nids = cnt;
-
- return item->forbidden_ids.st;
- }
- }
-
- /* Insertion sort: usable for near-sorted ids list */
- static inline void
- rspamd_ids_insertion_sort (guint *a, guint n)
- {
- for (guint i = 1; i < n; i++) {
- guint32 tmp = a[i];
- guint j = i;
-
- while (j > 0 && tmp < a[j - 1]) {
- a[j] = a[j - 1];
- j --;
- }
-
- a[j] = tmp;
- }
- }
-
- static inline void
- rspamd_symcache_add_id_to_list (rspamd_mempool_t *pool,
- struct rspamd_symcache_id_list *ls,
- guint32 id)
- {
- guint cnt = 0;
- guint *new_array;
-
- if (ls->st[0] == -1) {
- /* Dynamic array */
- if (ls->dyn.len < ls->dyn.allocated) {
- /* Trivial, append + sort */
- ls->dyn.n[ls->dyn.len++] = id;
- }
- else {
- /* Reallocate */
- g_assert (ls->dyn.allocated <= G_MAXINT16);
- ls->dyn.allocated *= 2;
-
- new_array = rspamd_mempool_alloc (pool,
- ls->dyn.allocated * sizeof (guint32));
- memcpy (new_array, ls->dyn.n, ls->dyn.len * sizeof (guint32));
- ls->dyn.n = new_array;
- ls->dyn.n[ls->dyn.len++] = id;
- }
-
- rspamd_ids_insertion_sort (ls->dyn.n, ls->dyn.len);
- }
- else {
- /* Static part */
- while (ls->st[cnt] != 0 && cnt < G_N_ELEMENTS (ls->st)) {
- cnt ++;
- }
-
- if (cnt < G_N_ELEMENTS (ls->st)) {
- ls->st[cnt] = id;
- }
- else {
- /* Switch to dynamic */
- new_array = rspamd_mempool_alloc (pool,
- G_N_ELEMENTS (ls->st) * 2 * sizeof (guint32));
- memcpy (new_array, ls->st, G_N_ELEMENTS (ls->st) * sizeof (guint32));
- ls->dyn.n = new_array;
- ls->dyn.e = -1;
- ls->dyn.allocated = G_N_ELEMENTS (ls->st) * 2;
- ls->dyn.len = G_N_ELEMENTS (ls->st);
-
- /* Recursively jump to dynamic branch that will handle insertion + sorting */
- rspamd_symcache_add_id_to_list (pool, ls, id);
- }
- }
- }
-
- void
- rspamd_symcache_process_settings_elt (struct rspamd_symcache *cache,
- struct rspamd_config_settings_elt *elt)
- {
- guint32 id = elt->id;
- ucl_object_iter_t iter;
- struct rspamd_symcache_item *item, *parent;
- const ucl_object_t *cur;
-
-
- if (elt->symbols_disabled) {
- /* Process denied symbols */
- iter = NULL;
-
- while ((cur = ucl_object_iterate (elt->symbols_disabled, &iter, true)) != NULL) {
- const gchar *sym = ucl_object_key (cur);
- item = rspamd_symcache_find_filter (cache, sym, false);
-
- if (item) {
- if (item->is_virtual) {
- /*
- * Virtual symbols are special:
- * we ignore them in symcache but prevent them from being
- * inserted.
- */
- rspamd_symcache_add_id_to_list (cache->static_pool,
- &item->forbidden_ids, id);
- msg_debug_cache ("deny virtual symbol %s for settings %ud (%s); "
- "parent can still be executed",
- sym, id, elt->name);
- }
- else {
- /* Normal symbol, disable it */
- rspamd_symcache_add_id_to_list (cache->static_pool,
- &item->forbidden_ids, id);
- msg_debug_cache ("deny symbol %s for settings %ud (%s)",
- sym, id, elt->name);
- }
- }
- else {
- msg_warn_cache ("cannot find a symbol to disable %s "
- "when processing settings %ud (%s)",
- sym, id, elt->name);
- }
- }
- }
-
- if (elt->symbols_enabled) {
- iter = NULL;
-
- while ((cur = ucl_object_iterate (elt->symbols_enabled, &iter, true)) != NULL) {
- /* Here, we resolve parent and explicitly allow it */
- const gchar *sym = ucl_object_key (cur);
- item = rspamd_symcache_find_filter (cache, sym, false);
-
- if (item) {
- if (item->is_virtual) {
- if (!(item->type & SYMBOL_TYPE_GHOST)) {
- parent = rspamd_symcache_find_filter (cache, sym, true);
-
- if (parent) {
- if (elt->symbols_disabled &&
- ucl_object_lookup (elt->symbols_disabled, parent->symbol)) {
- msg_err_cache ("conflict in %s: cannot enable disabled symbol %s, "
- "wanted to enable symbol %s",
- elt->name, parent->symbol, sym);
- continue;
- }
-
- rspamd_symcache_add_id_to_list (cache->static_pool,
- &parent->exec_only_ids, id);
- msg_debug_cache ("allow just execution of symbol %s for settings %ud (%s)",
- parent->symbol, id, elt->name);
- }
- }
- /* Ignore ghosts */
- }
-
- rspamd_symcache_add_id_to_list (cache->static_pool,
- &item->allowed_ids, id);
- msg_debug_cache ("allow execution of symbol %s for settings %ud (%s)",
- sym, id, elt->name);
- }
- else {
- msg_warn_cache ("cannot find a symbol to enable %s "
- "when processing settings %ud (%s)",
- sym, id, elt->name);
- }
- }
- }
- }
-
- gint
- rspamd_symcache_item_flags (struct rspamd_symcache_item *item)
- {
- if (item) {
- return item->type;
- }
-
- return 0;
- }
-
- const gchar*
- rspamd_symcache_item_name (struct rspamd_symcache_item *item)
- {
- return item ? item->symbol : NULL;
- }
-
- const struct rspamd_symcache_item_stat *
- rspamd_symcache_item_stat (struct rspamd_symcache_item *item)
- {
- return item ? item->st : NULL;
- }
-
- gboolean
- rspamd_symcache_item_is_enabled (struct rspamd_symcache_item *item)
- {
- if (item) {
- if (!item->enabled) {
- return FALSE;
- }
-
- if (item->is_virtual && item->specific.virtual.parent_item != NULL) {
- return rspamd_symcache_item_is_enabled (item->specific.virtual.parent_item);
- }
-
- return TRUE;
- }
-
- return FALSE;
- }
-
- struct rspamd_symcache_item * rspamd_symcache_item_get_parent (
- struct rspamd_symcache_item *item)
- {
- if (item && item->is_virtual && item->specific.virtual.parent_item != NULL) {
- return item->specific.virtual.parent_item;
- }
-
- return NULL;
- }
-
- const GPtrArray*
- rspamd_symcache_item_get_deps (struct rspamd_symcache_item *item)
- {
- struct rspamd_symcache_item *parent;
-
- if (item) {
- parent = rspamd_symcache_item_get_parent (item);
-
- if (parent) {
- item = parent;
- }
-
- return item->deps;
- }
-
- return NULL;
- }
-
- const GPtrArray*
- rspamd_symcache_item_get_rdeps (struct rspamd_symcache_item *item)
- {
- struct rspamd_symcache_item *parent;
-
- if (item) {
- parent = rspamd_symcache_item_get_parent (item);
-
- if (parent) {
- item = parent;
- }
-
- return item->rdeps;
- }
-
- return NULL;
- }
-
- void
- rspamd_symcache_enable_profile (struct rspamd_task *task)
- {
- struct cache_savepoint *checkpoint = task->checkpoint;
-
- if (checkpoint && !checkpoint->profile) {
- ev_now_update_if_cheap (task->event_loop);
- ev_tstamp now = ev_now (task->event_loop);
- checkpoint->profile_start = now;
-
- msg_debug_cache_task ("enable profiling of symbols for task");
- checkpoint->profile = TRUE;
- }
- }
|