1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719 |
- // Copyright 2016 Google Inc.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
- //
- ////////////////////////////////////////////////////////////////////////////////
-
- #include "compact_enc_det.h"
-
- #include <math.h> // for sqrt
- #include <stddef.h> // for size_t
- #include <stdio.h> // for printf, fprintf, NULL, etc
- #include <stdlib.h> // for qsort
- #include <string.h> // for memset, memcpy, memcmp, etc
- #include <memory>
- #include <string> // for string, operator==, etc
-
- #include "compact_enc_det_hint_code.h"
- #include "util/string_util.h"
- #include "util/basictypes.h"
- #include "util/commandlineflags.h"
- #include "util/logging.h"
-
- using std::string;
-
- // TODO as of 2007.10.09:
- //
- // Consider font=TT-BHxxx as user-defined => binary
- // Demote GB18030 if no 8x3x pair
- // Map byte2 ascii punct to 0x60, digits to 0x7e, gets them into hires
- // Consider removing/ignoring bytes 01-1F to avoid crap pollution
- // Possibly boost declared encoding in robust scan
- // googlebot tiny files
- // look for ranges of encodings
- // consider tags just as > < within aligned block of 32
- // flag too few characters in postproc (Latin 6 problem)
- // Remove slow scan beyond 16KB
- // Consider removing kMostLikelyEncoding or cut it in half
-
-
- // A note on mixed encodings
- //
- // The most common encoding error on the web is a page containing a mixture of
- // CP-1252 and UTF-8. A less common encoding error is a third-party feed that
- // has been converted from CP-1252 to UTF-8 and then those bytes converted a
- // second time to UTF-8. CED originally attempted to detect these error cases
- // by using two synthetic encodings, UTF8CP1252 and UTF8UTF8. The intended
- // implementation was to start these just below CP1252 and UTF8 respectively in
- // overall liklihood, and allow 1252 and UTF8 to fall behind if mixtures are
- // found.
- //
- // The UTF8UTF8 encoding is a possible outcome from CED, but unfortunately the
- // UTF8CP1252 internal encoding was added late and not put into encodings.proto,
- // so at the final step it is mapped to UTF8UTF8 also. This was a bad idea and
- // is removed in this November 2011 CL.
- //
- // Mixed encoding detection never worked out as well as envisioned, so the
- // ced_allow_utf8utf8 flag normally disables all this.
- //
- // The effect is that CP-1252 and UTF-8 mixtures will usually be detected as
- // UTF8, and the inputconverter code for UTF8 normally will convert bare
- // CP-1252 bytes to UTF-8, instead of the less-helpful FFFD substitution. UTF-8
- // and double-UTF-8 mixtures will be detected as UTF-8, and the double
- // conversion will stand.
- //
- // However, it is occasionally useful to use CED to detect double-converted
- // UTF-8 coming from third-party data feeds, so they can be fixed at the source.
- // For this purpose, the UTF8UTF8 encoding remains available under the
- // ced_allow_utf8utf8 flag.
- //
- // When UTF8UTF8 is detected, the inputconverter code will undo the double
- // conversion, giving good text.
-
- // Norbert Runge has noted these words in CP1252 that are mistakenly identified
- // as UTF-8 because of the last pair of characters:
- // NESTLÉ® 0xC9 0xAE U+00C9 U+00AE C9AE = U+026E;SMALL LEZH
- // drauß\u2019 0xDF 0x92 U+00DF U+2019 DF92 = U+07D2;NKO LETTER N
- // Mutterschoß\u201c 0xDF 0x93 U+00DF U+201C DF93 = U+07D3;NKO LETTER BA
- // Schoß\u201c 0xDF 0x93 U+00DF U+201C
- // weiß\u201c 0xDF 0x93 U+00DF U+00AB
- // Schnellfuß\u201c 0xDF 0x93 U+00DF U+201C
- // süß« 0xDF 0xAB U+00DF U+00AB DFAB = U+07EB;NKO HIGH TONE
- // These four byte combinations now explicitly boost Latin1/CP1252.
-
- // And for reference, here are a couple of Portuguese spellings
- // that may be mistaken as double-byte encodings.
- // informações 0xE7 0xF5
- // traição 0xE7 0xE3
-
-
- static const char* kVersion = "2.2";
-
- DEFINE_bool(ced_allow_utf8utf8, false, "Allow the UTF8UTF8 encoding, "
- "to handle mixtures of CP1252 "
- "converted to UTF-8 zero, one, "
- "or two times");
- DEFINE_int32(enc_detect_slow_max_kb, 16,
- "Maximum number of Kbytes to examine for "
- "7-bit-only (2022, Hz, UTF7) encoding detect. "
- "You are unlikely to want to change this.");
- DEFINE_int32(enc_detect_fast_max_kb, 256,
- "Maximum number of Kbytes to examine for encoding detect. "
- "You are unlikely to want to change this.");
-
- DEFINE_int32(ced_reliable_difference, 300, "30 * Bits of minimum probablility "
- "difference 1st - 2nd to be considered reliable \n"
- " 2 corresponds to min 4x difference\n"
- " 4 corresponds to min 16x difference\n"
- " 8 corresponds to min 256x difference\n"
- " 10 corresponds to min 1024x difference\n"
- " 20 corresponds to min 1Mx difference.");
-
- // Text debug output options
- DEFINE_bool(enc_detect_summary, false,
- "Print first 16 interesting pairs at exit.");
- DEFINE_bool(counts, false, "Count major-section usage");
-
- // PostScript debug output options
- DEFINE_bool(enc_detect_detail, false,
- "Print PostScript of every update, to stderr.");
- DEFINE_bool(enc_detect_detail2, false,
- "More PostScript detail of every update, to stderr.");
- DEFINE_bool(enc_detect_source, false, "Include source text in detail");
- // Encoding name must exactly match FIRST column of kI18NInfoByEncoding in
- // lang_enc.cc
-
- // Following flags are not in use. Replace them with constants to
- // avoid static initialization.
-
- //DEFINE_string(enc_detect_watch1, "", "Do detail2 about this encoding name.");
- //DEFINE_string(enc_detect_watch2, "", "Do detail2 about this encoding name.");
-
- static const char* const FLAGS_enc_detect_watch1 = "";
- static const char* const FLAGS_enc_detect_watch2 = "";
-
- // Only for experiments. Delete soon.
- DEFINE_bool(force127, false, "Force Latin1, Latin2, Latin7 based on trigrams");
-
- // Demo-mode/debugging experiment
- DEFINE_bool(demo_nodefault, false,
- "Default to all equal; no boost for declared encoding.");
- DEFINE_bool(dirtsimple, false, "Just scan and count for all encodings");
- DEFINE_bool(ced_echo_input, false, "Echo ced input to stderr");
-
-
- static const int XDECILOG2 = 3; // Multiplier for log base 2 ** n/10
- static const int XLOG2 = 30; // Multiplier for log base 2 ** n
-
- static const int kFinalPruneDifference = 10 * XLOG2;
- // Final bits of minimum
- // probability difference 1st-nth
- // to be pruned
-
- static const int kInititalPruneDifference = kFinalPruneDifference * 4;
- // Initial bits of minimum
- // probability difference 1st-nth
- // to be pruned
- //
- static const int kPruneDiffDecrement = kFinalPruneDifference;
- // Decrements bits of minimum
- // probability difference 1st-nth
- // to be pruned
-
- static const int kSmallInitDiff = 2 * XLOG2; // bits of minimum
- // probability difference, base to
- // superset encodings
-
- static const int kBoostInitial = 20 * XLOG2; // bits of boost for
- // initial byte patterns (BOM, 00)
-
- static const int kBadPairWhack = 20 * XLOG2; // bits of whack for
- // one bad pair
-
- static const int kBoostOnePair = 20 * XLOG2; // bits of boost for
- // one good pair in Hz, etc.
-
- static const int kGentleOnePair = 4 * XLOG2; // bits of boost for
- // one good sequence
- //
- static const int kGentlePairWhack = 2 * XLOG2; // bits of whack
- // for ill-formed sequence
-
- static const int kGentlePairBoost = 2 * XLOG2; // bits of boost
- // for well-formed sequence
-
- static const int kDeclaredEncBoost = 5 * XDECILOG2; // bits/10 of boost for
- // best declared encoding per bigram
-
- static const int kBestEncBoost = 5 * XDECILOG2; // bits/10 of boost for
- // best encoding per bigram
-
- static const int kTrigramBoost = 2 * XLOG2; // bits of boost for Latin127 tri
-
- static const int kMaxPairs = 48; // Max interesting pairs to look at
- // If you change this,
- // adjust *PruneDiff*
-
- static const int kPruneMask = 0x07; // Prune every 8 interesting pairs
-
-
- static const int kBestPairsCount = 16; // For first N pairs, do extra boost
- // based on most likely encoding
- // of pair over entire web
-
- static const int kDerateHintsBelow = 12; // If we have fewer than N bigrams,
- // weaken the hints enough that
- // unhinted encodings have a hope of
- // rising to the top
-
- static const int kMinRescanLength = 800; // Don't bother rescanning for
- // unreliable encoding if fewer
- // than this many bytes unscanned.
- // We will rescan at most last half
- // of this.
-
- static const int kStrongBinary = 12; // Make F_BINARY the only encoding
- static const int kWeakerBinary = 4; // Make F_BINARY likely encoding
-
- // These are byte counts from front of file
- static const int kBinaryHardAsciiLimit = 6 * 1024; // Not binary if all ASCII
- static const int kBinarySoftAsciiLimit = 8 * 1024; // " if mostly ASCII
-
- // We try here to avoid having title text dominate the encoding detection,
- // for the not-infrequent error case of title in encoding1, body in encoding2:
- // we want to bias toward encoding2 winning.
- //
- // kMaxBigramsTagTitleText should be a multiple of 2, 3, and 4, so that we
- // rarely cut off mid-character in the original (not-yet-detected) encoding.
- // This matters most for UTF-8 two- and three-byte codes and for
- // Shift-JIS three-byte codes.
- static const int kMaxBigramsTagTitleText = 12; // Keep only some tag text
- static const int kWeightshiftForTagTitleText = 4; // Give text in tags, etc.
- // 1/16 normal weight
-
- static const int kStrongPairs = 6; // Let reliable enc with this many
- // pairs overcome missing hint
-
- enum CEDInternalFlags {
- kCEDNone = 0, // The empty flag
- kCEDRescanning = 1, // Do not further recurse
- kCEDSlowscore = 2, // Do extra scoring
- kCEDForceTags = 4, // Always examine text inside tags
- };
-
- // Forward declaration
- Encoding InternalDetectEncoding(
- CEDInternalFlags flags, const char* text, int text_length,
- const char* url_hint, const char* http_charset_hint,
- const char* meta_charset_hint, const int encoding_hint,
- const Language language_hint, // User interface lang
- const CompactEncDet::TextCorpusType corpus_type,
- bool ignore_7bit_mail_encodings, int* bytes_consumed, bool* is_reliable,
- Encoding* second_best_enc);
-
- typedef struct {
- const uint8* hires[4]; // Pointers to possible high-resolution bigram deltas
- uint8 x_bar; // Average byte2 value
- uint8 y_bar; // Average byte1 value
- uint8 x_stddev; // Standard deviation of byte2 value
- uint8 y_stddev; // Standard deviation of byte1 value
- int so; // Scaling offset -- add to probabilities below
- uint8 b1[256]; // Unigram probability for first byte of aligned bigram
- uint8 b2[256]; // Unigram probability for second byte of aligned bigram
- uint8 b12[256]; // Unigram probability for cross bytes of aligned bigram
- } UnigramEntry;
-
- //typedef struct {
- // uint8 b12[256*256]; // Bigram probability for aligned bigram
- //} FullBigramEntry;
-
-
- // Include all the postproc-generated tables here:
- // RankedEncoding
- // kMapToEncoding
- // unigram_table
- // kMostLIkelyEncoding
- // kTLDHintProbs
- // kCharsetHintProbs
- // HintEntry, kMaxTldKey kMaxTldVector, etc.
- // =============================================================================
-
- #include "compact_enc_det_generated_tables.h"
-
-
- #define F_ASCII F_Latin1 // "ASCII" is a misnomer, so this code uses "Latin1"
-
- #define F_BINARY F_X_BINARYENC // We are mid-update for name change
- #define F_UTF8UTF8 F_X_UTF8UTF8 // We are mid-update for name change
- #define F_BIG5_CP950 F_BIG5 // We are mid-update for name change
- #define F_Unicode F_UTF_16LE // We are mid-update for name change
- // =============================================================================
-
- // 7-bit encodings have at least one "interesting" byte value < 0x80
- // (00 0E 1B + ~)
- // JIS 2022-cn 2022-kr hz utf7
- // Unicode UTF-16 UTF-32
- // 8-bit encodings have no interesting byte values < 0x80
- static const uint32 kSevenBitActive = 0x00000001; // needs <80 to detect
- static const uint32 kUTF7Active = 0x00000002; // <80 and +
- static const uint32 kHzActive = 0x00000004; // <80 and ~
- static const uint32 kIso2022Active = 0x00000008; // <80 and 1B 0E 0F
- static const uint32 kUTF8Active = 0x00000010;
- static const uint32 kUTF8UTF8Active = 0x00000020;
- static const uint32 kUTF1632Active = 0x00000040; // <80 and 00
- static const uint32 kBinaryActive = 0x00000080; // <80 and 00
- static const uint32 kTwobyteCode = 0x00000100; // Needs 8xxx
- static const uint32 kIsIndicCode = 0x00000200; //
- static const uint32 kHighAlphaCode = 0x00000400; // full alphabet in 8x-Fx
- static const uint32 kHighAccentCode = 0x00000800; // accents in 8x-Fx
- static const uint32 kEUCJPActive = 0x00001000; // Have to mess with phase
-
-
- // Debug only. not thread safe
- static int encdet_used = 0;
- static int rescore_used = 0;
- static int rescan_used = 0;
- static int robust_used = 0;
- static int looking_used = 0;
- static int doing_used = 0;
-
-
- // For debugging only -- about 256B/entry times about 500 = 128KB
- // TODO: only allocate this if being used
- typedef struct {
- int offset;
- int best_enc; // Best ranked encoding for this bigram, or
- // -1 for overhead entries
- string label;
- int detail_enc_prob[NUM_RANKEDENCODING];
- } DetailEntry;
-
- static int watch1_rankedenc = -1; // Debug. not threadsafe
- static int watch2_rankedenc = -1; // Debug. not threadsafe
- ////static int next_detail_entry = 0; // Debug. not threadsafe
- ////static DetailEntry details[kMaxPairs * 10]; // Allow 10 details per bigram
- // End For debugging only
-
- // Must match kTestPrintableAsciiTildePlus exit codes, minus one
- enum PairSet {AsciiPair = 0, OtherPair = 1, NUM_PAIR_SETS = 2};
-
- // The reasons for pruning
- enum PruneReason {PRUNE_NORMAL, PRUNE_SLOWEND, PRUNE_FINAL};
-
- static const char* kWhatSetName[] = {"Ascii", "Other"};
-
-
- // State for encodings that do shift-out/shift-in between one- and two-byte
- // regions (ISO-2022-xx, HZ)
- enum StateSoSi {SOSI_NONE, SOSI_ERROR, SOSI_ONEBYTE, SOSI_TWOBYTE};
-
- typedef struct {
- const uint8* initial_src; // For calculating byte offsets
- const uint8* limit_src; // Range of input source
- const uint8* prior_src; // Source consumed by prior call to BoostPrune
- const uint8* last_pair; // Last pair inserted into interesting_pairs
-
- DetailEntry* debug_data; // Normally NULL. Ptr to debug data for
- // FLAGS_enc_detect_detail PostScript data
- int next_detail_entry; // Debug
-
- bool done;
- bool reliable;
- bool hints_derated;
- int declared_enc_1; // From http/meta hint
- int declared_enc_2; // from http/meta hint
- int prune_count; // Number of times we have pruned
-
- int trigram_highwater_mark; // Byte offset of last trigram processing
- bool looking_for_latin_trigrams; // True if we should test for doing
- // Latin1/2/7 trigram processing
- bool do_latin_trigrams; // True if we actually are scoring trigrams
-
- // Miscellaneous state variables for difficult encodings
- int binary_quadrants_count; // Number of four bigram quadrants seen:
- // 0xxxxxxx0xxxxxxx 0xxxxxxx1xxxxxx
- // 1xxxxxxx0xxxxxxx 1xxxxxxx1xxxxxx
- int binary_8x4_count; // Number of 8x4 buckets seen:
- uint32 binary_quadrants_seen; // Bit[i] set if bigram i.......i....... seen
- uint32 binary_8x4_seen; // Bit[i] set if bigram iii.....ii...... seen
- int utf7_starts; // Count of possible UTF-7 beginnings seen
- int prior_utf7_offset; // Source consumed by prior UTF-7 string
- int next_utf8_ministate; // Mini state for UTF-8 sequences
- int utf8_minicount[6]; // Number of correct 2- 3- 4-byte seq, errors
- int next_utf8utf8_ministate; // Mini state for UTF8UTF8 sequences
- int utf8utf8_odd_byte; // UTF8UTF8 seq has odd number of bytes
- int utf8utf8_minicount[6]; // Number of correct 2- 3- 4-byte seq, errors
- StateSoSi next_2022_state; // Mini state for 2022 sequences
- StateSoSi next_hz_state; // Mini state for HZ sequences
- bool next_eucjp_oddphase; // Mini state for EUC-JP sequences
- int byte32_count[8]; // Count of top 3 bits of byte1 of bigram
- // 0x1x 2x3x 4x5x 6x7x 8x9x AxBx CxDx ExFx
- uint32 active_special; // Bits showing which special cases are active
-
- Encoding tld_hint; // Top TLD encoding or UNKNOWN
- Encoding http_hint; // What the document says about itself or
- Encoding meta_hint; // UNKNOWN_ENCODING. BOM is initial byte
- Encoding bom_hint; // order mark for UTF-xx
-
- // small cache of previous interesting bigrams
- int next_prior_bigram;
- int prior_bigram[4];
- int prior_binary[1];
-
- int top_rankedencoding; // Top two probabilities and families
- int second_top_rankedencoding;
- int top_prob;
- int second_top_prob;
- int prune_difference; // Prune things this much below the top prob
- int rankedencoding_list_len; // Number of active encodings
- int rankedencoding_list[NUM_RANKEDENCODING]; // List of active encodings
- //
- int enc_prob[NUM_RANKEDENCODING]; // Cumulative probability per enc
- // This is where all the action is
- int hint_prob[NUM_RANKEDENCODING]; // Initial hint probabilities
- int hint_weight[NUM_RANKEDENCODING]; // Number of hints for this enc
-
- // Two sets -- one for printable ASCII, one for the rest
- int prior_interesting_pair[NUM_PAIR_SETS]; // Pairs consumed by prior call
- int next_interesting_pair[NUM_PAIR_SETS]; // Next pair to write
- char interesting_pairs[NUM_PAIR_SETS][kMaxPairs * 2]; // Two bytes per pair
- int interesting_offsets[NUM_PAIR_SETS][kMaxPairs]; // Src offset of pair
- int interesting_weightshift[NUM_PAIR_SETS][kMaxPairs]; // weightshift of pair
- } DetectEncodingState;
-
-
- // Record a debug event that changes probabilities
- void SetDetailsEncProb(DetectEncodingState* destatep,
- int offset, int best_enc, const char* label) {
- int next = destatep->next_detail_entry;
- destatep->debug_data[next].offset = offset;
- destatep->debug_data[next].best_enc = best_enc;
- destatep->debug_data[next].label = label;
- memcpy(&destatep->debug_data[next].detail_enc_prob,
- &destatep->enc_prob,
- sizeof(destatep->enc_prob));
- ++destatep->next_detail_entry;
- }
-
- // Record a debug event that changes probabilities, copy offset
- void SetDetailsEncProbCopyOffset(DetectEncodingState* destatep,
- int best_enc, const char* label) {
- int next = destatep->next_detail_entry;
- destatep->debug_data[next].offset = destatep->debug_data[next - 1].offset;
- destatep->debug_data[next].best_enc = best_enc;
- destatep->debug_data[next].label = label;
- memcpy(&destatep->debug_data[next].detail_enc_prob,
- &destatep->enc_prob,
- sizeof(destatep->enc_prob));
- ++destatep->next_detail_entry;
- }
-
- // Record a debug event that changes probs and has simple text label
- void SetDetailsEncLabel(DetectEncodingState* destatep, const char* label) {
- int next = destatep->next_detail_entry;
- destatep->debug_data[next].offset = destatep->debug_data[next - 1].offset;
- destatep->debug_data[next].best_enc = -1;
- destatep->debug_data[next].label = label;
- memcpy(&destatep->debug_data[next].detail_enc_prob,
- &destatep->enc_prob,
- sizeof(destatep->enc_prob));
- ++destatep->next_detail_entry;
- }
-
- // Record a debug event that is just a text label, no change in probs
- void SetDetailsLabel(DetectEncodingState* destatep, const char* label) {
- int next = destatep->next_detail_entry;
- destatep->debug_data[next].offset = destatep->debug_data[next - 1].offset;
- destatep->debug_data[next].best_enc = -1;
- destatep->debug_data[next].label = label;
- memcpy(&destatep->debug_data[next].detail_enc_prob,
- &destatep->debug_data[next - 1].detail_enc_prob,
- sizeof(destatep->enc_prob));
- ++destatep->next_detail_entry;
- }
-
-
- // Maps superset encodings to base, to see if 2 encodings are compatible
- // (Non-identity mappings are marked "-->" below.)
- static const Encoding kMapEncToBaseEncoding[] = {
- ISO_8859_1, // 0: Teragram ASCII
- ISO_8859_2, // 1: Teragram Latin2
- ISO_8859_3, // 2: in BasisTech but not in Teragram
- ISO_8859_4, // 3: Teragram Latin4
- ISO_8859_5, // 4: Teragram ISO-8859-5
- ISO_8859_6, // 5: Teragram Arabic
- ISO_8859_7, // 6: Teragram Greek
- MSFT_CP1255, // 7: Teragram Hebrew --> 36
- ISO_8859_9, // 8: in BasisTech but not in Teragram
- ISO_8859_10, // 9: in BasisTech but not in Teragram
- JAPANESE_EUC_JP, // 10: Teragram EUC_JP
- JAPANESE_SHIFT_JIS, // 11: Teragram SJS
- JAPANESE_JIS, // 12: Teragram JIS
- CHINESE_BIG5, // 13: Teragram BIG5
- CHINESE_GB, // 14: Teragram GB
- CHINESE_EUC_CN, // 15: Teragram EUC-CN
- KOREAN_EUC_KR, // 16: Teragram KSC
- UNICODE, // 17: Teragram Unicode
- CHINESE_EUC_CN, // 18: Teragram EUC --> 15
- CHINESE_EUC_CN, // 19: Teragram CNS --> 15
- CHINESE_BIG5, // 20: Teragram BIG5_CP950 --> 13
- JAPANESE_SHIFT_JIS, // 21: Teragram CP932 --> 11
- UTF8, // 22
- UNKNOWN_ENCODING, // 23
- ISO_8859_1, // 24: ISO_8859_1 with all characters <= 127 --> 0
- RUSSIAN_KOI8_R, // 25: Teragram KOI8R
- RUSSIAN_CP1251, // 26: Teragram CP1251
- ISO_8859_1, // 27: CP1252 aka MSFT euro ascii --> 0
- RUSSIAN_KOI8_RU, // 28: CP21866 aka KOI8_RU, used for Ukrainian
- MSFT_CP1250, // 29: CP1250 aka MSFT eastern european
- ISO_8859_1, // 30: aka ISO_8859_0 aka ISO_8859_1 euroized --> 0
- ISO_8859_9, // 31: used for Turkish
- ISO_8859_13, // 32: used in Baltic countries --> 43
- ISO_8859_11, // 33: aka TIS-620, used for Thai
- ISO_8859_11, // 34: used for Thai --> 33
- MSFT_CP1256, // 35: used for Arabic
- MSFT_CP1255, // 36: Logical Hebrew Microsoft
- MSFT_CP1255, // 37: Iso Hebrew Logical --> 36
- MSFT_CP1255, // 38: Iso Hebrew Visual --> 36
- CZECH_CP852, // 39
- ISO_8859_2, // 40: aka ISO_IR_139 aka KOI8_CS --> 1
- MSFT_CP1253, // 41: used for Greek, but NOT a superset of 8859-7
- RUSSIAN_CP866, // 42
- ISO_8859_13, // 43
- ISO_2022_KR, // 44
- CHINESE_GB, // 45 GBK --> 14
- CHINESE_GB, // 46 GB18030 --> 14
- CHINESE_BIG5, // 47 BIG5_HKSCS --> 13
- ISO_2022_KR, // 48 ISO_2022_CN --> 44
- TSCII, // 49 Indic encoding
- TAMIL_MONO, // 50 Indic encoding - Tamil
- TAMIL_BI, // 51 Indic encoding - Tamil
- JAGRAN, // 52 Indic encoding - Devanagari
- MACINTOSH_ROMAN, // 53
- UTF7, // 54
- BHASKAR, // 55 Indic encoding - Devanagari
- HTCHANAKYA, // 56 Indic encoding - Devanagari
- UTF16BE, // 57
- UTF16LE, // 58
- UTF32BE, // 59
- UTF32LE, // 60
- BINARYENC, // 61
- HZ_GB_2312, // 62
- UTF8UTF8, // 63
- TAM_ELANGO, // 64 Elango - Tamil
- TAM_LTTMBARANI, // 65 Barani - Tamil
- TAM_SHREE, // 66 Shree - Tamil
- TAM_TBOOMIS, // 67 TBoomis - Tamil
- TAM_TMNEWS, // 68 TMNews - Tamil
- TAM_WEBTAMIL, // 69 Webtamil - Tamil
- KDDI_SHIFT_JIS, // 70 KDDI Shift_JIS
- DOCOMO_SHIFT_JIS, // 71 DoCoMo Shift_JIS
- SOFTBANK_SHIFT_JIS, // 72 SoftBank Shift_JIS
- KDDI_ISO_2022_JP, // 73 KDDI ISO-2022-JP
- SOFTBANK_ISO_2022_JP, // 74 SOFTBANK ISO-2022-JP
- };
-
- COMPILE_ASSERT(arraysize(kMapEncToBaseEncoding) == NUM_ENCODINGS,
- kMapEncToBaseEncoding_has_incorrect_size);
-
- // Maps base encodings to 0, supersets to 1+, undesired to -1
- // (Non-identity mappings are marked "-->" below.)
- static const int kMapEncToSuperLevel[] = {
- 0, // 0: Teragram ASCII
- 0, // 1: Teragram Latin2
- 0, // 2: in BasisTech but not in Teragram
- 0, // 3: Teragram Latin4
- 0, // 4: Teragram ISO-8859-5
- 0, // 5: Teragram Arabic
- 0, // 6: Teragram Greek
- 0, // 7: Teragram Hebrew
- 0, // 8: in BasisTech but not in Teragram
- 0, // 9: in BasisTech but not in Teragram
- 0, // 10: Teragram EUC_JP
- 0, // 11: Teragram SJS
- 0, // 12: Teragram JIS
- 0, // 13: Teragram BIG5
- 0, // 14: Teragram GB
- 0, // 15: Teragram EUC-CN
- 0, // 16: Teragram KSC
- 0, // 17: Teragram Unicode
- -1, // 18: Teragram EUC --> 15
- -1, // 19: Teragram CNS --> 15
- 1, // 20: Teragram BIG5_CP950 --> 13
- 1, // 21: Teragram CP932 --> 11
- 0, // 22
- -1, // 23
- -1, // 24: ISO_8859_1 with all characters <= 127 --> 0
- 0, // 25: Teragram KOI8R
- 0, // 26: Teragram CP1251
- 1, // 27: CP1252 aka MSFT euro ascii --> 0
- 0, // 28: CP21866 aka KOI8_RU, used for Ukrainian
- 0, // 29: CP1250 aka MSFT eastern european
- 1, // 30: aka ISO_8859_0 aka ISO_8859_1 euroized --> 0
- 0, // 31: used for Turkish
- 1, // 32: used in Baltic countries --> 43
- 0, // 33: aka TIS-620, used for Thai
- 1, // 34: used for Thai --> 33
- 0, // 35: used for Arabic
- 0, // 36: Logical Hebrew Microsoft
- -1, // 37: Iso Hebrew Logical --> 36
- -1, // 38: Iso Hebrew Visual --> 7
- 0, // 39
- 1, // 40: aka ISO_IR_139 aka KOI8_CS --> 1
- 0, // 41: used for Greek, NOT superset of 8859-7
- 0, // 42
- 0, // 43
- 0, // 44
- 1, // 45 GBK --> 14
- 1, // 46 GB18030 --> 14
- 1, // 47 BIG5_HKSCS --> 13
- 1, // 48 ISO_2022_CN --> 44
- 0, // 49 Indic encoding
- 0, // 50 Indic encoding - Tamil
- 0, // 51 Indic encoding - Tamil
- 0, // 52 Indic encoding - Devanagari
- 0, // 53
- 0, // 54
- 0, // 55 Indic encoding - Devanagari
- 0, // 56 Indic encoding - Devanagari
- 0, // 57
- 0, // 58
- 0, // 59
- 0, // 60
- 0, // 61
- 0, // 62
- 2, // 63
- 0, 0, 0, 0, 0, 0, // add six more Tamil
- 0, 0, 0, 0, 0, // add five encodings with emoji
- };
-
- COMPILE_ASSERT(arraysize(kMapEncToSuperLevel) == NUM_ENCODINGS,
- kMapEncToSuperLevel_has_incorrect_size);
-
-
-
- // Subscripted by Encoding enum value
- static const uint32 kSpecialMask[] = {
- kHighAccentCode, // 0
- kHighAccentCode,
- kHighAccentCode,
- kHighAccentCode,
- kHighAlphaCode, // 4
- kHighAlphaCode,
- kHighAlphaCode,
- kHighAlphaCode,
- kHighAccentCode,
- kHighAccentCode,
-
- kTwobyteCode + kEUCJPActive, // 10 euc-jp
- kTwobyteCode,
- kSevenBitActive + kIso2022Active, // jis
- kTwobyteCode,
- kTwobyteCode,
- kTwobyteCode,
- kTwobyteCode,
- kSevenBitActive + kUTF1632Active, // Unicode
- kTwobyteCode,
- kTwobyteCode,
-
- kTwobyteCode, // 20
- kTwobyteCode,
- kUTF8Active, // UTF-8
- 0,
- 0,
- kHighAlphaCode, // 25
- kHighAlphaCode,
- kHighAccentCode,
- kHighAlphaCode,
- kHighAccentCode,
-
- kHighAccentCode, // 30
- kHighAccentCode,
- kHighAccentCode,
- kHighAlphaCode,
- kHighAlphaCode,
- kHighAlphaCode, // 35
- kHighAlphaCode,
- kHighAlphaCode,
- kHighAlphaCode,
- 0,
-
- 0, // 40
- kHighAlphaCode,
- kHighAlphaCode,
- kHighAccentCode,
- kSevenBitActive + kIso2022Active, // 2022-kr
- kTwobyteCode,
- kTwobyteCode,
- kTwobyteCode,
- kSevenBitActive + kIso2022Active, // 2022-cn
- kHighAlphaCode + kIsIndicCode, // 49 TSCII
-
- kHighAlphaCode + kIsIndicCode, // 50 TAMIL_MONO
- kHighAlphaCode + kIsIndicCode, // 51 TAMIL_BI
- kHighAlphaCode + kIsIndicCode, // 52 JAGRAN
- kHighAccentCode, // 53 MACINTOSH_ROMAN
- kSevenBitActive + kUTF7Active, // 54 UTF-7
- kHighAlphaCode + kIsIndicCode, // 55 BHASKAR Indic encoding - Devanagari
- kHighAlphaCode + kIsIndicCode, // 56 HTCHANAKYA Indic encoding - Devanagari
- kSevenBitActive + kUTF1632Active, // 57 UTF16BE
- kSevenBitActive + kUTF1632Active, // 58 UTF16LE
- kSevenBitActive + kUTF1632Active, // 59 UTF32BE
- kSevenBitActive + kUTF1632Active, // 60 UTF32LE
-
- kSevenBitActive + kBinaryActive, // 61 BINARYENC
- kSevenBitActive + kHzActive, // 62 HZ_GB_2312
- kHighAccentCode + kUTF8Active + kUTF8UTF8Active, // 63 UTF8UTF8
- kHighAlphaCode + kIsIndicCode, // 64 Elango - Tamil
- kHighAlphaCode + kIsIndicCode, // 65 Barani - Tamil
- kHighAlphaCode + kIsIndicCode, // 66 Shree - Tamil
- kHighAlphaCode + kIsIndicCode, // 67 TBoomis - Tamil
- kHighAlphaCode + kIsIndicCode, // 68 TMNews - Tamil
- kHighAlphaCode + kIsIndicCode, // 69 Webtamil - Tamil
- kTwobyteCode, // 70 KDDI Shift_JIS
- kTwobyteCode, // 71 DoCoMo Shift_JIS
- kTwobyteCode, // 72 SoftBank Shift_JIS
- kSevenBitActive + kIso2022Active, // 73 KDDI-ISO-2022-JP
- kSevenBitActive + kIso2022Active, // 74 SOFTBANK-ISO-2022-JP
- };
-
- COMPILE_ASSERT(arraysize(kSpecialMask) == NUM_ENCODINGS,
- kSpecialMask_has_incorrect_size);
-
-
- /***
- kHighAlphaCode -- full alphabet in 8x-Fx range, not just accents
-
- ISO_8859_5, // 4: Teragram ISO-8859-5 Cyrl UL bd
- RUSSIAN_CP1251, // 26: Teragram CP1251 UL cdef
- RUSSIAN_KOI8_R, // 25: Teragram KOI8R LU cdef
- RUSSIAN_KOI8_RU, // 28: CP21866 aka KOI8_RU, LU cdef
- RUSSIAN_CP866, // 42 89ae
-
- ISO_8859_6, // 5: Teragram Arabic nocase cde
- MSFT_CP1256, // 35: used for Arabic nocase cde
-
- ISO_8859_7, // 6: Teragram Greek UL cdef
- MSFT_CP1253, // 41: used for Greek UL cdef
-
- ISO_8859_8, // 7: Teragram Hebrew nocase ef
- MSFT_CP1255, // 36: Logical Hebrew Microsoft nocase ef
- ISO_8859_8_I, // 37: Iso Hebrew Logical nocase ef
- HEBREW_VISUAL, // 38: Iso Hebrew Visual nocase ef
-
- ISO_8859_11, // 33: aka TIS-620, used for Thai nocase abcde
- MSFT_CP874, // 34: used for Thai nocase abcde
-
- TSCII, // 49 8-f
- TAMIL_MONO, // 50
- TAMIL_BI, // 51
- JAGRAN, // 52
- BHASKAR, // 55 Indic encoding - Devanagari
- HTCHANAKYA, // 56 Indic encoding - Devanagari
- ***/
-
- // We can scan bytes using this at about 500 MB/sec 2.8GHz P4
- // Slow scan uses this, stopping on NUL ESC SO SI bad C0 and + ~
- // We allow FF, 0x0C, here because it gives a better result for old
- // Ascii text formatted for a TTY
- // non-zero exits scan loop -- 1 for printable ASCII, 2 otherwise
- static const char kTestPrintableAsciiTildePlus[256] = {
- 2,2,2,2,2,2,2,2, 2,0,0,2,0,0,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
- 0,0,0,0,0,0,0,0, 0,0,0,1,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,1,2,
-
- 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
- 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
- 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
- 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
- };
-
- // We can scan bytes using this at about 550 MB/sec 2.8GHz P4
- // Slow scan uses this, stopping on NUL ESC SO SI and bad C0
- // after Hz and UTF7 are pruned away
- // We allow Form Feed, 0x0C, here
- static const char kTestPrintableAscii[256] = {
- 2,2,2,2,2,2,2,2, 2,0,0,2,0,0,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
- 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,2,
-
- 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
- 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
- 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
- 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
- };
-
- // Used in first-four-byte testing
- static const char kIsPrintableAscii[256] = {
- 0,0,0,0,0,0,0,0, 0,1,1,0,0,1,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
- 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
- 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
- 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,0,
-
- 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
- };
-
-
- static const signed char kBase64Value[256] = {
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,62,-1,-1,-1,63,
- 52,53,54,55,56,57,58,59, 60,61,-1,-1,-1,-1,-1,-1,
-
- -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,
- 15,16,17,18,19,20,21,22, 23,24,25,-1,-1,-1,-1,-1,
- -1,26,27,28,29,30,31,32, 33,34,35,36,37,38,39,40,
- 41,42,43,44,45,46,47,48, 49,50,51,-1,-1,-1,-1,-1,
-
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
-
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
- -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,
- };
-
-
- // Subscripted by <state, byte/16>
- // Accepts Cx->8x Dx->8x Ex->8x->8x Fx->8x->8x->8x
- //
- // Fixed Problem: GB has sequences like B2DB B8D6 BDE1 B9B9
- // which we can mis-parse as an error byte followed by good UTF-8:
- // B2 DBB8 D6BD E1B9B9
- // To counteract this, we now require an ASCII7 byte to resync out
- // of the error state
- // Next problem: good UTF-8 with bad byte
- // efbc a012 eea4 bee7 b280 c2b7
- // efbca0 12 eea4be e7b280 c2b7
- // ^^ bad byte
- // fix: change state0 byte 1x to be don't-care
- //
- // Short UTF-8 ending in ASCII7 byte should resync immediately:
- // E0 20 E0 A6 AA should give one error and resync at 2nd E0
- //
- static const char kMiniUTF8State[8][16] = {
- {0,0,0,0,0,0,0,0, 7,7,7,7,1,1,2,4,}, // [0] start char (allow cr/lf/ht)
- {0,7,0,0,0,0,0,0, 0,0,0,0,7,7,7,7,}, // [1] continue 1 of 2
- {0,7,0,0,0,0,0,0, 3,3,3,3,7,7,7,7,}, // [2] continue 1 of 3
- {0,7,0,0,0,0,0,0, 0,0,0,0,7,7,7,7,}, // [3] continue 2 of 3
- {0,7,0,0,0,0,0,0, 5,5,5,5,7,7,7,7,}, // [4] continue 1 of 4
- {0,7,0,0,0,0,0,0, 6,6,6,6,7,7,7,7,}, // [5] continue 2 of 4
- {0,7,0,0,0,0,0,0, 0,0,0,0,7,7,7,7,}, // [6] continue 3 of 4
- {0,7,0,0,0,0,0,0, 7,7,7,7,7,7,7,7,}, // [7] error, soak up continues,
- // ONLY resync after Ascii char
- // then restart
- };
- // Counter to increment: 0-don'tcare 1-error 2-good_2B 3-good_3B 4-good_4B
- static const char kMiniUTF8Count[8][16] = {
- {0,0,0,0,0,0,0,0, 1,1,1,1,0,0,0,0,}, // [0] start char (allow cr/lf/ht)
- {1,1,1,1,1,1,1,1, 2,2,2,2,1,1,1,1,}, // [1] continue 1 of 2
- {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [2] continue 1 of 3
- {1,1,1,1,1,1,1,1, 3,3,3,3,1,1,1,1,}, // [3] continue 2 of 3
- {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [4] continue 1 of 4
- {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [5] continue 2 of 4
- {1,1,1,1,1,1,1,1, 4,4,4,4,1,1,1,1,}, // [6] continue 3 of 4
- {0,1,0,0,0,0,0,0, 1,1,1,1,1,1,1,1,}, // [7] error, soak up continues,
- // then restart
- };
-
- // Subscripted by <state, f(byte1) + g(byte2)>
- // where f(x)= E2->4, Cx->8 and C3->12 and 0 otherwise
- // and g(x) = (x >> 4) & 3 8x->0 9x->1 Ax->2 Bx->3 Cx->0, etc.
- // (no checking for illegal bytes)
- // Here are example patterns of CP1252 converted to UTF-8 0/1/2 times. We want
- // to detect two, so we can back-convert to one.
- // zero one two pattern
- // ---- ------ ---------------- -----------------
- // 81 C281 C382C281 C3->8x->C2->xx
- // 98 CB9C C38BC593 C3->8x->C5->xx
- // C3 C383 C383C692 C3->8x->C6->xx
- // C8 C388 C383CB86 C3->8x->CB->xx
- // 83 C692 C386E28099 C3->8x->E2->xx->8x
- // 80 E282AC C3A2E2809AC2AC C3->A2->E2->xx->xx->Cx->xx
- // 92 E28099 C3A2E282ACE284A2 C3->A2->E2->xx->xx->E2->xx->xx
- //
- // We also want to detect bare-byte extra UTF-8 conversions:
- // zero one two pattern
- // ---- ------ ---------------- -----------------
- // C3 C3 C383 C3->8x->C2->xx
- // D3 D3 C393 C3->9x->C2->xx->C2->xx
- // E3 E3 C3A3 C3->Ax->C2->xx->C2->xx->C2->xx
- // F3 F3 C3B2 C3->Bx->C2->xx->C2->xx->C2->xx->C2->xx
- //
-
- /**
- CP1252 => UTF8 => UTF8UTF8
- 80 => E282AC => C3A2E2809AC2AC
- 81 => C281 => C382C281
- 82 => E2809A => C3A2E282ACC5A1
- 83 => C692 => C386E28099
- 84 => E2809E => C3A2E282ACC5BE
- 85 => E280A6 => C3A2E282ACC2A6
- 86 => E280A0 => C3A2E282ACC2A0
- 87 => E280A1 => C3A2E282ACC2A1
- 88 => CB86 => C38BE280A0
- 89 => E280B0 => C3A2E282ACC2B0
- 8A => C5A0 => C385C2A0
- 8B => E280B9 => C3A2E282ACC2B9
- 8C => C592 => C385E28099
- 8D => C28D => C382C28D
- 8E => C5BD => C385C2BD
- 8F => C28F => C382C28F
- 90 => C290 => C382C290
- 91 => E28098 => C3A2E282ACCB9C
- 92 => E28099 => C3A2E282ACE284A2
- 93 => E2809C => C3A2E282ACC593
- 94 => E2809D => C3A2E282ACC29D
- 95 => E280A2 => C3A2E282ACC2A2
- 96 => E28093 => C3A2E282ACE2809C
- 97 => E28094 => C3A2E282ACE2809D
- 98 => CB9C => C38BC593
- 99 => E284A2 => C3A2E2809EC2A2
- 9A => C5A1 => C385C2A1
- 9B => E280BA => C3A2E282ACC2BA
- 9C => C593 => C385E2809C
- 9D => C29D => C382C29D
- 9E => C5BE => C385C2BE
- 9F => C5B8 => C385C2B8
- A0 => C2A0 => C382C2A0
- A1 => C2A1 => C382C2A1
- A2 => C2A2 => C382C2A2
- A3 => C2A3 => C382C2A3
- A4 => C2A4 => C382C2A4
- A5 => C2A5 => C382C2A5
- A6 => C2A6 => C382C2A6
- A7 => C2A7 => C382C2A7
- A8 => C2A8 => C382C2A8
- A9 => C2A9 => C382C2A9
- AA => C2AA => C382C2AA
- AB => C2AB => C382C2AB
- AC => C2AC => C382C2AC
- AD => C2AD => C382C2AD
- AE => C2AE => C382C2AE
- AF => C2AF => C382C2AF
- B0 => C2B0 => C382C2B0
- B1 => C2B1 => C382C2B1
- B2 => C2B2 => C382C2B2
- B3 => C2B3 => C382C2B3
- B4 => C2B4 => C382C2B4
- B5 => C2B5 => C382C2B5
- B6 => C2B6 => C382C2B6
- B7 => C2B7 => C382C2B7
- B8 => C2B8 => C382C2B8
- B9 => C2B9 => C382C2B9
- BA => C2BA => C382C2BA
- BB => C2BB => C382C2BB
- BC => C2BC => C382C2BC
- BD => C2BD => C382C2BD
- BE => C2BE => C382C2BE
- BF => C2BF => C382C2BF
- C0 => C380 => C383E282AC
- C1 => C381 => C383C281
- C2 => C382 => C383E2809A
- C3 => C383 => C383C692
- C4 => C384 => C383E2809E
- C5 => C385 => C383E280A6
- C6 => C386 => C383E280A0
- C7 => C387 => C383E280A1
- C8 => C388 => C383CB86
- C9 => C389 => C383E280B0
- CA => C38A => C383C5A0
- CB => C38B => C383E280B9
- CC => C38C => C383C592
- CD => C38D => C383C28D
- CE => C38E => C383C5BD
- CF => C38F => C383C28F
- D0 => C390 => C383C290
- D1 => C391 => C383E28098
- D2 => C392 => C383E28099
- D3 => C393 => C383E2809C
- D4 => C394 => C383E2809D
- D5 => C395 => C383E280A2
- D6 => C396 => C383E28093
- D7 => C397 => C383E28094
- D8 => C398 => C383CB9C
- D9 => C399 => C383E284A2
- DA => C39A => C383C5A1
- DB => C39B => C383E280BA
- DC => C39C => C383C593
- DD => C39D => C383C29D
- DE => C39E => C383C5BE
- DF => C39F => C383C5B8
- E0 => C3A0 => C383C2A0
- E1 => C3A1 => C383C2A1
- E2 => C3A2 => C383C2A2
- E3 => C3A3 => C383C2A3
- E4 => C3A4 => C383C2A4
- E5 => C3A5 => C383C2A5
- E6 => C3A6 => C383C2A6
- E7 => C3A7 => C383C2A7
- E8 => C3A8 => C383C2A8
- E9 => C3A9 => C383C2A9
- EA => C3AA => C383C2AA
- EB => C3AB => C383C2AB
- EC => C3AC => C383C2AC
- ED => C3AD => C383C2AD
- EE => C3AE => C383C2AE
- EF => C3AF => C383C2AF
- F0 => C3B0 => C383C2B0
- F1 => C3B1 => C383C2B1
- F2 => C3B2 => C383C2B2
- F3 => C3B3 => C383C2B3
- F4 => C3B4 => C383C2B4
- F5 => C3B5 => C383C2B5
- F6 => C3B6 => C383C2B6
- F7 => C3B7 => C383C2B7
- F8 => C3B8 => C383C2B8
- F9 => C3B9 => C383C2B9
- FA => C3BA => C383C2BA
- FB => C3BB => C383C2BB
- FC => C3BC => C383C2BC
- FD => C3BD => C383C2BD
- FE => C3BE => C383C2BE
- FF => C3BF => C383C2BF
- **/
-
- // Subscripted by <state, f(byte1) + g(byte2)>
- // where f(x)= E2->4, C2/5/6/B->8 and C3->12 and 0 otherwise
- // and g(x) = (x >> 4) & 3 8x->0 9x->1 Ax->2 Bx->3 Cx->0, etc.
-
- // 81 C281 C382C281 C3->8x->C2->xx
- // 98 CB9C C38BC593 C3->8x->C5->xx
- // C3 C383 C383C692 C3->8x->C6->xx
- // C8 C388 C383CB86 C3->8x->CB->xx
- // [0] [2] [0]
- // 83 C692 C386E28099 C3->8x->E2->xx->xx
- // odd_byte=0 [0] [2] [0+] odd_byte flipped
- // odd_byte=1 [0+] [2] [0] [0] odd_byte unflipped
- // 80 E282AC C3A2E2809AC2AC C3->A2->E2->xx->xx->Cx->xx
- // odd_byte=0 [0] [3] [4] [0+]
- // odd_byte=1 [0+] [3] [4] [4] [0]
- // 92 E28099 C3A2E282ACE284A2 C3->A2->E2->xx->xx->E2->xx->xx
- // odd_byte=0 [0] [3] [4] [0] [0]
- // odd_byte=1 [0+] [3] [4] [4] [0+]
- //
- // When an E2xxxx sequence is encountered, we absorb the two bytes E2xx and flip
- // the odd_byte state. If that goes from 0 to 1, the next pair is offset up
- // by one byte, picking up the two bytes just after E2xxxx. If odd_byte goes
- // from 1 to 0, the next two bytes picked up are the two bytes xxxx of E2xxxx.
- // These are absorbed with no error in state 0 or state 4
- //
- // C3 C3 C383 C3->8x->C2->xx
- // D3 D3 C393 C3->9x->C2->xx->C2->xx
- // E3 E3 C3A3 C3->Ax->C2->xx->C2->xx->C2->xx
- // F3 F3 C3B2 C3->Bx->C2->xx->C2->xx->C2->xx->C2->xx
- // Counter3 for Fx Ex sequences is incremented at last C2
-
- static const char kMiniUTF8UTF8State[8][16] = {
- // xxxx E2xx CXxx C3xx
- // 8 9 a b 8 9 a b 8 9 a b
- {0,0,0,0,1,1,1,1, 1,1,1,1,2,2,3,5,}, // [0] looking for C38x/C3Ax/2020/8x8x, or err
- {0,0,0,0,1,1,1,1, 1,1,1,1,2,2,3,5,}, // [1] error, back to looking
- {1,1,1,1,0,0,0,0, 0,0,0,0,1,1,1,1,}, // [2] C38x looking for CXxx/E2xxxx
- // + + + + // E2xxxx flips odd_byte
- {1,1,1,1,4,4,4,4, 7,7,7,7,1,1,1,1,}, // [3] C3Ax looking for E2xx or C2xxC2xx
- // + + + + // E2xxxx flips odd_byte
- {4,4,4,4,0,0,0,0, 0,0,0,0,1,1,1,1,}, // [4] C3AxE2xx-- looking for C2xx/E2xxxx
- // + + + + // E2xxxx flips odd_byte
- {1,1,1,1,1,1,1,1, 6,6,6,6,1,1,1,1,}, // [5] C3Bx -- looking for C2xxC2xxC2xx
- {1,1,1,1,1,1,1,1, 7,7,7,7,1,1,1,1,}, // [6] C3Bx -- looking for C2xxC2xx
- {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [7] C3Bx -- looking for C2xx
- };
- // Counter to increment: 0-don'tcare 1-error 2-good_2B 3-good_3B 4-good_4B
- static const char kMiniUTF8UTF8Count[8][16] = {
- // xxxx E2xx C2Xx C3xx
- // 8 9 a b 8 9 a b 8 9 a b
- {0,0,0,0,1,1,1,1, 1,1,1,1,0,0,0,0,}, // [0] looking for C38x/C3Ax/2020/8x8x, or err
- {0,0,0,0,1,1,1,1, 1,1,1,1,0,0,0,0,}, // [1] error, back to looking
- {1,1,1,1,3,3,3,3, 2,2,2,2,1,1,1,1,}, // [2] C38x looking for CXxx/E2xxxx
- // + + + + // E2xxxx flips odd_byte
- {1,1,1,1,0,0,0,0, 0,0,0,0,1,1,1,1,}, // [3] C3Ax looking for E2xx
- // + + + + // E2xxxx flips odd_byte
- {1,1,1,1,4,4,4,4, 4,4,4,4,1,1,1,1,}, // [4] C3AxE2xx-- looking for C2xx/E2xxxx
- // + + + + // E2xxxx flips odd_byte
- {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [5] C3Bx -- looking for C2xxC2xxC2xx
- {1,1,1,1,1,1,1,1, 0,0,0,0,1,1,1,1,}, // [6] C3Bx -- looking for C2xxC2xx
- {1,1,1,1,1,1,1,1, 3,3,3,3,1,1,1,1,}, // [7] C3Bx -- looking for C2xx
- };
-
- static const char kMiniUTF8UTF8Odd[8][16] = {
- // xxxx E2xx C2Xx C3xx
- // 8 9 a b 8 9 a b 8 9 a b
- {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [0] looking for C38x/C3Ax/2020/8x8x, or err
- {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [1] error, back to looking
- {0,0,0,0,1,1,1,1, 0,0,0,0,0,0,0,0,}, // [2] C38x looking for CXxx/E2xxxx
- // + + + + // E2xxxx flips odd_byte
- {0,0,0,0,1,1,1,1, 0,0,0,0,0,0,0,0,}, // [3] C3Ax looking for E2xx
- // + + + + // E2xxxx flips odd_byte
- {0,0,0,0,1,1,1,1, 0,0,0,0,0,0,0,0,}, // [4] C3AxE2xx-- looking for C2xx/E2xxxx
- // + + + + // E2xxxx flips odd_byte
- {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [5] C3Bx -- looking for C2xxC2xxC2xx
- {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [6] C3Bx -- looking for C2xxC2xx
- {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,}, // [7] C3Bx -- looking for C2xx
- };
-
- // Turn a pair of bytes into the subscript for UTF8UTF8 tables above
- int UTF88Sub(char s0, char s1) {
- int sub = (s1 >> 4) & 0x03;
- uint8 u0 = static_cast<uint8>(s0);
- if (u0 == 0xc3) {
- sub += 12;
- } else if ((u0 & 0xf0) == 0xc0) {
- if ((u0 == 0xc2) || (u0 == 0xc5) || (u0 == 0xc6) || (u0 == 0xcb)) {
- sub += 8;
- }
- } else if (u0 == 0xe2) {
- sub += 4;
- }
- return sub;
- }
-
-
-
-
-
- // Default probability for an encoding rankedencoding
- // Based on a scan of 55M web pages
- // These values are 255 - log base 2**1/10 (occurrences / total)
- // Large values are most likely. This the reverse of some Google code
- // 255 = 1.0, 245 = 1/2, 235 = 1/4, 15 = 1/2**24, 0 = 0 (< 1/50M)
- //
- // TODO change this to be per encoding, not permuted
- //
-
-
- // Support function for unit test program
- // Return ranked encoding corresponding to enc
- // (also exported to compact_enc_det_text.cc)
- int CompactEncDet::BackmapEncodingToRankedEncoding(Encoding enc) {
- for (int i = 0; i < NUM_RANKEDENCODING; ++i) {
- if (kMapToEncoding[i] == enc) {
- return i;
- }
- }
- return -1;
- }
-
-
- string DecodeActive(uint32 active) {
- string temp("");
- if (active & kBinaryActive) {
- temp.append("Binary ");
- }
- if (active & kUTF1632Active) {
- temp.append("UTF1632 ");
- }
- if (active & kUTF8UTF8Active) {
- temp.append("UTF8UTF8 ");
- }
- if (active & kUTF8Active) {
- temp.append("UTF8 ");
- }
- if (active & kIso2022Active) {
- temp.append("Iso2022 ");
- }
- if (active & kHzActive) {
- temp.append("Hz ");
- }
- if (active & kUTF7Active) {
- temp.append("UTF7A ");
- }
- if (active & kSevenBitActive) {
- temp.append("SevenBit ");
- }
- if (active & kIsIndicCode) {
- temp.append("Indic ");
- }
- if (active & kHighAlphaCode) {
- temp.append("HighAlpha ");
- }
- if (active & kHighAccentCode) {
- temp.append("HighAccent ");
- }
- if (active & kEUCJPActive) {
- temp.append("EUCJP ");
- }
- return temp;
- }
-
- static inline bool SevenBitEncoding(int enc) {
- return ((kSpecialMask[enc] & kSevenBitActive) != 0);
- }
- static inline bool TwoByteEncoding(int enc) {
- return ((kSpecialMask[enc] & kTwobyteCode) != 0);
- }
- static inline bool IndicEncoding(int enc) {
- return ((kSpecialMask[enc] & kIsIndicCode) != 0);
- }
- static inline bool HighAlphaEncoding(int enc) {
- return ((kSpecialMask[enc] & kHighAlphaCode) != 0);
- }
- static inline bool HighAccentEncoding(int enc) {
- return ((kSpecialMask[enc] & kHighAccentCode) != 0);
- }
-
-
- static inline bool AnyActive(DetectEncodingState* destatep) {
- return (destatep->active_special != 0);
- }
- static inline bool SevenBitActive(DetectEncodingState* destatep) {
- return (destatep->active_special & kSevenBitActive) != 0;
- }
- static inline bool HzActive(DetectEncodingState* destatep) {
- return (destatep->active_special & kHzActive) != 0;
- }
- static inline bool Iso2022Active(DetectEncodingState* destatep) {
- return (destatep->active_special & kIso2022Active) != 0;
- }
- static inline bool UTF8Active(DetectEncodingState* destatep) {
- return (destatep->active_special & kUTF8Active) != 0;
- }
- static inline bool UTF8UTF8Active(DetectEncodingState* destatep) {
- return (destatep->active_special & kUTF8UTF8Active) != 0;
- }
- static inline bool UTF1632Active(DetectEncodingState* destatep) {
- return (destatep->active_special & kUTF1632Active) != 0;
- }
- static inline bool BinaryActive(DetectEncodingState* destatep) {
- return (destatep->active_special & kBinaryActive) != 0;
- }
- static inline bool UTF7OrHzActive(DetectEncodingState* destatep) {
- return (destatep->active_special & (kHzActive + kUTF7Active)) != 0;
- }
- static inline bool EUCJPActive(DetectEncodingState* destatep) {
- return ((destatep->active_special & kEUCJPActive) != 0);
- }
- static inline bool OtherActive(DetectEncodingState* destatep) {
- return (destatep->active_special & (kIso2022Active + kBinaryActive +
- kUTF8Active + kUTF8UTF8Active +
- kUTF1632Active + kEUCJPActive)) != 0;
- }
-
-
- static inline bool CEDFlagRescanning(CEDInternalFlags flags) {
- return (flags & kCEDRescanning) != 0;
- }
-
- static inline bool CEDFlagForceTags(CEDInternalFlags flags) {
- return (flags & kCEDForceTags) != 0;
- }
-
-
- static inline int maxint(int a, int b) {return (a > b) ? a : b;}
- static inline int minint(int a, int b) {return (a < b) ? a : b;}
-
- static inline const char* MyRankedEncName(int r_enc) {
- return MyEncodingName(kMapToEncoding[r_enc]);
- }
-
-
- // Only for debugging. not thread safe
- static const int kPsSourceWidth = 32;
- static int pssourcenext = 0; // debug only. not threadsafe. dump only >= this
- static int pssourcewidth = 0; // debug only.
- static char* pssource_mark_buffer = NULL;
- int next_do_src_line;
- int do_src_offset[16];
-
-
- void PsSourceInit(int len) {
- pssourcenext = 0;
- pssourcewidth = len;
- delete[] pssource_mark_buffer;
- // Allocate 2 Ascii characters per input byte
- pssource_mark_buffer = new char[(pssourcewidth * 2) + 8]; // 8 = overscan
- memset(pssource_mark_buffer, ' ', pssourcewidth * 2);
- memset(pssource_mark_buffer + (pssourcewidth * 2), '\0', 8);
-
- next_do_src_line = 0;
- memset(do_src_offset, 0, sizeof(do_src_offset));
- }
-
- void PsSourceFinish() {
- // Print preceding mark buffer
- int j = (pssourcewidth * 2) - 1;
- while ((0 <= j) && (pssource_mark_buffer[j] == ' ')) {--j;} // trim
- pssource_mark_buffer[j + 1] = '\0';
- fprintf(stderr, "( %s) do-src\n", pssource_mark_buffer);
- memset(pssource_mark_buffer, ' ', pssourcewidth * 2);
- memset(pssource_mark_buffer + (pssourcewidth * 2), '\0', 8);
-
- delete[] pssource_mark_buffer;
- pssource_mark_buffer = NULL;
- }
-
- // Dump aligned len bytes src... if not already dumped
- void PsSource(const uint8* src, const uint8* isrc, const uint8* srclimit) {
- int offset = src - isrc;
- offset -= (offset % pssourcewidth); // round down to multiple of len bytes
- if (offset < pssourcenext) {
- return;
- }
- pssourcenext = offset + pssourcewidth; // Min offset for next dump
-
- // Print preceding mark buffer
- int j = (pssourcewidth * 2) - 1;
- while ((0 <= j) && (pssource_mark_buffer[j] == ' ')) {--j;} // trim
- pssource_mark_buffer[j + 1] = '\0';
- fprintf(stderr, "( %s) do-src\n", pssource_mark_buffer);
- memset(pssource_mark_buffer, ' ', pssourcewidth * 2);
- memset(pssource_mark_buffer + (pssourcewidth * 2), '\0', 8);
-
- // Print source bytes
- const uint8* src_aligned = isrc + offset;
- int length = srclimit - src_aligned;
- length = minint(pssourcewidth, length);
-
- fprintf(stderr, "(%05x ", offset);
- for (int i = 0; i < length; ++i) {
- char c = src_aligned[i];
- if (c == '\n') {c = ' ';}
- if (c == '\r') {c = ' ';}
- if (c == '\t') {c = ' ';}
- if (c == '(') {
- fprintf(stderr, "%s", "\\( ");
- } else if (c == ')') {
- fprintf(stderr, "%s", "\\) ");
- } else if (c == '\\') {
- fprintf(stderr, "%s", "\\\\ ");
- } else if ((0x20 <= c) && (c <= 0x7e)) {
- fprintf(stderr, "%c ", c);
- } else {
- fprintf(stderr, "%02x", c);
- }
- }
- fprintf(stderr, ") do-src\n");
- // Remember which source offsets are where, mod 16
- do_src_offset[next_do_src_line & 0x0f] = offset;
- ++next_do_src_line;
- }
-
- // Mark bytes in just-previous source bytes
- void PsMark(const uint8* src, int len, const uint8* isrc, int weightshift) {
- int offset = src - isrc;
- offset = (offset % pssourcewidth); // mod len bytes
- char mark = (weightshift == 0) ? '-' : 'x';
-
- pssource_mark_buffer[(offset * 2)] = '=';
- pssource_mark_buffer[(offset * 2) + 1] = '=';
- for (int i = 1; i < len; ++i) {
- pssource_mark_buffer[(offset + i) * 2] = mark;
- pssource_mark_buffer[((offset + i) * 2) + 1] = mark;
- }
- }
-
-
- // Highlight trigram bytes in just-previous source bytes
- // Unfortunately, we have to skip back N lines since source was printed for
- // up to 8 bigrams before we get here. Match on src+1 to handle 0/31 better
- void PsHighlight(const uint8* src, const uint8* isrc, int trigram_val, int n) {
- int offset = (src + 1) - isrc;
- int offset32 = (offset % pssourcewidth); // mod len bytes
- offset -= offset32; // round down to multiple of len bytes
-
- for (int i = 1; i <= 16; ++i) {
- if (do_src_offset[(next_do_src_line - i) & 0x0f] == offset) {
- fprintf(stderr, "%d %d %d do-highlight%d\n",
- i, offset32 - 1, trigram_val, n);
- break;
- }
- }
- }
-
-
- void InitDetectEncodingState(DetectEncodingState* destatep) {
- destatep->initial_src = NULL; // Filled in by caller
- destatep->limit_src = NULL;
- destatep->prior_src = NULL;
- destatep->last_pair = NULL;
-
- destatep->debug_data = NULL;
- destatep->next_detail_entry = 0;
-
- destatep->done = false;
- destatep->reliable = false;
- destatep->hints_derated = false;
- //destatep->declared_enc_1 init in ApplyHints
- //destatep->declared_enc_2 init in ApplyHints
- destatep->prune_count = 0;
-
- destatep->trigram_highwater_mark = 0;
- destatep->looking_for_latin_trigrams = false;
- destatep->do_latin_trigrams = false;
-
- // Miscellaneous state variables for difficult encodings
- destatep->binary_quadrants_count = 0;
- destatep->binary_8x4_count = 0;
- destatep->binary_quadrants_seen = 0;
- destatep->binary_8x4_seen = 0;
- destatep->utf7_starts = 0;
- destatep->prior_utf7_offset = 0;
- destatep->next_utf8_ministate = 0;
- for (int i = 0; i < 6; i++) {destatep->utf8_minicount[i] = 0;}
- destatep->next_utf8utf8_ministate = 0;
- destatep->utf8utf8_odd_byte = 0;
- for (int i = 0; i < 6; i++) {destatep->utf8utf8_minicount[i] = 0;}
- destatep->next_2022_state = SOSI_NONE;
- destatep->next_hz_state = SOSI_NONE;
- destatep->next_eucjp_oddphase = false;
- for (int i = 0; i < 8; i++) {destatep->byte32_count[i] = 0;}
- destatep->active_special = 0xffffffff;
- destatep->tld_hint = UNKNOWN_ENCODING;
- destatep->http_hint = UNKNOWN_ENCODING;
- destatep->meta_hint = UNKNOWN_ENCODING;
- destatep->bom_hint = UNKNOWN_ENCODING;
- destatep->top_rankedencoding = 0; // ASCII [seven-bit] is the default
- destatep->second_top_rankedencoding = 0; // ASCII [seven-bit] is the default
- destatep->top_prob = -1;
- destatep->second_top_prob = -1;
- // This is wide for first pruning, shrinks for 2nd and later
- destatep->prune_difference = kInititalPruneDifference;
-
- destatep->next_prior_bigram = 0;
- destatep->prior_bigram[0] = -1;
- destatep->prior_bigram[1] = -1;
- destatep->prior_bigram[2] = -1;
- destatep->prior_bigram[3] = -1;
-
- destatep->prior_binary[0] = -1;
-
- // Initialize with all but Indic encodings, which we never detect
- int k = 0;
- for (int rankedencoding = 0;
- rankedencoding < NUM_RANKEDENCODING;
- rankedencoding++) {
- Encoding enc = kMapToEncoding[rankedencoding];
- if (!IndicEncoding(enc)) {
- destatep->rankedencoding_list[k++] = rankedencoding;
- }
- }
- destatep->rankedencoding_list_len = k;
-
- // This is where all the action is
- memset(destatep->enc_prob, 0, sizeof(destatep->enc_prob));
-
- memset(destatep->hint_prob, 0, sizeof(destatep->hint_prob));
- memset(destatep->hint_weight, 0, sizeof(destatep->hint_weight));
-
- destatep->prior_interesting_pair[AsciiPair] = 0;
- destatep->prior_interesting_pair[OtherPair] = 0;
- destatep->next_interesting_pair[AsciiPair] = 0;
- destatep->next_interesting_pair[OtherPair] = 0;
- // interesting_pairs/offsets/weightshifts not initialized; no need
- }
-
- // Probability strings are uint8, with zeros removed via simple run-length:
- // (<skip-take byte> <data bytes>)*
- // skip-take:
- // 00 end
- // x0 skip 16 x locations, take 0 data values
- // xy skip x locations, take y data values
- // Multiply all the incoming values by 3 to account for 3x unigram sums
- //
- // {{0x77,0x69,0x6e,0x64,0x31,0x32,0x35,0x35,
- // 0x01,0xc2,0x10,0x41,0xfe,0x71,0xba,0x00,}}, // "wind1255"
- //
- // Weight is 0..100 percent
- //
- // Returns subscript of largest (most probable) value
- //
-
-
- // {{0x6e,0x6c,0x5f,0x5f, 0x05,0xb2,0xae,0xa0,0x32,0xa1,0x36,0x31,0x42,0x39,0x3b,0x33,0x45,0x11,0x6f,0x00,}}, // "nl__"
- // // ASCII-7-bit=178 Latin1=174 UTF8=160 GB=50 CP1252=161 BIG5=49 Latin2=66 CP1251=57 CP1256=59 CP1250=51 Latin5=69 ISO-8859-15=111 [top ASCII-7-bit]
- int ApplyCompressedProb(const char* iprob, int len,
- int weight, DetectEncodingState* destatep) {
- int* dst = &destatep->enc_prob[0];
- int* dst2 = &destatep->hint_weight[0];
- const uint8* prob = reinterpret_cast<const uint8*>(iprob);
- const uint8* problimit = prob + len;
-
- int largest = -1;
- int subscript_of_largest = 0;
-
- // Continue with first byte and subsequent ones
- while (prob < problimit) {
- int skiptake = *prob++;
- int skip = (skiptake & 0xf0) >> 4;
- int take = skiptake & 0x0f;
- if (skiptake == 00) {
- break;
- } else if (take == 0) {
- dst += (skip << 4);
- dst2 += (skip << 4);
- } else {
- dst += skip; // Normal case
- dst2 += skip; // Normal case
- for (int i = 0; i < take; i++) {
- int enc = static_cast<int>(dst - &destatep->enc_prob[0]) + i;
- if (largest < prob[i]) {
- largest = prob[i];
- subscript_of_largest = enc;
- }
-
- int increment = prob[i] * 3; // The actual increment
-
- // Do maximum of previous hints plus this new one
- if (weight > 0) {
- increment = (increment * weight) / 100;
- dst[i] = maxint(dst[i], increment);
- dst2[i] = 1; // New total weight
- }
- }
- prob += take;
- dst += take;
- dst2 += take;
- }
- }
- return subscript_of_largest;
- }
-
-
- // Returns subscript of largest (most probable) value [for unit test]
- int TopCompressedProb(const char* iprob, int len) {
- const uint8* prob = reinterpret_cast<const uint8*>(iprob);
- const uint8* problimit = prob + len;
- int next_prob_sub = 0;
- int topprob = 0;
- int toprankenc = 0;
-
- while (prob < problimit) {
- int skiptake = *prob++;
- int skip = (skiptake & 0xf0) >> 4;
- int take = skiptake & 0x0f;
- if (skiptake == 0) {
- break;
- } else if (take == 0) {
- next_prob_sub += (skip << 4);
- } else {
- next_prob_sub += skip; // Normal case
- for (int i = 0; i < take; i++) {
- if (topprob < prob[i]) {
- topprob = prob[i];
- toprankenc = next_prob_sub + i;
- }
- }
- prob += take;
- next_prob_sub += take;
- }
- }
- return toprankenc;
- }
-
-
- // Find subscript of matching key in first 8 bytes of sorted hint array, or -1
- int HintBinaryLookup8(const HintEntry* hintprobs, int hintprobssize,
- const char* norm_key) {
- // Key is always in range [lo..hi)
- int lo = 0;
- int hi = hintprobssize;
- while (lo < hi) {
- int mid = (lo + hi) >> 1;
- int comp = memcmp(&hintprobs[mid].key_prob[0], norm_key, 8);
- if (comp < 0) {
- lo = mid + 1;
- } else if (comp > 0) {
- hi = mid;
- } else {
- return mid;
- }
- }
- return -1;
- }
-
- // Find subscript of matching key in first 4 bytes of sorted hint array, or -1
- int HintBinaryLookup4(const HintEntry* hintprobs, int hintprobssize,
- const char* norm_key) {
- // Key is always in range [lo..hi)
- int lo = 0;
- int hi = hintprobssize;
- while (lo < hi) {
- int mid = (lo + hi) >> 1;
- int comp = memcmp(&hintprobs[mid].key_prob[0], norm_key, 4);
- if (comp < 0) {
- lo = mid + 1;
- } else if (comp > 0) {
- hi = mid;
- } else {
- return mid;
- }
- }
- return -1;
- }
-
- static inline void Boost(DetectEncodingState* destatep, int r_enc, int boost) {
- destatep->enc_prob[r_enc] += boost;
- }
-
- static inline void Whack(DetectEncodingState* destatep, int r_enc, int whack) {
- destatep->enc_prob[r_enc] -= whack;
- }
-
- // Apply initial probability hint based on top level domain name
- // Weight is 0..100 percent
- // Return 1 if name match found
- int ApplyTldHint(const char* url_tld_hint, int weight,
- DetectEncodingState* destatep) {
- if (url_tld_hint[0] == '~') {
- return 0;
- }
- string normalized_tld = MakeChar4(string(url_tld_hint));
- int n = HintBinaryLookup4(kTLDHintProbs, kTLDHintProbsSize,
- normalized_tld.c_str());
- if (n >= 0) {
- // TLD is four bytes, probability table is ~12 bytes
- int best_sub = ApplyCompressedProb((const char *)&kTLDHintProbs[n].key_prob[kMaxTldKey],
- kMaxTldVector, weight, destatep);
- // Never boost ASCII7; do CP1252 instead
- if (best_sub == F_ASCII_7_bit) {best_sub = F_CP1252;}
- destatep->declared_enc_1 = best_sub;
- if (destatep->debug_data != NULL) {
- // Show TLD hint
- SetDetailsEncProb(destatep, 0, best_sub, url_tld_hint);
- }
- return 1;
- }
- return 0;
- }
-
- // Apply initial probability hint based on charset= name
- // Weight is 0..100 percent
- // Return 1 if name match found
- int ApplyCharsetHint(const char* charset_hint, int weight,
- DetectEncodingState* destatep) {
- if (charset_hint[0] == '~') {
- return 0;
- }
- string normalized_charset = MakeChar44(string(charset_hint));
- int n = HintBinaryLookup8(kCharsetHintProbs, kCharsetHintProbsSize,
- normalized_charset.c_str());
- if (n >= 0) {
- // Charset is eight bytes, probability table is ~eight bytes
- int best_sub = ApplyCompressedProb((const char *)&kCharsetHintProbs[n].key_prob[kMaxCharsetKey],
- kMaxCharsetVector, weight, destatep);
- // Never boost ASCII7; do CP1252 instead
- if (best_sub == F_ASCII_7_bit) {best_sub = F_CP1252;}
- destatep->declared_enc_1 = best_sub;
-
- // If first explicitly declared charset is confusable with Latin1/1252, put
- // both declared forms in declared_enc_*, displacing Latin1/1252.
- // This avoids a bit of Latin1 creep.
- // Also boost the declared encoding and its pair
- // TODO: This should all be folded into postproc-enc-detect.cc
- if ((destatep->http_hint == UNKNOWN_ENCODING) &&
- (destatep->meta_hint == UNKNOWN_ENCODING)) {
- // This is the first charset=hint
- switch (best_sub) {
- case F_Latin2: // 8859-2 Latin2, east euro
- destatep->declared_enc_2 = F_CP1250;
- Boost(destatep, F_Latin2, kGentleOnePair);
- Boost(destatep, F_CP1250, kGentleOnePair);
- break;
- case F_CP1250:
- destatep->declared_enc_2 = F_Latin2;
- Boost(destatep, F_Latin2, kGentleOnePair);
- Boost(destatep, F_CP1250, kGentleOnePair);
- break;
-
- case F_Latin3: // 8859-3 Latin3, south euro, Esperanto
- destatep->declared_enc_2 = F_ASCII_7_bit;
- Boost(destatep, F_Latin3, kGentleOnePair);
- break;
-
- case F_Latin4: // 8859-4 Latin4, north euro
- destatep->declared_enc_2 = F_ASCII_7_bit;
- Boost(destatep, F_Latin4, kGentleOnePair);
- break;
-
- case F_ISO_8859_5: // 8859-5 Cyrillic
- destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost 1251
- Boost(destatep, F_ISO_8859_5, kGentleOnePair); // (too different)
- break;
- case F_CP1251:
- destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost -5
- Boost(destatep, F_CP1251, kGentleOnePair); // (too different)
- break;
-
- case F_Arabic: // 8859-6 Arabic
- destatep->declared_enc_2 = F_CP1256;
- Boost(destatep, F_Arabic, kGentleOnePair);
- Boost(destatep, F_CP1256, kGentleOnePair);
- break;
- case F_CP1256:
- destatep->declared_enc_2 = F_Arabic;
- Boost(destatep, F_Arabic, kGentleOnePair);
- Boost(destatep, F_CP1256, kGentleOnePair);
- break;
-
- case F_Greek: // 8859-7 Greek
- destatep->declared_enc_2 = F_CP1253;
- Boost(destatep, F_Greek, kGentleOnePair);
- Boost(destatep, F_CP1253, kGentleOnePair);
- break;
- case F_CP1253:
- destatep->declared_enc_2 = F_Greek;
- Boost(destatep, F_Greek, kGentleOnePair);
- Boost(destatep, F_CP1253, kGentleOnePair);
- break;
-
- case F_Hebrew: // 8859-8 Hebrew
- destatep->declared_enc_2 = F_CP1255;
- Boost(destatep, F_Hebrew, kGentleOnePair);
- Boost(destatep, F_CP1255, kGentleOnePair);
- break;
- case F_CP1255:
- destatep->declared_enc_2 = F_Hebrew;
- Boost(destatep, F_Hebrew, kGentleOnePair);
- Boost(destatep, F_CP1255, kGentleOnePair);
- break;
-
- case F_Latin5: // 8859-9 Latin5, Turkish
- destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost 1254
- Boost(destatep, F_Latin5, kGentleOnePair); // (too different)
- break;
- case F_CP1254:
- destatep->declared_enc_2 = F_ASCII_7_bit; // Don't boost Latin5
- Boost(destatep, F_CP1254, kGentleOnePair); // (too different)
- break;
-
- case F_Latin6: // 8859-10 Latin6, Nordic
- destatep->declared_enc_2 = F_ASCII_7_bit;
- Boost(destatep, F_Latin6, kGentleOnePair);
- break;
-
- case F_ISO_8859_11: // 8859-11 Thai,
- destatep->declared_enc_2 = F_CP874;
- Boost(destatep, F_ISO_8859_11, kGentleOnePair);
- Boost(destatep, F_CP874, kGentleOnePair);
- break;
- case F_CP874:
- destatep->declared_enc_2 = F_ISO_8859_11;
- Boost(destatep, F_ISO_8859_11, kGentleOnePair);
- Boost(destatep, F_CP874, kGentleOnePair);
- break;
-
- case F_ISO_8859_13: // 8859-13 Latin7, Baltic
- destatep->declared_enc_2 = F_CP1257;
- Boost(destatep, F_ISO_8859_13, kGentleOnePair);
- Boost(destatep, F_CP1257, kGentleOnePair);
- break;
- case F_CP1257:
- destatep->declared_enc_2 = F_ISO_8859_13;
- Boost(destatep, F_ISO_8859_13, kGentleOnePair);
- Boost(destatep, F_CP1257, kGentleOnePair);
- break;
-
- case F_ISO_8859_15: // 8859-15 Latin9, Latin0, Euro-ized Latin1
- destatep->declared_enc_2 = F_ASCII_7_bit;
- Boost(destatep, F_ISO_8859_15, kGentleOnePair);
- break;
-
-
- // Greek all-caps is confusable with KOI8x all-lower and Hebrew.
- // This turns some Greek documents into Cyrillic, etc. by mistake.
- // Greek and Hebrew are boosted explicitly above; do KOI8x here.
- // Boosting the declared encodingmakes it harder for the wrong one to
- // creep up.
- case F_KOI8R:
- Boost(destatep, F_KOI8R, kGentleOnePair);
- break;
- case F_KOI8U:
- Boost(destatep, F_KOI8U, kGentleOnePair);
- break;
-
- default:
- break;
- }
- }
-
- if (destatep->debug_data != NULL) {
- // Show charset hint
- SetDetailsEncProb(destatep, 0, best_sub, charset_hint);
- }
-
- //
- // Some fix-ups for the declared encodings
- //
-
- // If non-UTF8, non-Latin1/1252 encoding declared, disable UTF8 combos
- // TODO: This should all be folded into postproc-enc-detect.cc
- if ((best_sub != F_UTF8) &&
- (best_sub != F_Latin1) &&
- (best_sub != F_CP1252)) {
- Whack(destatep, F_UTF8UTF8, kBadPairWhack * 4); // demote
- }
-
- // Latin2 and CP1250 differ in the overlap part, such as B1 or B9
- // The initial probabilites for charset=Latin2 explicitly put CP1250
- // down twice as far as normal, and vice versa. This is done in
- // postproc-enc-detect.cc
-
- // If charset=user-defined, treat as Binary --
- // we can safely only do low ASCII, might be Indic
- if (normalized_charset.substr(0,4) == "user") {
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- }
-
- return 1;
- }
- return 0;
- }
-
- // Apply initial probability hint based on caller-supplied encoding
- // Negative hint whacks ~encoding, non-negative boosts encoding
- //
- // Negative hints are an experiment to see if they might be useful.
- // Not operator used instead of unary minus to allow specifying not-zero
- int ApplyEncodingHint(const int encoding_hint, int weight,
- DetectEncodingState* destatep) {
- Encoding enc_hint = static_cast<Encoding>((encoding_hint < 0) ?
- ~encoding_hint : encoding_hint);
- // Map to the right internal subscript
- int rankedenc_hint = CompactEncDet::BackmapEncodingToRankedEncoding(enc_hint);
-
- // I'm not sure how strong this hint should be. Weight 100% = 1 bigram
- int increment = (kBoostOnePair * weight) / 100;
-
- if (encoding_hint < 0) {
- destatep->enc_prob[rankedenc_hint] -= increment;
- } else {
- destatep->enc_prob[rankedenc_hint] += increment;
- }
-
- if (destatep->debug_data != NULL) {
- // Show encoding hint
- SetDetailsEncProb(destatep, 0, -1, MyEncodingName(enc_hint));
- }
- return 1;
- }
-
- // Apply initial probability hint based on user interface language
- // Weight is 0..100 percent
- // Return 1 if name match found
- int ApplyUILanguageHint(const Language language_hint,
- int weight, DetectEncodingState* destatep) {
- if (language_hint == UNKNOWN_LANGUAGE) {
- return 0;
- }
- string normalized_lang = MakeChar8(LanguageName(language_hint));
- int n = HintBinaryLookup8(kLangHintProbs, kLangHintProbsSize,
- normalized_lang.c_str());
- if (n >= 0) {
- // Language is eight bytes, probability table is ~eight bytes
- int best_sub = ApplyCompressedProb((const char *)&kLangHintProbs[n].key_prob[kMaxLangKey],
- kMaxLangVector, weight, destatep);
- // Never boost ASCII7; do CP1252 instead
- if (best_sub == F_ASCII_7_bit) {best_sub = F_CP1252;}
- destatep->declared_enc_1 = best_sub;
- if (destatep->debug_data != NULL) {
- // Show language hint
- SetDetailsEncProb(destatep, 0, best_sub, normalized_lang.c_str());
- }
- return 1;
- }
- return 0;
- }
-
- // Apply initial probability hint based on corpus type (web, email, etc)
- // Return 1 if name match found
- int ApplyDefaultHint(const CompactEncDet::TextCorpusType corpus_type,
- DetectEncodingState* destatep) {
-
- for (int i = 0; i < NUM_RANKEDENCODING; i++) {
- // Set the default probability
- destatep->enc_prob[i] = kDefaultProb[i] * 3;
- // Deliberately set 2022 seven-bit encodings to zero,
- // so we can look for actual use
- // TODO: This should all be folded into postproc-enc-detect.cc
- if (SevenBitEncoding(kMapToEncoding[i])) {
- destatep->enc_prob[i] = 0;
- }
- }
-
- // A little corpus distinction
- switch (corpus_type) {
- case CompactEncDet::WEB_CORPUS:
- case CompactEncDet::XML_CORPUS:
- // Allow double-converted UTF-8 to start nearly equal to normal UTF-8
- destatep->enc_prob[F_UTF8UTF8] =
- destatep->enc_prob[F_UTF8] - kSmallInitDiff;
- break;
- case CompactEncDet::QUERY_CORPUS:
- case CompactEncDet::EMAIL_CORPUS:
- default:
- break;
- }
-
- if (FLAGS_demo_nodefault) {
- // Demo, make initial probs all zero
- for (int i = 0; i < NUM_RANKEDENCODING; i++) {
- destatep->enc_prob[i] = 0;
- }
- }
-
- if (destatep->debug_data != NULL) {
- // Show default hint
- SetDetailsEncProb(destatep, 0, -1, "Default");
- }
- return 1;
- }
-
-
-
- // Do reverse search for c in [str..str+len)
- // Note: initial pointer is to FRONT of string, not back
- const char* MyMemrchr(const char* str, char c, size_t len) {
- const char* ret = str + len;
- while (str <= --ret) {
- if (*ret == c) {return ret;}
- }
- return NULL;
- }
-
-
- // Minimum real URL is 11 bytes: "http://a.bc" -- shorter is assumed to be TLD
- // Now that we are no longer trying to do Indic font-based encodigns, we
- // don't need the full URL and can go back to simple TLD. This test remains for
- // backwards compatility with any caller using full URL.
- static const int kMinURLLength = 11;
-
- // Extract TLD from a full URL or just a TLD
- // Return hostname and length if a full URL
- void ExtractTLD(const char* url_hint, char* tld_hint, int tld_hint_len,
- const char** ret_host_start, int* ret_host_len) {
- // url_hint can either be a full URL (preferred) or just top-level domain name
- // Extract the TLD from a full URL and use it for
- // a normal TLD hint
-
- strncpy(tld_hint, "~", tld_hint_len);
- tld_hint[tld_hint_len - 1] = '\0';
- *ret_host_start = NULL;
- *ret_host_len = 0;
-
- int url_len = (url_hint != NULL) ? strlen(url_hint) : 0;
- if (url_len == 0) {
- // Empty TLD
- return;
- }
-
- // Minimum real URL is 11 bytes: "http://a.bc" -- shorter is assumed to be TLD
- if (kMinURLLength <= url_len) {
- // See if it really is a URL
- const char* first_slash = strchr(url_hint, '/');
- if ((first_slash != NULL) && (first_slash != url_hint) &&
- (first_slash[-1] == ':') && (first_slash[1] == '/') &&
- (memrchr(url_hint, '.', first_slash - url_hint) == NULL)) {
- // We found :// and no dot in front of it, so declare a real URL
-
- const char* hostname_start = first_slash + 2;
- const char* hostname_end = strchr(hostname_start, '/');
- if (hostname_end == NULL) {
- // No slash; end is first byte off end of the URL string
- hostname_end = url_hint + url_len;
- }
- size_t hostname_len = hostname_end - hostname_start;
- const char* port_start =
- (const char*)memchr(hostname_start, ':', hostname_len);
- if (port_start != NULL) {
- // Port; shorten hostname
- hostname_end = port_start;
- hostname_len = hostname_end - hostname_start;
- }
-
- const char* tld_start = MyMemrchr(hostname_start, '.', hostname_len);
- if (tld_start != NULL) {
- // Remember the TLD we just found
- int tld_len = hostname_start + hostname_len - tld_start - 1;
- if (tld_len > (tld_hint_len - 1)) {
- tld_len = tld_hint_len - 1;
- }
- memcpy(tld_hint, tld_start + 1, tld_len);
- tld_hint[tld_len] = '\0';
- }
- *ret_host_start = hostname_start;
- *ret_host_len = hostname_len;
- return;
- }
- } else {
- strncpy(tld_hint, url_hint, tld_hint_len);
- tld_hint[tld_hint_len - 1] = '\0';
- }
- }
-
- // Apply hints, if any, to probabilities
- // NOTE: Encoding probabilites are all zero at this point
- void ApplyHints(const char* url_hint,
- const char* http_charset_hint,
- const char* meta_charset_hint,
- const int encoding_hint,
- const Language language_hint,
- const CompactEncDet::TextCorpusType corpus_type,
- DetectEncodingState* destatep) {
- int hint_count = 0;
- // url_hint can either be a full URL (preferred) or just top-level domain name
- // Extract the TLD from a full URL and use it for
- // a normal TLD hint
-
- char tld_hint[16];
- const char* hostname_start = NULL;
- int hostname_len = 0;
- ExtractTLD(url_hint, tld_hint, sizeof(tld_hint),
- &hostname_start, &hostname_len);
-
-
- // Initial hints give slight boost to Ascii-7-bit and code page 1252
- // ApplyXxx routines copy enc_1 to enc_2 then update declared_enc_1
- // This gives a boost to 1252 if one of HTTP/META is specified,
- // but this could be the wrong thing to do if Latin2/3/4/etc. is specified
- destatep->declared_enc_1 = F_CP1252;
- destatep->declared_enc_2 = F_ASCII_7_bit;
-
- // Applying various hints takes max of new hint and any old hint.
- // This does better on multiple hints that a weighted average
-
- // Weight is 0..100 percent
- if ((http_charset_hint != NULL) && (http_charset_hint[0] != '~')) {
- destatep->declared_enc_2 = destatep->declared_enc_1;
- hint_count += ApplyCharsetHint(http_charset_hint, 100, destatep);
- destatep->http_hint = kMapToEncoding[destatep->declared_enc_1];
- if ((destatep->declared_enc_1 == F_CP1252) ||
- (destatep->declared_enc_1 == F_Latin1)) {
- destatep->looking_for_latin_trigrams = true;
- }
- }
- if ((meta_charset_hint != NULL) && (meta_charset_hint[0] != '~')) {
- destatep->declared_enc_2 = destatep->declared_enc_1;
- hint_count += ApplyCharsetHint(meta_charset_hint, 100, destatep);
- destatep->meta_hint = kMapToEncoding[destatep->declared_enc_1];
- if ((destatep->declared_enc_1 == F_CP1252) ||
- (destatep->declared_enc_1 == F_Latin1)) {
- destatep->looking_for_latin_trigrams = true;
- }
- }
- if (encoding_hint != UNKNOWN_ENCODING) {
- destatep->declared_enc_2 = destatep->declared_enc_1;
- hint_count += ApplyEncodingHint(encoding_hint, 50, destatep);
- }
- if (language_hint != UNKNOWN_LANGUAGE) {
- destatep->declared_enc_2 = destatep->declared_enc_1;
- hint_count += ApplyUILanguageHint(language_hint, 50, destatep);
- }
- // Use top level domain if not .com and <=1 other hint was available
- if (url_hint != NULL) {
- destatep->tld_hint = CompactEncDet::TopEncodingOfTLDHint(tld_hint);
- if (hint_count == 0) {
- // Apply with weight 100%
- destatep->declared_enc_2 = destatep->declared_enc_1;
- hint_count += ApplyTldHint(tld_hint, 100, destatep);
- if ((destatep->declared_enc_1 == F_CP1252) ||
- (destatep->declared_enc_1 == F_Latin1)) {
- destatep->looking_for_latin_trigrams = true;
- }
- if (strcmp("hu", tld_hint) == 0) {
- // Hungarian is particularly difficult to separate Latin2 from Latin1,
- // so always look for trigram scanning if bare TLD=hu hint
- destatep->looking_for_latin_trigrams = true;
- }
- // Treat .com as no TLD hint at all
- } else if ((hint_count == 1) && (strcmp("com", tld_hint) != 0)) {
- // Either shift weighting or consider doing no TLD here -- seems to
- // distract from correct charset= hints. Or perhaps apply only if
- // charset = Latin1/1252...
- // Apply with weight 50%
- destatep->declared_enc_2 = destatep->declared_enc_1;
- hint_count += ApplyTldHint(tld_hint, 50, destatep);
- if ((destatep->declared_enc_1 == F_CP1252) ||
- (destatep->declared_enc_1 == F_Latin1)) {
- destatep->looking_for_latin_trigrams = true; // These need trigrams
- }
- }
- // Else ignore TLD hint entirely
- }
-
- // Use all-web default distribution if not even a TLD hint
- if (hint_count == 0) {
- destatep->looking_for_latin_trigrams = true; // Default needs trigrams
- destatep->declared_enc_2 = destatep->declared_enc_1;
- hint_count += ApplyDefaultHint(corpus_type, destatep);
- }
-
-
- // ISO-Microsoft Pairs
- // F_Latin1, F_CP1252,
- // F_Latin2, F_CP1250, NOT really strict subset/superset pairs
- // F_Latin3,
- // F_Latin4,
- // F_ISO_8859_5, F_CP1251,
- // F_Arabic, F_CP1256, NOT
- // F_Greek, F_CP1253, NOT really pairs
- // (or upgrade incvt to make Greek use CP)
- // F_Hebrew, F_CP1255, NOT really pairs
- // F_Latin5, F_CP1254,
- // F_Latin6,
- // F_ISO_8859_11,
- // F_ISO_8859_13, F_CP1257,
- // F_ISO_8859_15,
- // ISO-Microsoft Pairs
-
- // Get important families started together
- // // This should fall out of the initializatoin vectors for charset,
- // but we need to get rid of families alltogetrher
- //
- // TODO make this more graceful
-
- // Add small bias for subsets
-
- // Subtract small bias for supersets
- destatep->enc_prob[F_CP932] = destatep->enc_prob[F_SJS] - kSmallInitDiff;
-
- destatep->enc_prob[F_GBK] = destatep->enc_prob[F_GB] - kSmallInitDiff;
- destatep->enc_prob[F_GB18030] = destatep->enc_prob[F_GB] - kSmallInitDiff;
-
- destatep->enc_prob[F_BIG5_CP950] = destatep->enc_prob[F_BIG5] -
- kSmallInitDiff;
- destatep->enc_prob[F_BIG5_HKSCS] = destatep->enc_prob[F_BIG5] -
- kSmallInitDiff;
-
- // Deliberate over-bias Ascii7 and underbias Binary [unneeded]
- // destatep->enc_prob[F_ASCII_7_bit] = destatep->enc_prob[F_ASCII_7_bit] + kSmallInitDiff;
- // destatep->enc_prob[F_BINARY] = destatep->enc_prob[F_BINARY] - (kBoostInitial / 2);
-
- if (destatep->debug_data != NULL) {
- // Show state at end of hints
- SetDetailsEncProb(destatep, 0, -1, "Endhints");
- if(FLAGS_enc_detect_detail2) {
- // Add a line showing the watched encoding(s)
- if (watch1_rankedenc >= 0) {
- SetDetailsEncProb(destatep, 0,
- watch1_rankedenc, FLAGS_enc_detect_watch1);
- }
- if (watch2_rankedenc >= 0) {
- SetDetailsEncProb(destatep, 0,
- watch2_rankedenc, FLAGS_enc_detect_watch2);
- }
- } // End detail2
- }
-
- // If duplicate hints, set second one to ASCII_7BIT to prevent double-boost
- if (destatep->declared_enc_1 == destatep->declared_enc_2) {
- destatep->declared_enc_2 = F_ASCII_7_bit;
- }
-
- if (FLAGS_force127) {
- destatep->do_latin_trigrams = true;
- if (FLAGS_enc_detect_source) {
- PsHighlight(0, destatep->initial_src, 0, 2);
- }
- }
-
-
- if (FLAGS_counts && destatep->looking_for_latin_trigrams) {++looking_used;}
- if (FLAGS_counts && destatep->do_latin_trigrams) {++doing_used;}
-
- //
- // At this point, destatep->enc_prob[] is an initial probability vector based
- // on the given hints/default. In general, it spreads out least-likely
- // encodings to be about 2**-25 below the most-likely encoding.
- // For input text with lots of bigrams, an unlikely encoding can rise to
- // the top at a rate of about 2**6 per bigram, and more commonly 2**2 per
- // bigram. So more than 4 bigrams and commonly more than 12 are
- // needed to overcome the initial hints when the least-likely encoding
- // is in fact the correct answer. So if the entire text has very few bigrams
- // (as a two-word query might), it can be impossible for the correct
- // encoding to win.
- //
- // To compensate for this, we take the initial hint vector and effectively
- // apply it at the rate of 1/16 every bigram for the first 16 bigrams. The
- // actual mechanism is done just before the last prune.
- //
-
- // Remember Initial hint probabilities
- memcpy(destatep->hint_prob, destatep->enc_prob, sizeof(destatep->enc_prob));
- }
-
- // Look for specific high-value patterns in the first 4 bytes
- // Byte order marks (BOM)
- // EFBBBF UTF-8
- // FEFF UTF-16 BE
- // FFFE UTF-16 LE
- // FFFE0000 UTF-32 BE
- // 0000FEFF UTF-32 LE
- //
- // Likely UTF-x of seven-bit ASCII
- // 00xx UTF-16 BE xx printable ASCII
- // xx00 UTF-16 LE
- // 000000xx UTF-32 BE
- // xx000000 UTF-32 LE
- //
- void InitialBytesBoost(const uint8* src,
- int text_length,
- DetectEncodingState* destatep) {
- if (text_length < 4) {return;}
-
- uint32 pair01 = (src[0] << 8) | src[1];
- uint32 pair23 = (src[2] << 8) | src[3];
- uint32 quad0123 = (pair01 << 16) | pair23;
-
- bool utf_16_indication = false;
- bool utf_32_indication = false;
- int best_enc = -1;
-
- // Byte order marks
- // UTF-8
- if ((quad0123 & 0xffffff00) == 0xEFBBBF00) {
- destatep->bom_hint = UTF8;
- Boost(destatep, F_UTF8, kBoostInitial * 2);
- Boost(destatep, F_UTF8UTF8, kBoostInitial * 2);
- best_enc = F_UTF8;
- // UTF-32 (test before UTF-16)
- } else if (quad0123 == 0x0000FEFF) {
- destatep->bom_hint = UTF32BE;
- Boost(destatep, F_UTF_32BE, kBoostInitial * 2);
- best_enc = F_UTF_32BE;
- } else if (quad0123 == 0xFFFE0000) {
- destatep->bom_hint = UTF32LE;
- Boost(destatep, F_UTF_32LE, kBoostInitial * 2);
- best_enc = F_UTF_32LE;
- // UTF-16
- } else if (pair01 == 0xFEFF) {
- destatep->bom_hint = UTF16BE;
- Boost(destatep, F_UTF_16BE, kBoostInitial * 3);
- best_enc = F_UTF_16BE;
- } else if (pair01 == 0xFFFE) {
- destatep->bom_hint = UTF16LE;
- Boost(destatep, F_UTF_16LE, kBoostInitial * 3);
- best_enc = F_UTF_16LE;
-
- // Possible seven-bit ASCII encoded as UTF-16/32
- // UTF-32 (test before UTF-16)
- } else if (((quad0123 & 0xffffff00) == 0) &&
- (kIsPrintableAscii[src[3]] != 0)) {
- Boost(destatep, F_UTF_32BE, kBoostInitial);
- Whack(destatep, F_UTF_32LE, kBadPairWhack); // Illegal char
- best_enc = F_UTF_32BE;
- } else if (((quad0123 & 0x00ffffff) == 0) &&
- (kIsPrintableAscii[src[0]] != 0)) {
- Boost(destatep, F_UTF_32LE, kBoostInitial);
- Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal char
- best_enc = F_UTF_32LE;
- } else if ((src[0] == 0x00) && (kIsPrintableAscii[src[1]] != 0)) {
- Boost(destatep, F_UTF_16BE, kBoostInitial);
- best_enc = F_UTF_16BE;
- } else if ((src[1] == 0x00) && (kIsPrintableAscii[src[0]] != 0)) {
- Boost(destatep, F_UTF_16LE, kBoostInitial);
- best_enc = F_UTF_16LE;
-
- // Whack if 0000 or FFFF
- // UTF-32 (test before UTF-16)
- } else if (quad0123 == 0x00000000) {
- Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal char
- Whack(destatep, F_UTF_32LE, kBadPairWhack);
- Whack(destatep, F_UTF_16BE, kBadPairWhack);
- Whack(destatep, F_UTF_16LE, kBadPairWhack);
- best_enc = -1;
- } else if (quad0123 == 0xffffffff) {
- Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal char
- Whack(destatep, F_UTF_32LE, kBadPairWhack);
- Whack(destatep, F_UTF_16BE, kBadPairWhack);
- Whack(destatep, F_UTF_16LE, kBadPairWhack);
- best_enc = -1;
- } else if (pair01 == 0x0000) {
- Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal char
- Whack(destatep, F_UTF_16LE, kBadPairWhack);
- best_enc = -1;
- } else if (pair01 == 0xffff) {
- Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal char
- Whack(destatep, F_UTF_16LE, kBadPairWhack);
- best_enc = -1;
-
-
- // These are the first four bytes of some known binary file formats
-
- // Boost BINARY bigtime if JPEG FFD8FFxx
- // Boost BINARY bigtime if png 89504E47 (.PNG)
- // Boost BINARY bigtime if gif 47494638 (GIF8)
- // Boost BINARY bigtime if zip 504B0304 (PK..)
- // Boost BINARY bigtime if gzip 1F8B08xx
- // Boost BINARY bigtime if gzip 78DAxxxx
- // Boost BINARY if PDF 25504446 (%PDF)
- // Boost BINARY if SWF (FWSx or CWSx where x <= 0x1f)
- } else if ((quad0123 & 0xffffff00) == 0xFFD8FF00) { // JPEG FFD8FFxx
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if (quad0123 == 0x89504E47) { // Hex 89 P N G
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if (quad0123 == 0x47494638) { // Hex GIF8
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if (quad0123 == 0x504B0304) { // Hex P K 03 04
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if ((quad0123 & 0xffffff00) == 0x1F8B0800) { // gzip 1F8B08xx
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if (pair01 == 0x78DA) { // gzip 78DAxxxx
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if (quad0123 == 0x25504446) { // Hex %PDF
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if ((quad0123 & 0xffffff1f) == 0x66535700) { // Hex FWSx
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if ((quad0123 & 0xffffff1f) == 0x63535700) { // Hex CWSx
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
-
- // More binary detect prefixes
- // 7F E L F Executable and linking format
- // M M 00 * TIFF (little-endian)
- // * 00 M M TIFF (big-endian)
- // 01 f c p Final cut pro
- } else if (quad0123 == 0x7F454C46) { // Hex 7F E L F
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if (quad0123 == 0x4D4D002A) { // Hex M M 00 *
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if (quad0123 == 0x2A004D4D) { // Hex * 00 M M
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if (quad0123 == 0x01666370) { // Hex 01 f c p
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
-
- // More binary detect prefixes; all-ASCII names; heavy weight to avoid ASCII
- // prefix overcoming binary
- // C C S D USGS ISIS 3-D cube files
- // S I M P FITS image header "SIMPLE "
- } else if (quad0123 == 0x43435344) { // Hex C C S D
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if (quad0123 == 0x53494D50) { // Hex S I M P
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
-
- // More binary detect prefixes; all-ASCII names; lighter weight
- // H W P Hangul word processor
- // 8 B P S Photoshop
- // P D S _ xx "PDS_VERSION_ID "
- } else if (quad0123 == 0x48575020) { // Hex H W P
- if ((19 <= text_length) &&
- (memcmp(src, "HWP.Document.File.V", 19) == 0)) {
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if ((19 <= text_length) &&
- (memcmp(src, "HWP Document File V", 19) == 0)) {
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else {
- Boost(destatep, F_BINARY, kBoostInitial * kWeakerBinary);
- }
- } else if (quad0123 == 0x38425053) { // Hex 8 B P S
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else if (quad0123 == 0x5044535F) { // Hex P D S _
- if ((14 <= text_length) && (memcmp(src, "PDS_VERSION_ID", 14) == 0)) {
- Boost(destatep, F_BINARY, kBoostInitial * kStrongBinary);
- } else {
- Boost(destatep, F_BINARY, kBoostInitial * kWeakerBinary);
- }
- }
-
- // There are several main Windows EXE file formats.
- // Not examined here (prefix too short; never see them in Google pipeline)
- // M Z DOS .exe Mark Zbikowski
- // N E DOS 4.0 16-bit
- // L E OS/2 VxD drivers
- // L X OS/2
- // P E Windows NT
-
-
- // More user-defined
- // http://www.freenet.am/armscii/ Armenian
-
- // If any hints or BOM, etc. keep UTF 16/32 around
- if ((destatep->enc_prob[F_UTF_16BE] > 0) ||
- (destatep->enc_prob[F_UTF_16LE] > 0)) {
- utf_16_indication = true;
- }
- if ((destatep->enc_prob[F_UTF_32BE] > 0) ||
- (destatep->enc_prob[F_UTF_32LE] > 0)) {
- utf_32_indication = true;
- }
-
-
- // Kill UTF16/32 right now if no positive indication of them
- // Otherwise, they tend to rise to the top in 7-bit files with an
- // occasional 0x02 byte in some comment or javascript
- if (!utf_16_indication) {
- Whack(destatep, F_UTF_16BE, kBadPairWhack * 8);
- Whack(destatep, F_UTF_16LE, kBadPairWhack * 8);
- Whack(destatep, F_Unicode, kBadPairWhack * 8);
- }
- if (!utf_32_indication) {
- Whack(destatep, F_UTF_32BE, kBadPairWhack * 8);
- Whack(destatep, F_UTF_32LE, kBadPairWhack * 8);
- }
-
- // Usually kill mixed encodings
- if (!FLAGS_ced_allow_utf8utf8) {
- Whack(destatep, F_UTF8UTF8, kBadPairWhack * 8);
- }
- // 2011.11.07 never use UTF8CP1252 -- answer will be UTF8 instead
- Whack(destatep, F_UTF8CP1252, kBadPairWhack * 8);
-
- if (destatep->debug_data != NULL) {
- // Show first four bytes of the input
- char buff[16];
- snprintf(buff, sizeof(buff), "%04x%04x", pair01, pair23);
- SetDetailsEncProb(destatep, 0, best_enc, buff);
- }
- }
-
-
-
- // Descending order
- int IntCompare(const void* v1, const void* v2) {
- const int* p1 = reinterpret_cast<const int*>(v1);
- const int* p2 = reinterpret_cast<const int*>(v2);
- if (*p1 < *p2) {return 1;}
- if (*p1 > *p2) {return -1;}
- return 0;
- }
-
- bool Base64Char(uint8 c) {
- if (('A' <= c) && (c <= 'Z')) {return true;}
- if (('a' <= c) && (c <= 'z')) {return true;}
- if (('0' <= c) && (c <= '9')) {return true;}
- if ('+' == c) {return true;}
- if ('/' == c) {return true;}
- return false;
- }
-
- int Base64ScanLen(const uint8* start, const uint8* limit) {
- // We have a plausible beginning; scan entire base64 string
- const uint8* ib64str = start;
- const uint8* b64str = ib64str;
- const uint8* b64strlimit = limit;
- // if starts with + +++, assume it is drawing, so bogus
- if (((limit - start) > 3) && (start[0] == '+') &&
- (start[1] == '+') && (start[2] == '+')) {
- return 81;
- }
- // Scan over base64
- while ((b64str < b64strlimit) && (kBase64Value[*b64str++] >= 0)) {
- }
- b64str--; // We overshot by 1
- return b64str - ib64str;
- }
-
- // Input is at least 8-character legal base64 string after +.
- // But might be say + "Presse+Termine"
- bool GoodUnicodeFromBase64(const uint8* start, const uint8* limit) {
- // Reject base64 string len N if density of '+' is > 1 + N/16 (expect 1/64)
- // Reject base64 string len N if density of A-Z is < 1 + N/16 (expect 26/64)
- // Reject base64 string len N if density of a-z is < 1 + N/16 (expect 26/64)
- // Reject base64 string len N if density of 0-9 is < 1 + N/32 (expect 10/64)
- // NOTE: this requires at least one lower AND one upper AND one digit to pass
- //
- int plus_count = 0;
- int lower_count = 0;
- int upper_count = 0;
- int digit_count = 0;
- int len = limit - start;
- for (const uint8* src = start; src < limit; ++src) {
- uint8 c = *src;
- if (('a' <= c) && (c <= 'z')) {
- ++lower_count;
- } else if (('A' <= c) && (c <= 'Z')) {
- ++upper_count;
- } else if (('0' <= c) && (c <= '0')) {
- ++digit_count;
- } else if (*src == '+') {
- ++plus_count;
- }
- }
-
- if (plus_count > (1 + (len >> 4))) {return false;}
- if (lower_count < (1 + (len >> 4))) {return false;}
- if (upper_count < (1 + (len >> 4))) {return false;}
- if (digit_count < (1 + (len >> 5))) {return false;}
-
- // checking the last character to reduce false positive
- // since the last character may be padded to 0 bits at the end.
- // refer to http://en.wikipedia.org/wiki/UTF-7
- int nmod8 = len & 7;
- const uint8 last = *(start+len-1);
- // When UTF-7 string length%8=3, the last two bits must be padded as 0
- if ((nmod8 == 3) && (kBase64Value[last] & 3)) {return false;}
- // When UTF-7 string length%8=6, the last four bits must be padded as 0
- if ((nmod8 == 6) && (kBase64Value[last] & 15)) {return false;}
- return true;
- }
-
- // Prune here after N bytes
- // Boost here for seven-bit sequences (at every prune)
- // if (sevenbitrankedencoding)
- // + UTF7 scan and boost/demote len mod 8 = 0 3 6
- // ~ Hz scan and boost/demote len mod 8 = 0 2 4 6
- // 1B 2022 scan and boost/demote len mod 8 = 0 2 4 6
- // 0E 2022 scan and boost/demote len mod 8 = 0 2 4 6
- // [0F 2022 boost/demote]
- // 00 UTF16/32 scan and boost/demote offset = even/odd
- //
- // If still some seven-bit possibilities > pure ASCII,
- // scan each possibility for clearer prob, s.t. about
- // two good sequences is a clear win
- // A-Z 00-19 00xx-64xx (B = 04xx)
- // a-z 1A-33 68xx-CCxx (f = 7Cxx)
- // 0-9 34-3D D0xx-F4xx (1 = D4xx)
- // + 3E F8xx
- // / 3F FCxx
- // do another chunk with slow scan
-
-
- // Boost, whack, or leave alone UTF-7 probablilty
- void UTF7BoostWhack(DetectEncodingState* destatep, int next_pair, uint8 byte2) {
- int off = destatep->interesting_offsets[AsciiPair][next_pair];
- if (off >= destatep->prior_utf7_offset) {
- // Not part of a previous successful UTF-7 string
- ++destatep->utf7_starts;
-
- if (byte2 == '-') {
- // +- encoding for '+' neutral
- } else if (!Base64Char(byte2)) {
- // Not base64 -- not UTF-7, whack
- Whack(destatep, F_UTF7, kBadPairWhack); // Illegal pair
- } else {
- // Starts with base64 byte, might be a good UTF7 sequence
- const uint8* start = destatep->initial_src + off + 1; // over the +
- int n = Base64ScanLen(start, destatep->limit_src);
- int nmod8 = n & 7;
- if ((n == 3) || (n == 6)) {
- // short but legal -- treat as neutral
- } else if ((nmod8 == 0) | (nmod8 == 3) | (nmod8 == 6)) {
- // Good length. Check for good Unicode.
- if (GoodUnicodeFromBase64(start, start + n)) {
- // Good length and Unicode, boost
- Boost(destatep, F_UTF7, kBoostOnePair); // Found good
- destatep->prior_utf7_offset = off + n + 1;
- } else {
- // Bad Unicode. Whack
- Whack(destatep, F_UTF7, kBadPairWhack); // Illegal length
- }
- } else {
- // Bad length. Whack
- Whack(destatep, F_UTF7, kBadPairWhack); // Illegal length
- }
- }
- }
- }
-
- // Boost, whack, or leave alone HZ probablilty
- void HzBoostWhack(DetectEncodingState* destatep, uint8 byte2) {
- if ((byte2 == '{') || (byte2 == '}')) {
- Boost(destatep, F_HZ_GB_2312, kBoostOnePair); // Found ~{ or ~}
- } else if ((byte2 == '~') || (byte2 == '\n')) {
- destatep->enc_prob[F_HZ_GB_2312] += 0; // neutral
- } else {
- Whack(destatep, F_HZ_GB_2312, kBadPairWhack); // Illegal pair
- }
- }
-
- // Boost, whack, or leave alone BINARY probablilty
- void BinaryBoostWhack(DetectEncodingState* destatep, uint8 byte1, uint8 byte2) {
- int quadrant = ((byte1 & 0x80) >> 6) | ((byte2 & 0x80) >> 7);
- int bucket8x4 = ((byte1 & 0xe0) >> 3) | ((byte2 & 0xc0) >> 6);
- uint32 quad_mask = 1 << quadrant;
- uint32 bucket8x4_mask = 1 << bucket8x4;
- if ((destatep->binary_quadrants_seen & quad_mask) == 0) {
- destatep->binary_quadrants_seen |= quad_mask;
- destatep->binary_quadrants_count += 1;
- if (destatep->binary_quadrants_count == 4) {
- Boost(destatep, F_BINARY, kBoostOnePair * 2); // Found all 4 quadrants,
- // boost 2 pairs
- }
- }
- if ((destatep->binary_8x4_seen & bucket8x4_mask) == 0) {
- destatep->binary_8x4_seen |= bucket8x4_mask;
- destatep->binary_8x4_count += 1;
- if (destatep->binary_8x4_count >= 11) {
- Boost(destatep, F_BINARY, kBoostOnePair * 4); // Found 11+/20 buckets,
- // boost 4 pairs each time
- }
- }
- }
-
-
- // Demote UTF-16/32 on 0000 or FFFF, favoring Binary
- void UTF1632BoostWhack(DetectEncodingState* destatep, int offset, uint8 byte1) {
- if (byte1 == 0) { // We have 0000
- Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal pair
- Whack(destatep, F_UTF_16LE, kBadPairWhack); // Illegal pair
- switch (offset & 3) {
- case 0: // We get called with 0 4 8, etc. for ASCII/BMP as UTF-32BE
- Whack(destatep, F_UTF_32LE, kBadPairWhack); // Illegal pair
- Boost(destatep, F_UTF_32BE, kSmallInitDiff); // Good pair
- break;
- case 1: // We get called with 1 5 9, etc. for ASCII as UTF-32LE
- case 2: // We get called with 2 6 10, etc. for BMP as UTF-32LE
- Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal pair
- Boost(destatep, F_UTF_32LE, kSmallInitDiff); // Good pair
- break;
- case 3: // ambiguous
- break;
- }
- } else { // We have ffff
- Whack(destatep, F_UTF_32BE, kBadPairWhack); // Illegal pair
- Whack(destatep, F_UTF_32LE, kBadPairWhack); // Illegal pair
- Whack(destatep, F_UTF_16BE, kBadPairWhack); // Illegal pair
- Whack(destatep, F_UTF_16LE, kBadPairWhack); // Illegal pair
- }
- }
-
- // Make even offset
- void UTF16MakeEven(DetectEncodingState* destatep, int next_pair) {
- destatep->interesting_offsets[OtherPair][next_pair] &= ~1;
- }
-
- bool ConsecutivePair(DetectEncodingState* destatep, int i) {
- if (i <= 0) {
- return false;
- }
- return destatep->interesting_offsets[OtherPair][i] ==
- (destatep->interesting_offsets[OtherPair][i - 1] + 2);
- }
-
- // boost, whack, or leave alone UTF-8 probablilty
- // Any whacks are also applied to UTF8UTF8; CheckUTF8UTF8Seq assumes good UTF8
- // Returns total boost
- int CheckUTF8Seq(DetectEncodingState* destatep, int weightshift) {
- int startcount = destatep->prior_interesting_pair[OtherPair];
- int endcount = destatep->next_interesting_pair[OtherPair];
-
- int demotion_count = 0;
- for (int i = startcount; i < endcount; ++i) {
- int sub;
- char* s = &destatep->interesting_pairs[OtherPair][i * 2];
- // Demote four byte patterns that are more likely Latin1 than UTF-8
- // C9AE, DF92, DF93, DFAB. See note at top.
- // Demotion also boosts Latin1 and CP1252
- uint8 s0 = static_cast<uint8>(s[0]);
- uint8 s1 = static_cast<uint8>(s[1]);
- if ((s0 == 0xc9) && (s1 == 0xae)) {++demotion_count;}
- if ((s0 == 0xdf) && (s1 == 0x92)) {++demotion_count;}
- if ((s0 == 0xdf) && (s1 == 0x93)) {++demotion_count;}
- if ((s0 == 0xdf) && (s1 == 0xab)) {++demotion_count;}
-
- if (!ConsecutivePair(destatep, i)) {
- // Insert a blank into the sequence; avoid wrong splices
- sub = (' ' >> 4) & 0x0f;
- ++destatep->utf8_minicount[
- static_cast<int>(kMiniUTF8Count[static_cast<int>(destatep->next_utf8_ministate)][sub])];
- destatep->next_utf8_ministate =
- kMiniUTF8State[destatep->next_utf8_ministate][sub];
- }
- // Byte 0
- sub = (s0 >> 4) & 0x0f;
- ++destatep->utf8_minicount[
- static_cast<int>(kMiniUTF8Count[static_cast<int>(destatep->next_utf8_ministate)][sub])];
- destatep->next_utf8_ministate =
- kMiniUTF8State[destatep->next_utf8_ministate][sub];
- // Byte 1
- sub = (s1 >> 4) & 0x0f;
- ++destatep->utf8_minicount[
- static_cast<int>(kMiniUTF8Count[static_cast<int>(destatep->next_utf8_ministate)][sub])];
- destatep->next_utf8_ministate =
- kMiniUTF8State[destatep->next_utf8_ministate][sub];
- DCHECK((0 <= destatep->next_utf8_ministate) &&
- (destatep->next_utf8_ministate < 8));
- }
-
-
- // For the four specific byte combinations above, Latin1/CP1252 is more likely
- if (demotion_count > 0) {
- Boost(destatep, F_Latin1, kGentleOnePair * demotion_count);
- Boost(destatep, F_CP1252, kGentleOnePair * demotion_count);
- }
-
- // Boost UTF8 for completed good sequences
- int total_boost = 2 * destatep->utf8_minicount[2] +
- 3 * destatep->utf8_minicount[3] +
- 4 * destatep->utf8_minicount[4];
- // But not so much for demoted bytes
- total_boost -= (3 * demotion_count);
-
- total_boost *= kGentleOnePair;
- total_boost >>= weightshift;
- // Design: boost both UTF8 and UTF8UTF8 for each good sequence
- Boost(destatep, F_UTF8, total_boost);
- Boost(destatep, F_UTF8UTF8, total_boost);
-
- destatep->utf8_minicount[5] += destatep->utf8_minicount[2]; // total chars
- destatep->utf8_minicount[5] += destatep->utf8_minicount[3]; // total chars
- destatep->utf8_minicount[5] += destatep->utf8_minicount[4]; // total chars
- destatep->utf8_minicount[2] = 0;
- destatep->utf8_minicount[3] = 0;
- destatep->utf8_minicount[4] = 0;
-
- // Whack (2 bytes) for errors
- int error_whack = 2 * destatep->utf8_minicount[1];
- error_whack *= kGentlePairWhack;
- error_whack >>= weightshift;
- Whack(destatep, F_UTF8, error_whack);
- Whack(destatep, F_UTF8UTF8, error_whack);
- destatep->utf8_minicount[1] = 0;
-
- return total_boost - error_whack;
- }
-
-
- // Boost, whack, or leave alone UTF8UTF8 probablilty
- //
- // We are looking for
- // (1) chars ONLY in set UTF8(0080)..UTF8(00FF), including for 80..9F the
- // MS CP1252 mappings, and
- // (2) sequences of 2 or more such characters
- //
- // If so, we could be looking at some non-7-bit encoding extra-converted
- // to UTF-8. The most common observed is CP1252->UTF8 twice,
- // 1252=>UTF8 : 1252=>UTF8
- // where the colon means "take those bytes and pretend that they are 1252".
- // We have a couple of examples of BIG5 bytes converted as though
- // they were 1252,
- // BIG5 : 1252=>UTF8
- //
- // Of course, we don't want correctly converted 1252 to be flagged here
- // 1252=>UTF8
- // So we want the input high bytes to be in pairs or longer, hence the
- // output UTF8 in groups of four bytes or more
- //
- // Good chars: C2xx, C3xx,
- // Good chars: C592, C593, C5A0, C5A1, C5B8, C5BD, C5BE, C692, CB86, CB9C
- // Good chars: E280xx E282AC E284A2
- // C2xx 1100001x 10xxxxxx (128/128)
- // C5xx 11000101 10xx00xx (16/4)
- // C5xx 11000101 10111xxx (8/3)
- // C692 11000110 10010010 (1/1)
- // CBxx 11001011 100xx1x0 (8/2)
- // E28x 11100010 10000xx0 (4/3)
- //
- // Returns total boost
- int CheckUTF8UTF8Seq(DetectEncodingState* destatep, int weightshift) {
- int this_pair = destatep->prior_interesting_pair[OtherPair];
- int startbyteoffset = this_pair * 2;
- int endbyteoffset = destatep->next_interesting_pair[OtherPair] * 2;
- char* startbyte = &destatep->interesting_pairs[OtherPair][startbyteoffset];
- char* endbyte = &destatep->interesting_pairs[OtherPair][endbyteoffset];
-
- int pair_number = this_pair;
- for (char* s = startbyte; s < endbyte; s += 2) {
- int next = destatep->next_utf8utf8_ministate;
- if (!ConsecutivePair(destatep, pair_number)) {
- // Insert two blanks into the sequence to avoid wrong splices
- // go back to no odd-byte offset
- destatep->utf8utf8_odd_byte = 0;
- int sub = UTF88Sub(' ', ' ');
- ++destatep->utf8utf8_minicount[static_cast<int>(kMiniUTF8UTF8Count[next][sub])];
- next = kMiniUTF8UTF8State[next][sub];
- }
-
- int odd = destatep->utf8utf8_odd_byte;
- if (s + 1 + odd >= endbyte) continue;
- int sub = UTF88Sub(s[0 + odd], s[1 + odd]);
- destatep->utf8utf8_odd_byte ^= kMiniUTF8UTF8Odd[next][sub];
- ++destatep->utf8utf8_minicount[
- static_cast<int>(kMiniUTF8UTF8Count[next][sub])];
- destatep->next_utf8utf8_ministate = kMiniUTF8UTF8State[next][sub];
- ++pair_number;
- }
-
- // Boost for completed good sequences; each count covers two chars.
- // Design: boost UTF8UTF8 above UTF8 for each good sequence
- int total_boost = (2) * destatep->utf8utf8_minicount[2] +
- (2) * destatep->utf8utf8_minicount[3] +
- (2) * destatep->utf8utf8_minicount[4];
- total_boost *= kGentleOnePair;
- total_boost >>= weightshift;
- Boost(destatep, F_UTF8UTF8, total_boost);
-
- // Track total characters
- destatep->utf8utf8_minicount[5] += destatep->utf8utf8_minicount[2];
- destatep->utf8utf8_minicount[5] += destatep->utf8utf8_minicount[3];
- destatep->utf8utf8_minicount[5] += destatep->utf8utf8_minicount[4];
- destatep->utf8utf8_minicount[2] = 0;
- destatep->utf8utf8_minicount[3] = 0;
- destatep->utf8utf8_minicount[4] = 0;
-
- // Design: Do not whack UTF8UTF8 below UTF8 for each bad sequence
-
- destatep->utf8utf8_minicount[1] = 0;
- return total_boost;
- }
-
-
- // We give a gentle boost for each paired SO ... SI, whack others
- void CheckIso2022ActiveSeq(DetectEncodingState* destatep) {
- int this_pair = destatep->prior_interesting_pair[OtherPair];
- int startbyteoffset = this_pair * 2;
- int endbyteoffset = destatep->next_interesting_pair[OtherPair] * 2;
- char* startbyte = &destatep->interesting_pairs[OtherPair][startbyteoffset];
- char* endbyte = &destatep->interesting_pairs[OtherPair][endbyteoffset];
-
- // Initial <esc> char must precede SO/SI
- // HZ_GB_2312 has no alternation constraint on 1- and 2-byte segments
- // ISO-2022-JP (JIS) has no alternation constraint on 1- and 2-byte segments
- // ISO-2022-CN has no alternation constraint on 1- and 2-byte segments
- // ISO-2022-KR requires alternation between 1- and 2-byte segments
- // JIS:
- // <esc> ( B ISO-2022-JP [1b 28 42] SI to ASCII
- // <esc> ( J ISO-2022-JP [1b 28 4a] SI to X0201
- // <esc> $ @ ISO-2022-JP [1b 24 40] SO to X0208-78 twobyte
- // <esc> $ B ISO-2022-JP [1b 24 42] SO to X0208-83 twobyte
- for (char* s = startbyte; s < endbyte; s += 2) {
- if (s[0] == 0x1b) {
- if (s[1] == 0x24) {
- // <esc> $ is SO
- destatep->next_2022_state = SOSI_TWOBYTE; // SO to two-byte
- } else if (s[1] == 0x28) {
- if (destatep->next_2022_state == SOSI_TWOBYTE) {
- Boost(destatep, F_JIS, kGentlePairBoost);
- } else if (destatep->next_2022_state == SOSI_ONEBYTE) {
- Whack(destatep, F_JIS, kGentlePairWhack);
- }
- destatep->next_2022_state = SOSI_ONEBYTE; // JIS SI to one-byte
- } else {
- Whack(destatep, F_JIS, kBadPairWhack);
- Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
- Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
- destatep->next_2022_state = SOSI_ERROR; // not 2022
- }
- } else if (s[0] == 0x0e) {
- // <so>
- Whack(destatep, F_JIS, kBadPairWhack);
- if (destatep->next_2022_state != SOSI_NONE) {
- destatep->next_2022_state = SOSI_TWOBYTE; // SO to two-byte
- } else {
- // ESC required before SO/SI
- Whack(destatep, F_ISO_2022_CN, kBadPairWhack * 4);
- Whack(destatep, F_ISO_2022_KR, kBadPairWhack * 4);
- destatep->next_2022_state = SOSI_ERROR; // SO not after SI
- }
- } else if (s[0] == 0x0f) {
- // <si>
- Whack(destatep, F_JIS, kBadPairWhack);
- if (destatep->next_2022_state != SOSI_NONE) {
- if (destatep->next_2022_state == SOSI_TWOBYTE) {
- Boost(destatep, F_ISO_2022_CN, kGentlePairBoost);
- Boost(destatep, F_ISO_2022_KR, kGentlePairBoost);
- } else if (destatep->next_2022_state == SOSI_ONEBYTE) {
- Whack(destatep, F_ISO_2022_CN, kGentlePairWhack);
- Whack(destatep, F_ISO_2022_KR, kGentlePairWhack);
- }
- destatep->next_2022_state = SOSI_ONEBYTE; // SI to one-byte
- } else {
- // ESC required before SO/SI
- Whack(destatep, F_ISO_2022_CN, kBadPairWhack * 4);
- Whack(destatep, F_ISO_2022_KR, kBadPairWhack * 4);
- destatep->next_2022_state = SOSI_ERROR; // SI not after SO
- }
- } else if (s[0] <= 0x1f) {
- // Some other control code. Allow ht lf [ff] cr
- if ((s[0] != 0x09) && (s[0] != 0x0a) &&
- (s[0] != 0x0c) && (s[0] != 0x0d)) {
- // Otherwise these can float to the top on bad bytes
- Whack(destatep, F_JIS, kBadPairWhack);
- Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
- Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
- }
- }
- }
-
- // If no start, keep the probability pinned at zero (or below)
- if (destatep->next_2022_state == SOSI_NONE) {
- destatep->enc_prob[F_ISO_2022_CN] =
- minint(0, destatep->enc_prob[F_ISO_2022_CN]);
- destatep->enc_prob[F_ISO_2022_KR] =
- minint(0, destatep->enc_prob[F_ISO_2022_KR]);
- destatep->enc_prob[F_JIS] =
- minint(0, destatep->enc_prob[F_JIS]);
- }
- }
-
- // We give a gentle boost for each paired ~{ ... ~}, whack others
- void CheckHzActiveSeq(DetectEncodingState* destatep) {
- int this_pair = destatep->prior_interesting_pair[AsciiPair];
- int startbyteoffset = this_pair * 2;
- int endbyteoffset = destatep->next_interesting_pair[AsciiPair] * 2;
- char* startbyte = &destatep->interesting_pairs[AsciiPair][startbyteoffset];
- char* endbyte = &destatep->interesting_pairs[AsciiPair][endbyteoffset];
-
- for (char* s = startbyte; s < endbyte; s += 2) {
- // Look for initial ~{ pair
- if ((s[0] == '~') && (s[1] == '{')) {
- destatep->next_hz_state = SOSI_TWOBYTE; // SO to two-byte
- }
- // Also look for closing ~} pair
- if ((s[0] == '~') && (s[1] == '}')) {
- if (destatep->next_hz_state == SOSI_TWOBYTE) {
- Boost(destatep, F_HZ_GB_2312, kGentlePairBoost);
- } else if (destatep->next_hz_state == SOSI_ONEBYTE) {
- Whack(destatep, F_HZ_GB_2312, kGentlePairWhack);
- }
- destatep->next_hz_state = SOSI_ONEBYTE; // SI to one-byte
- }
- }
-
- // If no start, keep the probability pinned at zero (or below)
- if (destatep->next_hz_state == SOSI_NONE) {
- destatep->enc_prob[F_HZ_GB_2312] =
- minint(0, destatep->enc_prob[F_HZ_GB_2312]);
- }
- }
-
- // We give a gentle boost after an odd number of 8Fxxxx triples, which
- // put subsequent bigrams out of phase until a low byte or another 8Fxxxx
- void CheckEucJpSeq(DetectEncodingState* destatep) {
- int this_pair = destatep->prior_interesting_pair[OtherPair];
- int startbyteoffset = this_pair * 2;
- int endbyteoffset = destatep->next_interesting_pair[OtherPair] * 2;
- char* startbyte = &destatep->interesting_pairs[OtherPair][startbyteoffset];
- char* endbyte = &destatep->interesting_pairs[OtherPair][endbyteoffset];
-
- for (char* s = startbyte; s < endbyte; s += 2) {
- // Boost if out of phase (otherwise, EUC-JP will score badly after 8Fxxxx)
- if (destatep->next_eucjp_oddphase) {
- //printf(" EucJp boost[%02x%02x]\n", s[0], s[1]); // TEMP
- Boost(destatep, F_EUC_JP, kGentlePairBoost * 2);
- }
-
- uint8 s0 = static_cast<uint8>(s[0]);
- uint8 s1 = static_cast<uint8>(s[1]);
- // Look for phase flip at 8F
- if ((s0 & 0x80) == 0x00) {
- destatep->next_eucjp_oddphase = false;
- } else if (s0 == 0x8f) {
- destatep->next_eucjp_oddphase = !destatep->next_eucjp_oddphase;
- }
- if ((s1 & 0x80) == 0x00) {
- destatep->next_eucjp_oddphase = false;
- } else if (s1 == 0x8f) {
- destatep->next_eucjp_oddphase = !destatep->next_eucjp_oddphase;
- }
- }
- }
-
- // Boost, whack, or leave alone BINARY probablilty
- // Also called if UTF 16/32 active
- void CheckBinaryDensity(const uint8* src, DetectEncodingState* destatep,
- int delta_otherpairs) {
- // No change if not much gathered information
- if (delta_otherpairs == 0) {
- // Only ASCII pairs this call
- return;
- }
- int next_pair = destatep->next_interesting_pair[OtherPair];
-
- // Look at density of interesting pairs [0..src)
- int delta_offset = static_cast<int>(src - destatep->initial_src); // actual
-
- // Look at density of interesting pairs [0..next_interesting)
- int low_byte = destatep->interesting_offsets[OtherPair][0];
- //int high_byte = destatep->interesting_offsets[OtherPair][next_pair - 1] + 2;
- //int byte_span = high_byte - low_byte;
- int byte_span = delta_offset - low_byte;
-
- // If all ASCII for the first 4KB, reject
- // If mostly ASCII in the first 5KB, reject
- if ((low_byte >= kBinaryHardAsciiLimit) || (delta_offset >= kBinarySoftAsciiLimit)) {
- // Not binary early enough in text
- Whack(destatep, F_BINARY, kBadPairWhack * 4);
- Whack(destatep, F_UTF_32BE, kBadPairWhack * 4);
- Whack(destatep, F_UTF_32LE, kBadPairWhack * 4);
- Whack(destatep, F_UTF_16BE, kBadPairWhack * 4);
- Whack(destatep, F_UTF_16LE, kBadPairWhack * 4);
- return;
- }
-
- // Density 1.0 for N pairs takes 2*N bytes
- // Whack if < 1/16 after first non_ASCII pair
- if ((next_pair * 2 * 16) < byte_span) {
- // Not dense enough
- Whack(destatep, F_BINARY, kBadPairWhack * 4);
- Whack(destatep, F_UTF_32BE, kBadPairWhack * 4);
- Whack(destatep, F_UTF_32LE, kBadPairWhack * 4);
- Whack(destatep, F_UTF_16BE, kBadPairWhack * 4);
- Whack(destatep, F_UTF_16LE, kBadPairWhack * 4);
- }
-
- if (next_pair < 8) {
- // Fewer than 8 non-ASCII total; too soon to boost
- return;
- }
-
- // Density 1.0 for N pairs takes 2*N bytes
- // Boost if density >= 1/4, whack if < 1/16
- if ((next_pair * 2 * 4) >= byte_span) {
- // Very dense
- // Only boost if at least 2 quadrants seen
- if (destatep->binary_quadrants_count >= 2) {
- Boost(destatep, F_BINARY, kSmallInitDiff);
- Boost(destatep, F_UTF_32BE, kSmallInitDiff);
- Boost(destatep, F_UTF_32LE, kSmallInitDiff);
- Boost(destatep, F_UTF_16BE, kSmallInitDiff);
- Boost(destatep, F_UTF_16LE, kSmallInitDiff);
- }
- }
- }
-
-
- // Look at a number of special-case encodings whose reliable detection depends
- // on sequencing or other properties
- // AsciiPair probibilities (UTF7 and HZ) are all done here
- void ActiveSpecialBoostWhack(const uint8* src, DetectEncodingState* destatep) {
- int delta_asciipairs = destatep->next_interesting_pair[AsciiPair] -
- destatep->prior_interesting_pair[AsciiPair];
- int delta_otherpairs = destatep->next_interesting_pair[OtherPair] -
- destatep->prior_interesting_pair[OtherPair];
-
- // The two pure ASCII encodings
- if (UTF7OrHzActive(destatep) && (delta_asciipairs > 0)) {
- // Adjust per pair
- for (int i = 0; i < delta_asciipairs; ++i) {
- int next_pair = destatep->prior_interesting_pair[AsciiPair] + i;
- uint8 byte1 = destatep->interesting_pairs[AsciiPair][next_pair * 2 + 0];
- uint8 byte2 = destatep->interesting_pairs[AsciiPair][next_pair * 2 + 1];
- if (byte1 == '+') {
- // Boost, whack, or leave alone UTF-7 probablilty
- UTF7BoostWhack(destatep, next_pair, byte2);
- if (destatep->debug_data != NULL) {
- // Show UTF7 entry
- char buff[16];
- snprintf(buff, sizeof(buff), "%02x%02x+", byte1, byte2);
- SetDetailsEncProb(destatep,
- destatep->interesting_offsets[AsciiPair][next_pair],
- kMostLikelyEncoding[(byte1 << 8) + byte2],
- buff);
- }
- } else if (byte1 == '~') {
- // Boost, whack, or leave alone HZ probablilty
- HzBoostWhack(destatep, byte2);
- if (destatep->debug_data != NULL) {
- // Show Hz entry
- char buff[16];
- snprintf(buff, sizeof(buff), "%02x%02x~", byte1, byte2);
- SetDetailsEncProb(destatep,
- destatep->interesting_offsets[AsciiPair][next_pair],
- kMostLikelyEncoding[(byte1 << 8) + byte2],
- buff);
- }
- }
- }
-
- // Kill UTF-7 now if at least 8 + pairs and not confirmed valid UTF-7
- if ((destatep->utf7_starts >= 8) && (destatep->prior_utf7_offset == 0)) {
- Whack(destatep, F_UTF7, kBadPairWhack * 8); // flush
- }
- }
-
-
-
- // All the other encodings
- if (OtherActive(destatep) && (delta_otherpairs > 0)) {
- // Adjust per pair
- int biggest_weightshift = 0;
- for (int i = 0; i < delta_otherpairs; ++i) {
- int next_pair = destatep->prior_interesting_pair[OtherPair] + i;
- uint8 byte1 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 0];
- uint8 byte2 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 1];
- int off = destatep->interesting_offsets[OtherPair][next_pair];
- int weightshift = destatep->interesting_weightshift[OtherPair][next_pair];
- biggest_weightshift = maxint(biggest_weightshift, weightshift);
-
- if (byte1 == 0x00) {
- if (byte2 == 0x00) {
- UTF1632BoostWhack(destatep, off, byte1);
- } else if ((kIsPrintableAscii[byte2] != 0) && ((off & 1) != 0)) {
- // We have 00xx at an odd offset. Turn into preceding even offset
- // for possible Ascii text in UTF-16LE or UTF-32LE (vs BE)
- // This will cascade into caller's probability update
- // 00 is illegal for all other encodings, so it doesn't matter to them
- UTF16MakeEven(destatep, next_pair);
- }
- if (destatep->debug_data != NULL) {
- // Show 0000 detail entry for this bigram
- char buff[16];
- snprintf(buff, sizeof(buff), "%02x%02xZ", byte1, byte2);
- SetDetailsEncProb(destatep,
- destatep->interesting_offsets[OtherPair][next_pair],
- kMostLikelyEncoding[(byte1 << 8) + byte2],
- buff);
- }
- }
- if (byte1 == 0xff) {
- if (byte2 == 0xff) {
- UTF1632BoostWhack(destatep, off, byte1);
- }
- if (destatep->debug_data != NULL) {
- // Show FFFF detail entry for this bigram
- char buff[16];
- snprintf(buff, sizeof(buff), "%02x%02xF", byte1, byte2);
- SetDetailsEncProb(destatep,
- destatep->interesting_offsets[OtherPair][next_pair],
- kMostLikelyEncoding[(byte1 << 8) + byte2],
- buff);
- }
- }
- if (BinaryActive(destatep)) {
- BinaryBoostWhack(destatep, byte1, byte2);
- }
- } // End for i
-
- // Adjust per entire-pair-span
- if (UTF8Active(destatep)) {
- CheckUTF8Seq(destatep, biggest_weightshift);
- }
-
- if (UTF8UTF8Active(destatep)) {
- CheckUTF8UTF8Seq(destatep, biggest_weightshift);
- }
-
- if (Iso2022Active(destatep)) {
- CheckIso2022ActiveSeq(destatep);
- }
-
- if (HzActive(destatep)) {
- CheckHzActiveSeq(destatep);
- }
-
- if (EUCJPActive(destatep)) {
- CheckEucJpSeq(destatep);
- }
-
- if (BinaryActive(destatep) || UTF1632Active(destatep)) {
- CheckBinaryDensity(src, destatep, delta_otherpairs);
- }
- }
- // ISO-2022 do OK on their own, using stright probabilities? Not on bad bytes
-
- if (destatep->debug_data != NULL) {
- // Show sequencing result
- SetDetailsEncLabel(destatep, "seq");
- }
- }
-
-
- void PrintTopEnc(DetectEncodingState* destatep, int n) {
- // Print top n or fewer
- int temp_sort[NUM_RANKEDENCODING];
- for (int j = 0; j < destatep->rankedencoding_list_len; ++j) {
- int rankedencoding = destatep->rankedencoding_list[j];
- temp_sort[j] = destatep->enc_prob[rankedencoding];
- }
-
- qsort(temp_sort, destatep->rankedencoding_list_len,
- sizeof(temp_sort[0]), IntCompare);
-
- int top_n = minint(n, destatep->rankedencoding_list_len);
- int showme = temp_sort[top_n - 1]; // Print this value and above
-
- printf("rankedencodingList top %d: ", top_n);
- for (int j = 0; j < destatep->rankedencoding_list_len; ++j) {
- int rankedencoding = destatep->rankedencoding_list[j];
- if (showme <= destatep->enc_prob[rankedencoding]) {
- printf("%s=%d ",
- MyEncodingName(kMapToEncoding[rankedencoding]),
- destatep->enc_prob[rankedencoding]);
- }
- }
- printf("\n\n");
- }
-
- // If the same bigram repeats, don't boost its best encoding too much
- bool RepeatedBigram(DetectEncodingState* destatep, uint8 byte1, uint8 byte2) {
- int this_bigram = (byte1 << 8) | byte2;
- // If 00xx 01xx 02xx ... 1fxx, take out bottom 4 bits of xx.
- // This ignores parts of Yahoo 0255 0254 0243 0247 0245 0243 0250 0255 ...
- // It may screw up UTF-16BE
- // It may screw up ISO-2022 (1b24 suppresses 1b28)
- if (byte1 < 0x20) {
- this_bigram &= 0xfff0;
- }
- if (this_bigram == destatep->prior_bigram[0]) {return true;}
- if (this_bigram == destatep->prior_bigram[1]) {return true;}
- if (this_bigram == destatep->prior_bigram[2]) {return true;}
- if (this_bigram == destatep->prior_bigram[3]) {return true;}
- // Round-robin replacement
- destatep->prior_bigram[destatep->next_prior_bigram] = this_bigram;
- destatep->next_prior_bigram = (destatep->next_prior_bigram + 1) & 3;
- return false;
- }
-
- // Sometimes illegal bytes are used as markers between text that Javascript
- // is going to decode. Don't overboost the Binary encoding for markers 01-FF.
- // Just count first pair per 8x4 bucket
- bool RepeatedBinary(DetectEncodingState* destatep, uint8 byte1, uint8 byte2) {
- int bucket8x4 = ((byte1 & 0xe0) >> 3) | ((byte2 & 0xc0) >> 6);
- uint32 bucket8x4_mask = 1 << bucket8x4;
- if ((destatep->binary_8x4_seen & bucket8x4_mask) == 0) {
- destatep->binary_8x4_seen |= bucket8x4_mask;
- destatep->binary_8x4_count += 1;
- return false;
- }
- return true;
- }
-
-
-
-
- // Find current top two rankedencoding probabilities
- void ReRank(DetectEncodingState* destatep) {
- destatep->top_prob = -1;
- destatep->second_top_prob = -1;
- // Leave unchanged
- //destatep->top_rankedencoding =
- // destatep->rankedencoding_list[0]; // Just to make well-defined
- //destatep->second_top_rankedencoding =
- // destatep->rankedencoding_list[1]; // Just to make well-defined
- for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
- int rankedencoding = destatep->rankedencoding_list[j];
- if (destatep->top_prob < destatep->enc_prob[rankedencoding]) {
- // Make sure top 2 are in different superset groups
- if (kMapEncToBaseEncoding[kMapToEncoding[destatep->top_rankedencoding]] !=
- kMapEncToBaseEncoding[kMapToEncoding[rankedencoding]]) {
- destatep->second_top_prob =
- destatep->top_prob; // old top to second
- destatep->second_top_rankedencoding =
- destatep->top_rankedencoding; // old top to second
- }
- destatep->top_prob = destatep->enc_prob[rankedencoding];
- destatep->top_rankedencoding = rankedencoding;
- } else if (destatep->second_top_prob < destatep->enc_prob[rankedencoding]) {
- if (kMapEncToBaseEncoding[kMapToEncoding[destatep->top_rankedencoding]] !=
- kMapEncToBaseEncoding[kMapToEncoding[rankedencoding]]) {
- destatep->second_top_prob = destatep->enc_prob[rankedencoding];
- destatep->second_top_rankedencoding = rankedencoding;
- }
- }
- }
- }
-
- void SimplePrune(DetectEncodingState* destatep, int prune_diff) {
- // Prune the list of active encoding families
- int keep_prob = destatep->top_prob - prune_diff;
-
- destatep->active_special = 0;
- int k = 0;
- for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
- bool keep = true;
- int rankedencoding = destatep->rankedencoding_list[j];
-
- // If count is too low, ditch it
- if (destatep->enc_prob[rankedencoding] < keep_prob) {keep = false;}
-
- // Keep it. This will always keep at least top_prob rankedencoding
- if (keep) {
- destatep->active_special |= kSpecialMask[kMapToEncoding[rankedencoding]];
- destatep->rankedencoding_list[k++] = rankedencoding;
- }
- }
-
- destatep->rankedencoding_list_len = k;
- }
-
- // Recalculate reliable
- void CalcReliable(DetectEncodingState* destatep) {
- // Encoding result is reliable if big difference in top two, or if
- // only Ascii7 ever encountered
- // Also reliable if exactly one OtherPair and it's best encoding matches top
- destatep->reliable = false;
- if (destatep->next_interesting_pair[OtherPair] == 0) {
- // Only 7-bit ASCII
- destatep->reliable = true;
- return;
- }
- if ((destatep->top_prob - destatep->second_top_prob) >=
- FLAGS_ced_reliable_difference) {
- destatep->reliable = true;
- return;
- }
- if (destatep->next_interesting_pair[OtherPair] == 1) {
- uint8 byte1 = destatep->interesting_pairs[OtherPair][0];
- uint8 byte2 = destatep->interesting_pairs[OtherPair][1];
- int best_enc = kMostLikelyEncoding[(byte1 << 8) + byte2];
- if (best_enc == destatep->top_rankedencoding) {
- destatep->reliable = true;
- return;
- }
- }
-
- // If we pruned to one encoding, we are done
- if (destatep->rankedencoding_list_len == 1) {
- destatep->reliable = true;
- destatep->done = true;
- return;
- }
-
- // If we pruned to two or three encodings in the same *superset/subset
- // rankedencoding* and enough pairs, we are done. Else keep going
- if (destatep->rankedencoding_list_len == 2) {
- Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
- Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
- if (kMapEncToBaseEncoding[enc0] == kMapEncToBaseEncoding[enc1]) {
- if (destatep->prune_count >= 3) {
- destatep->reliable = true;
- destatep->done = true;
- return;
- }
- }
- } else if (destatep->rankedencoding_list_len == 3) {
- Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
- Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
- Encoding enc2 = kMapToEncoding[destatep->rankedencoding_list[2]];
- Encoding base0 = kMapEncToBaseEncoding[enc0];
- Encoding base1 = kMapEncToBaseEncoding[enc1];
- Encoding base2 = kMapEncToBaseEncoding[enc2];
-
- if ((base0 == base1) && (base0 == base2)) {
- if (destatep->prune_count >= 3) {
- destatep->reliable = true;
- destatep->done = true;
- return;
- }
- }
- }
-
- }
-
-
- // Find current top two rankedencoding probabilities
- void FindTop2(DetectEncodingState* destatep,
- int* first_renc, int* second_renc,
- int* first_prob, int* second_prob) {
- *first_prob = -1;
- *second_prob = -1;
- *first_renc = 0;
- *second_renc = 0;
- for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
- int rankedencoding = destatep->rankedencoding_list[j];
- if (*first_prob < destatep->enc_prob[rankedencoding]) {
- *second_prob = *first_prob; // old top to second
- *second_renc = *first_renc; // old top to second
- *first_prob = destatep->enc_prob[rankedencoding];
- *first_renc = rankedencoding;
- } else if (*second_prob < destatep->enc_prob[rankedencoding]) {
- *second_prob = destatep->enc_prob[rankedencoding];
- *second_renc = rankedencoding;
- }
- }
- }
-
-
- void PrintRankedEncodingList(DetectEncodingState* destatep, const char* str) {
- printf("Current ranked encoding list %s\n", str);
- for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
- int rankedencoding = destatep->rankedencoding_list[j];
- if ((rankedencoding < 0) || (rankedencoding > NUM_RANKEDENCODING)) {
- printf(" [%d] BOGUS rankedencoding = %d\n", j, rankedencoding);
- } else {
- printf(" [%d] rankedencoding = %d %-12.12s enc_prob = %d\n",
- j, rankedencoding, MyRankedEncName(rankedencoding),
- destatep->enc_prob[rankedencoding]);
- }
- }
- printf("End current ranked encoding list\n\n");
- }
-
-
-
-
- // Map unencoded bytes down to five bits, largely preserving letters
- // This design struggles to put 33 values into 5 bits.
- #define XX 0 // Punctuation (00-7F range)
- #define HA 27 // High vowel a in Latin1/2/sometimes7
- #define HE 28 // High vowel e
- #define HI 29 // High vowel i
- #define HO 30 // High vowel o
- #define HU 30 // High vowel u on top of HO
- #define Hc 31 // High consonant (80-FF range)
- static const char kMapToFiveBits[256] = {
- XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
- XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
- XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
- XX,XX,XX,XX,XX,XX,XX,XX, XX,XX,XX,XX,XX,XX,XX,XX,
-
- XX, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,
- 16,17,18,19,20,21,22,23, 24,25,26,XX,XX,XX,XX,XX,
- XX, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,
- 16,17,18,19,20,21,22,23, 24,25,26,XX,XX,XX,XX,XX,
-
- Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
- Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
- Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
- Hc,HA,Hc,Hc,Hc,Hc,Hc,Hc, HO,Hc,Hc,Hc,Hc,Hc,Hc,Hc,
-
- Hc,HA,HA,HA,HA,Hc,Hc,Hc, Hc,HE,HE,HE,HI,HI,HI,Hc,
- Hc,Hc,Hc,HO,HO,HO,HO,Hc, Hc,HU,HU,HU,HU,Hc,Hc,Hc,
- Hc,HA,HA,HA,HA,Hc,Hc,Hc, Hc,HE,HE,HE,HI,HI,HI,Hc,
- Hc,Hc,Hc,HO,HO,HO,HO,Hc, Hc,HU,HU,HU,HU,Hc,Hc,Hc,
-
- };
- #undef XX
- #undef HA
- #undef HE
- #undef HI
- #undef HO
- #undef HU
- #undef Hc
-
- static const int kTriLatin1Likely = 1;
- static const int kTriLatin2Likely = 2;
- static const int kTriLatin7Likely = 3;
-
- // Each table entry has 32 times two bits, selected by byte[2]
- // Entry subscript is selected by byte[0] and byte[1]
- // Latin1/2/7 boost vector, generated 2007.09.26 by postproc-enc-detect-short.cc
- static const uint64 kLatin127Trigrams[1024] = {
- 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x304080c0402c3330ULL, 0x0008400004000000ULL, 0x082800000c200000ULL,
- 0x23a0000420800030ULL, 0x00000000000ccc00ULL, 0x0500100100100000ULL, 0x0388400000200010ULL,
- 0x0000000000000c00ULL, 0xd0f0300740f0cf00ULL, 0x2aa0a2a22882a2acULL, 0x081d800000000080ULL,
- 0x0c82000020000000ULL, 0x200a03c000a00000ULL, 0x0008400400290000ULL, 0x0400870000000000ULL,
- 0x00f040c00000c080ULL, 0x0008004000000410ULL, 0x0020300000000030ULL, 0x00a030002c300000ULL,
- 0x0c8030c020a00000ULL, 0x15410030f0f4c000ULL, 0x3000000300a00000ULL, 0xa2880980a0880a88ULL,
- 0x0900300000000000ULL, 0x0000040100300000ULL, 0x0888820020a00000ULL, 0xc044002242010000ULL,
- 0x000000121d300040ULL, 0x40100040440c0d54ULL, 0x00008423102f8144ULL, 0x0b40808400000280ULL,
- 0x0000000000000000ULL, 0x0680a000000c0000ULL, 0x0880008020aa0000ULL, 0x2aaa0141010a4940ULL,
- 0xcb80000000010000ULL, 0x2280000000000000ULL, 0x5248000001800000ULL, 0x8000401004040010ULL,
- 0x1540010201001010ULL, 0x0080080400000000ULL, 0x5a00044040000108ULL, 0x0288000282080008ULL,
- 0x4800008002200000ULL, 0x4a00000000010100ULL, 0x8a88040080000800ULL, 0x0140800000000400ULL,
- 0x40010050000c0000ULL, 0x0000008000000000ULL, 0x0028000020140040ULL, 0x8620401401005308ULL,
- 0xc082000000000400ULL, 0x05c0b004c0240600ULL, 0x0288000080000000ULL, 0x0000014000000000ULL,
- 0x00000000040000c0ULL, 0x8001861008004280ULL, 0x0200000000000300ULL, 0x0000240242288620ULL,
- 0x801000c05434c200ULL, 0x9020162040a2d2b4ULL, 0x0021840000240704ULL, 0x2a80280080084908ULL,
- 0x0000000000000000ULL, 0x0500004000000040ULL, 0x0080000000040000ULL, 0x0108058104440000ULL,
- 0x0900000000040000ULL, 0x00c0000000208008ULL, 0x2000005000000000ULL, 0x0080000000050000ULL,
- 0x0808000000001080ULL, 0x9880810100308000ULL, 0x2285480080081a08ULL, 0x8a80000080080000ULL,
- 0x1450000000600010ULL, 0x2210000100000000ULL, 0x8a88000100011000ULL, 0x1541804000000010ULL,
- 0xc084011140040100ULL, 0x0000000000000800ULL, 0x0400000000000030ULL, 0x2a800000a0890128ULL,
- 0x1140a00054000104ULL, 0x1440000101200404ULL, 0x028800400400d800ULL, 0x0000000000000000ULL,
- 0x0000000000002330ULL, 0x0020820228a02280ULL, 0xa2888a02aa8008a8ULL, 0xd0040a0044202500ULL,
- 0x8000044104a29424ULL, 0xc000100178b2c5b4ULL, 0x0000810100241504ULL, 0xd040030000380008ULL,
- 0x0000000000000000ULL, 0x26c08c0000200130ULL, 0x4a08000110080000ULL, 0x2aa0004001080800ULL,
- 0x0aac000000004000ULL, 0x2000000000200000ULL, 0x4240000100020000ULL, 0x4100000080000000ULL,
- 0x4900040000000000ULL, 0x0800000400300040ULL, 0x6a80000000040800ULL, 0x2a08182000588008ULL,
- 0x0a00000c81000008ULL, 0x0a000c0010000000ULL, 0x8a88001080280808ULL, 0x0020000200300600ULL,
- 0xaac00000900a0000ULL, 0x0000100004000000ULL, 0x0020081020000000ULL, 0x8220105010084110ULL,
- 0x4a80800000004000ULL, 0x050000c0c0200000ULL, 0x288c000084000000ULL, 0xa048082280000000ULL,
- 0x0000000000000000ULL, 0x8000900000032080ULL, 0xee889e81b8880820ULL, 0xc2200a8142800424ULL,
- 0xc020141543361010ULL, 0x10a000204a801634ULL, 0x3a808800802a00a0ULL, 0x28808b00803d0800ULL,
- 0x0000000000000000ULL, 0x0020000000000030ULL, 0x0808400121010040ULL, 0x0c28240100200040ULL,
- 0x2008200028800000ULL, 0xc10004c80f30c030ULL, 0x0400440114100000ULL, 0x2208200280a22220ULL,
- 0x0600000030c01000ULL, 0x1201001040c00000ULL, 0x0aa02ea22aa22aa0ULL, 0x30008000000200a0ULL,
- 0x20c8400400800000ULL, 0x08280b0420800000ULL, 0x0800100000210000ULL, 0x10000300c0100400ULL,
- 0xc8c0000420000000ULL, 0x1000000010000000ULL, 0x0420000400000000ULL, 0x0220000500204000ULL,
- 0x2200000420000000ULL, 0x0000540400000000ULL, 0x0000000020000000ULL, 0x00080c00a0810080ULL,
- 0x1540000000043000ULL, 0x0000000000100000ULL, 0x2e88a22220200a20ULL, 0xc06030e34ea503a0ULL,
- 0x0001100204048500ULL, 0x000000e0000c0d54ULL, 0x3000820310a31400ULL, 0x13088c0320e00280ULL,
- 0x0000000000000000ULL, 0x0480000000200000ULL, 0x4000200100000000ULL, 0x0000300040040000ULL,
- 0x4400000000000000ULL, 0x0401000002240000ULL, 0x0540000000040000ULL, 0x4004010000000000ULL,
- 0x4001111001100000ULL, 0x2880000000300040ULL, 0x4040004040002404ULL, 0x0200000000000000ULL,
- 0x0140040000100000ULL, 0x4040010040040080ULL, 0x0a00140000041004ULL, 0x0000a00400808000ULL,
- 0x1010200000430040ULL, 0x0010000000000000ULL, 0x0540000000104000ULL, 0x1400114005000000ULL,
- 0x0000204000440010ULL, 0x0500000000004400ULL, 0x4500000018000400ULL, 0x0000400000000000ULL,
- 0x000000300000cc00ULL, 0x0100001011300000ULL, 0x0040000000000000ULL, 0xc0e0000248a00444ULL,
- 0x0000040020340144ULL, 0x0000046445105454ULL, 0x32a0a80280880128ULL, 0x0880040000100100ULL,
- 0x0000000000000000ULL, 0x14003000030c0004ULL, 0x4a04001100000000ULL, 0x0a00108010000000ULL,
- 0x28a8004000200248ULL, 0x0100040000b00000ULL, 0x42000000000008c0ULL, 0x6008044010550010ULL,
- 0x0800401000010400ULL, 0x080080040cf80000ULL, 0x5080000001001010ULL, 0x2a80100000000000ULL,
- 0xcc8010010d401100ULL, 0x0200000001001000ULL, 0x0480001004001000ULL, 0x8d00800040b40210ULL,
- 0x6200800000300000ULL, 0x0000010000000000ULL, 0x0428004100010000ULL, 0x4320105141501100ULL,
- 0xe28c0000000c1000ULL, 0xd5c000c3c0e00300ULL, 0x0001000000100200ULL, 0x1004010202400008ULL,
- 0x0000000000003000ULL, 0x2aa038a0800aab08ULL, 0x2a88038000000000ULL, 0xc220040242f09720ULL,
- 0x8020200200ba0420ULL, 0x0020106105101004ULL, 0x0480800000220400ULL, 0x2280100080000008ULL,
- 0x0000000000000000ULL, 0x9000000000200000ULL, 0x0001000000100000ULL, 0x2aa40c0000080800ULL,
- 0x0040000040010000ULL, 0x0040000000c01000ULL, 0x4000000040000400ULL, 0x0000001000200000ULL,
- 0x0000010000000000ULL, 0x05808004000c0000ULL, 0x50400c0000000400ULL, 0x020040008f000040ULL,
- 0x0800000000100000ULL, 0x0000000000000000ULL, 0x0a08440000004000ULL, 0x0064000400008200ULL,
- 0x0010010010034170ULL, 0x0000000010000000ULL, 0x0100204021000000ULL, 0x022000d000010100ULL,
- 0x0840300000c00000ULL, 0x1400000040204400ULL, 0x09800c0040000000ULL, 0x0209708000000000ULL,
- 0x000000000000c040ULL, 0x90000c50204040a0ULL, 0x0000000000000000ULL, 0x00e1500040200004ULL,
- 0x8020260540204494ULL, 0x0020026150201054ULL, 0x0281800380105634ULL, 0x0884900481105000ULL,
- 0x0000000000000000ULL, 0x84203c00002c0200ULL, 0xc089040000000000ULL, 0xc2a8100040200004ULL,
- 0xe00c1c0000000000ULL, 0x0ce1330080200080ULL, 0x0000000000200000ULL, 0xc400110000404010ULL,
- 0x0088400000000000ULL, 0x00083cc00c00c00cULL, 0xcac01c00c000580cULL, 0xe300b0f000100000ULL,
- 0x0300000000000000ULL, 0xc0000f0000000000ULL, 0xc3c01c0400000000ULL, 0x81008004c0f40000ULL,
- 0xc3d8003000000440ULL, 0x0000000000000000ULL, 0xc430000000000000ULL, 0x0060000000001000ULL,
- 0x0800000000000000ULL, 0x00c03300f0fc0008ULL, 0x3000000400200010ULL, 0xa2a80892a0880a28ULL,
- 0x0500000040000004ULL, 0x0000000000000000ULL, 0xc80032070c200020ULL, 0x0220820060a296a0ULL,
- 0x802084021db486a0ULL, 0x00000d60080c0080ULL, 0xb281803313a32428ULL, 0x1808300320300000ULL,
- 0x0000000000000000ULL, 0x85208cc0ccac1f20ULL, 0x2081000186100808ULL, 0x22a80880000a0808ULL,
- 0xaaa8086880000000ULL, 0x802084800a2e9200ULL, 0xa280000000002008ULL, 0xa000000080080400ULL,
- 0x2080010000000008ULL, 0x802020c00c028c80ULL, 0x2080000000140810ULL, 0x2a80086080080008ULL,
- 0x2a800000a8000800ULL, 0xaa881800a2080800ULL, 0xaa98004080280808ULL, 0x004483d0c0300000ULL,
- 0xa280002080080000ULL, 0x0000000000300000ULL, 0x22a1030000000008ULL, 0xa8a0301088880880ULL,
- 0xaa80002080222808ULL, 0x85400c03fc030400ULL, 0x8a88000000000008ULL, 0xa008008010080008ULL,
- 0x0000000000010000ULL, 0x0040100000301040ULL, 0x28800000a0002008ULL, 0x122482306cbc0eacULL,
- 0x8020224222b8c6a0ULL, 0x802002004a82c284ULL, 0x0aa08fc440a41c80ULL, 0x888080d181385098ULL,
- 0x0000000000000000ULL, 0x00c0b000000c0080ULL, 0x2208001000000800ULL, 0x0a28000000200000ULL,
- 0x0000000300000000ULL, 0x00c1040000200000ULL, 0x0203020000000000ULL, 0x0248000000020000ULL,
- 0x0000840000100000ULL, 0x0a808c00c000008cULL, 0x5200040040000004ULL, 0x02000c00000080a0ULL,
- 0x0b0c000020000000ULL, 0x0b04000001000000ULL, 0x088c0010002000c0ULL, 0x80e08b00c0030c20ULL,
- 0x0280000200014040ULL, 0x0000000000000000ULL, 0x0e20a0a008000020ULL, 0x0e280fd03f00111cULL,
- 0x200080c020001000ULL, 0x8cc00c02c02f0400ULL, 0x480c0001000c404cULL, 0x0208014281080808ULL,
- 0x000000000000fcfcULL, 0x004403300cf00030ULL, 0x2200000000004400ULL, 0x02202000c08c0c20ULL,
- 0x02202022683a80a0ULL, 0x4020228028008c00ULL, 0x32208cc0002c0200ULL, 0x3ec00c0080304008ULL,
- 0x0000000000000000ULL, 0x34000c00002c0000ULL, 0x0b00000100100030ULL, 0x0823018000000000ULL,
- 0x0e8c001c01e00000ULL, 0x1200800600330000ULL, 0x4000110000000000ULL, 0x0080000300000000ULL,
- 0x0800000000000000ULL, 0x08c08c04000c0000ULL, 0x0080400000880000ULL, 0x0a08000080c00008ULL,
- 0x0800000304400000ULL, 0x0208000000c00000ULL, 0x2888300080400800ULL, 0x8dc0204400000000ULL,
- 0xc0000000c0800000ULL, 0x0000c10000000000ULL, 0x24000c4010c00000ULL, 0x272000541d811000ULL,
- 0x0200400000001000ULL, 0x0400000400001004ULL, 0xc08c007004001000ULL, 0x2048004000000000ULL,
- 0x000000000003fcfcULL, 0x2aa030000cf8c800ULL, 0xe280000000000000ULL, 0x0a21008142000340ULL,
- 0x0021002000b61040ULL, 0x800004064006d444ULL, 0x3aa0800300230008ULL, 0x0b00030000300000ULL,
- 0x0000000000000000ULL, 0x01c080000000040cULL, 0x0100000000004000ULL, 0x0aa8018010001000ULL,
- 0x0800000000100000ULL, 0x3000000000008c00ULL, 0x5400000013000000ULL, 0x02c0c00004004010ULL,
- 0x5241100010000c00ULL, 0x0e00080000000808ULL, 0x5281000000000800ULL, 0x0a08108020000800ULL,
- 0x0a80000000005210ULL, 0x0100000041000000ULL, 0x2a88000002080110ULL, 0x8520800000c00080ULL,
- 0x01000010108c0100ULL, 0x0000000000000000ULL, 0x42a0420080000000ULL, 0x0020001004010010ULL,
- 0xc4000000000c0000ULL, 0x01000c00c0200400ULL, 0x4600000100000000ULL, 0x0000000000000000ULL,
- 0x0010001000000010ULL, 0x910400900820d030ULL, 0x2280000000000000ULL, 0xc2212004400040e4ULL,
- 0x8001000000b61420ULL, 0xa00002a248e810b4ULL, 0x32008000002c0008ULL, 0x0c010034803c5010ULL,
- 0x0000000000000000ULL, 0x85008002002c0000ULL, 0x0204001000004010ULL, 0x0120008000200000ULL,
- 0x000010000c2000c0ULL, 0xccc0000000200000ULL, 0x0400000c00100040ULL, 0x0003300100004100ULL,
- 0x4000551040000004ULL, 0x0e0080000c820808ULL, 0xc000000000080800ULL, 0xc803000000000000ULL,
- 0x0a4000c000200000ULL, 0x0040000000c00000ULL, 0x0918145000405000ULL, 0x81400000c0300400ULL,
- 0x0050000000000000ULL, 0xd000045000000000ULL, 0x0400004000400000ULL, 0x0420104010000110ULL,
- 0x0700000000203000ULL, 0x34800300c0e00704ULL, 0x4440100044000400ULL, 0x0040000040000000ULL,
- 0x0030000044000000ULL, 0xeaaca0008808c880ULL, 0x0a01000000200000ULL, 0x1220a300403ccf20ULL,
- 0x002024c200b61044ULL, 0x802014346aa2d434ULL, 0x30008c00c0820c44ULL, 0x0a000000000c4800ULL,
- 0x0000000000000000ULL, 0x0000404000340c90ULL, 0x08a8a10820800280ULL, 0x8128009022201000ULL,
- 0x0020808228a000a0ULL, 0x0020400100410000ULL, 0x0400000110000000ULL, 0xa609000000200000ULL,
- 0x8008330000d00000ULL, 0x8060100040404010ULL, 0xeaa00ea0ea00808cULL, 0x200c8020a0000020ULL,
- 0x0408800020200000ULL, 0x0189001403200000ULL, 0xc00800000000c000ULL, 0x200430c00c300000ULL,
- 0x0100300100004000ULL, 0x0000040000000000ULL, 0x2420000400001000ULL, 0x89a1200400000000ULL,
- 0x20c8a000208c0000ULL, 0x8080000000000000ULL, 0x28a0108020210080ULL, 0xa2a84800a0880988ULL,
- 0x258008000400c000ULL, 0x0140000000100000ULL, 0xa028a222a0aa0228ULL, 0xc060012054044040ULL,
- 0x0010010400000000ULL, 0x00000050150c0114ULL, 0x0000008010c20010ULL, 0xaa088000a0200880ULL,
- 0x0000000000000000ULL, 0x0700b0c0000c0000ULL, 0x2200040000080030ULL, 0x2aa8808040240800ULL,
- 0x08b0500000000100ULL, 0x1000830400200000ULL, 0x4204000010000000ULL, 0x40c2200050040050ULL,
- 0x0104404001010000ULL, 0x1a808c8103c00030ULL, 0x30900010c0000b00ULL, 0x200812b283000008ULL,
- 0x000c000020e00000ULL, 0x2140000000400000ULL, 0x0288000080200000ULL, 0x8060a200c8a20280ULL,
- 0x0400114010215000ULL, 0x0000000000000000ULL, 0x082b200002000010ULL, 0x22a0030000031000ULL,
- 0x008100001000000cULL, 0x05400c00c0230400ULL, 0xca3000003c080100ULL, 0x0000000020000004ULL,
- 0x0000000100000000ULL, 0x8004320813f5c000ULL, 0xa280080200000800ULL, 0xc22000044e334c20ULL,
- 0x000004146e361024ULL, 0x800126806aa0d584ULL, 0xb000a0040023c41cULL, 0x0a083000803053d8ULL,
- 0x0000000000000000ULL, 0x0000100000020000ULL, 0x0000000010000010ULL, 0x0000000045040004ULL,
- 0x0000000000100000ULL, 0x0000020400000010ULL, 0x0003015000000000ULL, 0x0400000000000000ULL,
- 0x0000000400000000ULL, 0x0100000000000800ULL, 0x0000001000000000ULL, 0x0000000000000000ULL,
- 0x0000000040000000ULL, 0x0000000000000000ULL, 0x0004001000000000ULL, 0x0008001000000000ULL,
- 0x0010000000000004ULL, 0x0000010100001000ULL, 0x0004000000000004ULL, 0x0000014040050014ULL,
- 0x0014000000000040ULL, 0x5540000000041000ULL, 0x0000000000000000ULL, 0x0000040000000d00ULL,
- 0x0000000000000000ULL, 0x0000000000100000ULL, 0x0001000000000000ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x0000000000000000ULL, 0x4500000000040400ULL, 0x0000800000000400ULL,
- 0x0000000000000000ULL, 0x13e080000020000cULL, 0xcf00001005100000ULL, 0x04a8008000200300ULL,
- 0x00280100100000c0ULL, 0x1c8c000040200000ULL, 0x0600005000100000ULL, 0x050800000c104000ULL,
- 0x4c10101000110000ULL, 0x0c00000000300000ULL, 0x22040c00100000c0ULL, 0x0800700010100000ULL,
- 0x0000000000001000ULL, 0x0a08000010000040ULL, 0x0800034004210010ULL, 0x04e0000400000000ULL,
- 0x0800030020000000ULL, 0x0000005000000000ULL, 0x0400110101304110ULL, 0x0428000010a01000ULL,
- 0x060b000000800010ULL, 0x35810c00c020c000ULL, 0x00800c4321800000ULL, 0x4208088020000080ULL,
- 0x040000111003ff00ULL, 0x0020900020202080ULL, 0x22888180a8000888ULL, 0x0225200542005420ULL,
- 0x2020040400340020ULL, 0x10300424500cc444ULL, 0x3081a00400e00200ULL, 0x33001300c0300000ULL,
- 0x0000000000000000ULL, 0x04003c0000000000ULL, 0x0a04001000100100ULL, 0x1408000001000000ULL,
- 0x1800000044100000ULL, 0x3400040400000300ULL, 0x5000040801000040ULL, 0x4088401040000040ULL,
- 0x1010110130100000ULL, 0xca800c3000300000ULL, 0x5a01000000080100ULL, 0x020280000cd01300ULL,
- 0x0302000410200010ULL, 0x0000102000300000ULL, 0x0b09000000000000ULL, 0x20008004c4800004ULL,
- 0x28c0410010000000ULL, 0x0004015041000050ULL, 0x0a01006000200200ULL, 0x0020d00000100040ULL,
- 0x0010a00100900000ULL, 0x3500bf00c0030300ULL, 0x080c010000200d00ULL, 0x2248000004020010ULL,
- 0x0000c00000000000ULL, 0x8044b00200e08000ULL, 0xaaa82aa2aa8a2aa8ULL, 0x0220002241c08604ULL,
- 0x4200260440328444ULL, 0x68001226103008b4ULL, 0x3a0080c0b0000400ULL, 0x2a804804803c4008ULL,
- 0x0000000000000000ULL, 0x04008c0300000400ULL, 0x008000c0000c0000ULL, 0x088001000000001cULL,
- 0x0840000001000010ULL, 0x0400000000200c00ULL, 0x4244000101040000ULL, 0x4238007011100000ULL,
- 0x1000d00100000010ULL, 0x1d00800400300000ULL, 0x4204080c00000000ULL, 0x2a88080080000008ULL,
- 0x08001c0200001000ULL, 0x0a00000400000000ULL, 0x8a88003080080000ULL, 0x0521800400300000ULL,
- 0x3200051000201000ULL, 0x0000000000000000ULL, 0x0020801404000000ULL, 0x322010401c0c101cULL,
- 0x0c01100013000000ULL, 0x04003000c0204000ULL, 0x088c0020a0cc0000ULL, 0x2200000080000018ULL,
- 0x0404000044000000ULL, 0x82a0b000008820b0ULL, 0x0000040020440000ULL, 0xc2650004403f1420ULL,
- 0x0021340241b64464ULL, 0x8020040242c2d474ULL, 0x32018c0480288000ULL, 0x00800b0080300000ULL,
- 0x0000000000000000ULL, 0x05008c0000040130ULL, 0xc0d8000000800000ULL, 0x0020000020200200ULL,
- 0x23a2000120204000ULL, 0x5052100550104150ULL, 0x1000101100040000ULL, 0xc40001c301000000ULL,
- 0x8288000000c00000ULL, 0x5150040144d01404ULL, 0xea8c0ea028ae088cULL, 0xc31010c000000c80ULL,
- 0x0002000060000000ULL, 0xc80800f030000000ULL, 0x0000000400300000ULL, 0xc00080c00ff0c344ULL,
- 0x00080001200c0000ULL, 0x0000050080000000ULL, 0x0328000300300000ULL, 0x082030000cc01040ULL,
- 0xeb08800100004000ULL, 0x8030003300c80f00ULL, 0xfb0d0000e4ac0000ULL, 0x0020006080000008ULL,
- 0x0500100100040000ULL, 0x1140000000000000ULL, 0xcb883330a0e00000ULL, 0xc000010050000080ULL,
- 0x0010104005b54150ULL, 0x40111d5155001554ULL, 0x80000070140f0004ULL, 0x0b0830c3a0003380ULL,
- 0x0000000000000000ULL, 0x04c13000000f830cULL, 0x2808000000000000ULL, 0x2810000000000800ULL,
- 0x08c0080004400000ULL, 0x04c0240300801c20ULL, 0x4040000080000004ULL, 0x0000400100100010ULL,
- 0x020001008000c0c0ULL, 0x1d008c000c3c0000ULL, 0x0080003000000800ULL, 0x2288080080000008ULL,
- 0x0a84004020220000ULL, 0x0800080000100000ULL, 0xaa80004080400008ULL, 0x8024000400c01660ULL,
- 0x80841c2001000104ULL, 0x0001000000000000ULL, 0x0020028020020280ULL, 0x0860404011900100ULL,
- 0xec80080200000000ULL, 0x010103c100200400ULL, 0x0200004000000000ULL, 0x0000000000400400ULL,
- 0x000010000003fcfcULL, 0x8040083238c20000ULL, 0x08800220a0920a00ULL, 0x08210004483c0c24ULL,
- 0xc020240740b0a200ULL, 0x802006014a201494ULL, 0x3201233070ac0e00ULL, 0x08002806033a48a0ULL,
- 0x0000000000000000ULL, 0x8020820028a00680ULL, 0x2000002000000104ULL, 0x22a80801100a0808ULL,
- 0xa2a8002080000000ULL, 0xa000800008a08000ULL, 0x0000100000400000ULL, 0x8000002100000000ULL,
- 0x0000010000004404ULL, 0xa2a0088080000888ULL, 0x0000000010400800ULL, 0xa280082080080008ULL,
- 0x2280000080010008ULL, 0x2000000000000000ULL, 0x228800008c080808ULL, 0x8021828002a98200ULL,
- 0xa200002000080000ULL, 0x0000040000000000ULL, 0x22a0000080000000ULL, 0x202882c200800080ULL,
- 0xa000000001004000ULL, 0x000000c808a00600ULL, 0x0000000010000000ULL, 0x000001000000040cULL,
- 0x0000000000000000ULL, 0x802002a2a8aa82a0ULL, 0x20000024a8088228ULL, 0x8020820001000000ULL,
- 0x8020000000808280ULL, 0x8000000000000000ULL, 0x0020800000200280ULL, 0x2080082280a00888ULL,
- 0x0000000000000000ULL, 0x0000015000000040ULL, 0x0000040000040000ULL, 0x0100010010001000ULL,
- 0x0000003210008000ULL, 0x0000000404000000ULL, 0x0000000000000400ULL, 0x0200000000000000ULL,
- 0x0000000000000100ULL, 0x5180014400004050ULL, 0x1000000014000000ULL, 0x4200000000000000ULL,
- 0x0040200000000000ULL, 0x0201004000000000ULL, 0x0a00000000000010ULL, 0x0040200000800000ULL,
- 0x0040051000000500ULL, 0x0000000100800400ULL, 0x6000000000000000ULL, 0x0000000000000000ULL,
- 0x280000c1400040ccULL, 0x4180001000000000ULL, 0x00000000c1000104ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0080000000c00000ULL, 0x0004006066004000ULL,
- 0x0000005000040440ULL, 0x0000106005804044ULL, 0x0000a10511004440ULL, 0x0000000000000110ULL,
- 0x0000000000000000ULL, 0x0000000000080000ULL, 0xeb0808a020800080ULL, 0x29a80081002a1800ULL,
- 0x0b2c000202100100ULL, 0x0001000000888000ULL, 0x2280102010000000ULL, 0x020000602a004110ULL,
- 0x8a800160a6108100ULL, 0x0280000000000020ULL, 0x8a8000a0a8808208ULL, 0x0280882080500308ULL,
- 0x0b18010020804100ULL, 0xeb080000c0080080ULL, 0x2b08000000810130ULL, 0x0000000008040020ULL,
- 0xaa0a08e082894140ULL, 0x0000000000000000ULL, 0x202081409010001cULL, 0x8aa8805082806000ULL,
- 0xeb082900289c0000ULL, 0x0000000000008000ULL, 0xf80c2e20002e0000ULL, 0xa288080420880888ULL,
- 0x0000010000000000ULL, 0x0000000000102000ULL, 0x22880000a8a80808ULL, 0x022022a22aa880a0ULL,
- 0x0000222222aa0620ULL, 0x0000022002800000ULL, 0x208080004028a000ULL, 0x2b888800801c0828ULL,
- 0x0000000000000000ULL, 0x22e0828280a08028ULL, 0xaa88002082080308ULL, 0x0ea80080410a0040ULL,
- 0x2a28222000a00000ULL, 0x8aa2808028a0a2a0ULL, 0x0200001000000000ULL, 0x82080000a0000000ULL,
- 0x8800000082000808ULL, 0x2a008a0000300888ULL, 0x0a80080080080808ULL, 0xaa882800840b0808ULL,
- 0x0a80000080000040ULL, 0xea080820a0000000ULL, 0xaa88080080080808ULL, 0x8040a2800a8024a0ULL,
- 0xaa800020a0080808ULL, 0x0000040000000000ULL, 0x2a280a0080080880ULL, 0x2a20081080008a00ULL,
- 0x2a88882088aa0008ULL, 0x81800202c0a01480ULL, 0xea88082082200000ULL, 0xaa88002080080008ULL,
- 0x0000100000000000ULL, 0x802082a22aa0a2a0ULL, 0x2e80000000000000ULL, 0x0220a2a26aa0a2a8ULL,
- 0x800022a2228a22a0ULL, 0x880002212e82c0b0ULL, 0x02a0aa0002a82228ULL, 0x2d808b0080380008ULL,
- 0x0000000000000000ULL, 0x000407551c154244ULL, 0x2a00208088a02228ULL, 0x12a82182a2402a88ULL,
- 0xe32821e020826d00ULL, 0x801130100ccc1330ULL, 0x028010c000841008ULL, 0x88a08002a0a664a0ULL,
- 0x0048270080000100ULL, 0x00001f010cd10f30ULL, 0xe2242ce22aaea2a0ULL, 0xc2c00cc20ae22460ULL,
- 0xe208003128021c10ULL, 0x2a2021c010821080ULL, 0x2a88202082202020ULL, 0x4010111104941410ULL,
- 0xc80c02c182b00080ULL, 0x0000040000000000ULL, 0xe28030068002c300ULL, 0x2aa02024a2a22228ULL,
- 0xe20889328aa22080ULL, 0x0000000000210100ULL, 0xaa0028e0a9b221a0ULL, 0x2000008080400000ULL,
- 0x0000010041150404ULL, 0x0000105114410100ULL, 0xeaa82aa6aaaaaaa8ULL, 0x000000f44300c434ULL,
- 0x0000222222b00020ULL, 0x0000002000000000ULL, 0x0000004014000000ULL, 0x0039b3f73fbcd3fcULL,
- 0x0000000000000000ULL, 0x0000104015045040ULL, 0x20a80490a08800a0ULL, 0x40a8258410a909a0ULL,
- 0xe0a8a2022aa2e2a0ULL, 0xc111010014000500ULL, 0x2080044041840004ULL, 0x28a8200220a2aba0ULL,
- 0x008400a0a2840800ULL, 0x0101015451009464ULL, 0x20000ea0e02c2c2cULL, 0xe2a828a2aca2aaa8ULL,
- 0x682020a228a222a0ULL, 0xe8882ae22aa2a2a0ULL, 0xe9a80e6022a24140ULL, 0x0011055005001040ULL,
- 0x2aa8208229a0aaa4ULL, 0x0000040000000000ULL, 0x28a0228026a62260ULL, 0xe2a020a422a2a020ULL,
- 0xe808a0022aa1a220ULL, 0x0000010014000100ULL, 0x28ac22802aa2a020ULL, 0x0020000000000000ULL,
- 0x0100010100040000ULL, 0x0000000000000000ULL, 0x22a822a22a8aaaa0ULL, 0x0000000000000000ULL,
- 0x0000102410800100ULL, 0x0000000000000000ULL, 0x0000000002000000ULL, 0x00000fb2a08c0aa8ULL,
- 0x0000000000000000ULL, 0x4010005015440140ULL, 0x18c81c00b180001cULL, 0x2800048021820800ULL,
- 0x8ab820c06a802580ULL, 0x00100170f4040000ULL, 0x4000144041041404ULL, 0x0ac800d0002e440cULL,
- 0x20880820a2000808ULL, 0x400000f03f300c00ULL, 0xaa000ea22aa22aa0ULL, 0xa2880ac0a8942a20ULL,
- 0xaa880a81a1804188ULL, 0xeea022a0aaa02080ULL, 0xaaa820a2aaa66120ULL, 0x0000005115800150ULL,
- 0x2a880920a0840040ULL, 0x0000040000000000ULL, 0xaea82222aaa22a28ULL, 0x8a28041260055150ULL,
- 0xa28824008aa28880ULL, 0x0000025014019000ULL, 0xea882ae02aa200a0ULL, 0x0000000000000000ULL,
- 0x0000000040000400ULL, 0x0000000000000000ULL, 0xaaa82aa22aaaaaa0ULL, 0x0000000000000000ULL,
- 0x0000000000000000ULL, 0x002003003c80c000ULL, 0x0000020014000000ULL, 0x00200010a0980a20ULL,
- 0x0000000000000000ULL, 0x0020001200801240ULL, 0x0a88000089800020ULL, 0xcaa00080a1000000ULL,
- 0x0a200c0020a04080ULL, 0x4002034003840880ULL, 0x4690500190000050ULL, 0x2228004000601000ULL,
- 0x0a803f00803f400cULL, 0x400033e24dd0cf34ULL, 0xaa80a2a229a220a0ULL, 0x0a224000002c0000ULL,
- 0x028000202000008cULL, 0x0a08000070000030ULL, 0x00800c040020000cULL, 0x0000000002850000ULL,
- 0x02881cc310200000ULL, 0x0000040004000000ULL, 0xcba8000400000080ULL, 0xcaa02c0680000000ULL,
- 0xcc880002008c4080ULL, 0x300000f007f0cf0cULL, 0x0a80001080a00000ULL, 0x820880802a880a80ULL,
- 0x0000050001040004ULL, 0x0000011000000000ULL, 0x0a8020a2a0202000ULL, 0x0000022202008000ULL,
- 0x0000222212808000ULL, 0x0020226010000000ULL, 0x000033f33ff3c33cULL, 0x00288002a08c02a8ULL,
- 0x0000000000000000ULL, 0x04408e0000008200ULL, 0x0808004000900000ULL, 0x0aa8200010ca00c0ULL,
- 0x0ba80101005d4010ULL, 0x00018604802c8288ULL, 0x00049400101c0000ULL, 0x000c101110505010ULL,
- 0x0000000000100000ULL, 0x30000c00c022000cULL, 0xd0c00dd0d51d431cULL, 0x0008000010100000ULL,
- 0x000c1001a0280000ULL, 0x0bc80000c0000000ULL, 0x0a00000080280000ULL, 0x8000a00220308420ULL,
- 0x0808000010301000ULL, 0x0000040000000000ULL, 0x0d00031480100000ULL, 0x07200000108c0300ULL,
- 0x0bc0a0c000004000ULL, 0x8000b002c0208480ULL, 0x340c0100118c111cULL, 0x8008008020890000ULL,
- 0x0000000000040010ULL, 0x0020b00320c1d0b0ULL, 0x00002000000c0000ULL, 0x0020be226e2008a0ULL,
- 0x002010c03fb0a6a0ULL, 0x00202e222aaec284ULL, 0x00008f0000208400ULL, 0x0000000000300000ULL,
- };
- // Latin1 6%, Latin2 11%, Latin7 3%
-
-
-
- // Just for debugging. not thread-safe
- static char tri_string[4];
- char* Latin127Str(int trisub) {
- tri_string[0] = "_abcdefghijklmnopqrstuvwxyzAEIOC"[(trisub >> 10) & 0x1f];
- tri_string[1] = "_abcdefghijklmnopqrstuvwxyzAEIOC"[(trisub >> 5) & 0x1f];
- tri_string[2] = "_abcdefghijklmnopqrstuvwxyzAEIOC"[(trisub >> 0) & 0x1f];
- tri_string[3] = '\0';
- return tri_string;
- }
-
- // Returns two bits per three-byte trigram, indicating
- // dont-care, Latin1 likely, Latin2 likely, and Latin7 (ISO-8859-13) likely
- int TrigramValue(const uint8* trisrc) {
- int byte0_p = kMapToFiveBits[trisrc[0]];
- int byte1_p = kMapToFiveBits[trisrc[1]];
- int byte2_p = kMapToFiveBits[trisrc[2]];
- int subscr = ((byte0_p) << 5) | byte1_p;
- int temp = static_cast<int>((kLatin127Trigrams[subscr] >> (byte2_p * 2)));
- //printf("%s=%d ", Latin127Str((subscr << 5) | byte2_p), temp & 3);
- return temp & 3;
- }
-
-
- // Put out trigrams for surrounding 32 bytes for Latin encodings
- // Return true if more Latin2 & 7 than Latin1
- bool BoostLatin127Trigrams(int tri_block_offset,
- DetectEncodingState* destatep) {
- //printf("BoostLatin127Trigrams[%06x]\n", tri_block_offset);
- int excess_latin27 = 0;
- int srclen = destatep->limit_src - destatep->initial_src;
- int hi_limit = minint(tri_block_offset + 32, srclen - 2);
- const uint8* trisrc = &destatep->initial_src[tri_block_offset];
- const uint8* trisrclimit = &destatep->initial_src[hi_limit];
- while (trisrc < trisrclimit) {
- // Selectively boost Latin1, Latin2, or Latin7 and friends
- int trigram_val = TrigramValue(trisrc);
- if (trigram_val != 0) {
- if (FLAGS_enc_detect_source) {
- PsHighlight(trisrc, destatep->initial_src, trigram_val, 1);
- }
- if (trigram_val == kTriLatin1Likely) {
- Boost(destatep, F_Latin1, kTrigramBoost);
- Boost(destatep, F_CP1252, kTrigramBoost);
- // We don't want to upset the relative rank of a declared 8859-15
- Boost(destatep, F_ISO_8859_15, kTrigramBoost);
- --excess_latin27;
- } else if (trigram_val == kTriLatin2Likely) {
- Boost(destatep, F_Latin2, kTrigramBoost);
- Boost(destatep, F_CP1250, kTrigramBoost);
- ++excess_latin27;
- } else if (trigram_val == kTriLatin7Likely) {
- Boost(destatep, F_ISO_8859_13, kTrigramBoost);
- Boost(destatep, F_CP1257, kTrigramBoost);
- // We don't want to upset the relative rank of a declared 8859-4 or -6
- // for Estonian
- Boost(destatep, F_Latin4, kTrigramBoost);
- Boost(destatep, F_Latin6, kTrigramBoost);
- ++excess_latin27;
- }
- }
-
- ++trisrc;
- }
- //printf("\n");
-
- return (0 < excess_latin27);
- }
-
-
-
- // Boost any encodings that need extra detection help, then prune
- // src is first unscanned byte
- // slowend means extra pruning when dropping out of initial slow scan
- // final means last call -- no bigram at src
- void BoostPrune(const uint8* src, DetectEncodingState* destatep,
- int prunereason) {
- int delta_asciipairs = destatep->next_interesting_pair[AsciiPair] -
- destatep->prior_interesting_pair[AsciiPair];
- int delta_otherpairs = destatep->next_interesting_pair[OtherPair] -
- destatep->prior_interesting_pair[OtherPair];
-
- if (prunereason == PRUNE_FINAL) {
- // We are about done
- // If we get here with very little accumulated data, the initial hints
- // were too strong, so we derate them to n+1 / 12 for n bigrams
- if (!destatep->hints_derated &&
- (destatep->next_interesting_pair[OtherPair] < kDerateHintsBelow)) {
- int n = destatep->next_interesting_pair[OtherPair];
-
- // Map N pairs to (N+1)/12 portions of the initial hints, etc.
- // Floor of 3/12 -- 1/12 and 2/12 are too easy to overcome
- int m = maxint(3, (n + 1));
- for (int i = 0; i < NUM_RANKEDENCODING; ++i) {
- int original_delta = destatep->hint_prob[i];
- int scaled_delta = (original_delta * m) / kDerateHintsBelow;
- destatep->enc_prob[i] -= original_delta;
- destatep->enc_prob[i] += scaled_delta;
- }
- destatep->hints_derated = true;
- if (destatep->debug_data != NULL) {
- // Show derated-hint result
- char buff[32];
- snprintf(buff, sizeof(buff), "Hints %d/%d", m, kDerateHintsBelow);
- SetDetailsEncLabel(destatep, buff);
- }
- }
- }
-
-
- ++destatep->prune_count;
-
- if (prunereason != PRUNE_FINAL) {
- // Early outs
- if (destatep->rankedencoding_list_len <= 1) { // nothing to prune
- destatep->done = true;
- return;
- }
-
- if ((destatep->prune_count > 0) &&
- (delta_asciipairs + delta_otherpairs) == 0) {
- // Nothing to do; must have just been called earlier
- return;
- }
- }
-
-
-
- // INCREMENT
- // ====================
- // Accumulate OtherPair probibilities over all active families
- // AsciiPair probibilities are all done in ActiveSpecialBoostWhack
- uint8 prior_bad_byte1 = ' '; // won't match first bad pair
- uint8 prior_bad_byte2 = ' '; // won't match first bad pair
- uint8 or_byte1 = 0; // Track if any current pair has a high bit
- int counted_otherpairs = 0;
- uint8 prior_byte1x2x = 0;
- for (int i = 0; i < delta_otherpairs; ++i) {
- int watch1_incr = 0;
- int watch2_incr = 0;
- int next_pair = destatep->prior_interesting_pair[OtherPair] + i;
-
- uint8 byte1 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 0];
- uint8 byte2 = destatep->interesting_pairs[OtherPair][next_pair * 2 + 1];
- uint8 byte1x2x = (byte1 & 0xf0) | ((byte2 >> 4) & 0x0f);
- int weightshift = destatep->interesting_weightshift[OtherPair][next_pair];
-
- int offset_byte12 = destatep->interesting_offsets[OtherPair][next_pair];
-
- // To help distinguish some Cyrillic, Arabic, Greek, Hebrew, Thai
- // Remember if this is a CDEF pair immediately following the previous pair
- // 8xxx CxCx or CxCx 8xxx
- bool next_pair_consec_hi = false;
- if (ConsecutivePair(destatep, next_pair)) {
- if ((byte1x2x & 0xcc) == 0xcc) { // 8xxx CxCx
- next_pair_consec_hi = true;
- } else if ((prior_byte1x2x & 0xcc) == 0xcc) { // CxCx 8xxx
- next_pair_consec_hi = true;
- }
- }
- //printf("prior/cur/consec %02x %02x %d\n",
- // prior_byte1x2x, byte1x2x, next_pair_consec_hi);
- prior_byte1x2x = byte1x2x;
-
- or_byte1 |= byte1;
- uint8 byte1f = byte1;
- // Flip top bit of subscript to better separate quadrant 4 (esp. for Hebrew)
- byte1f ^= (byte2 & 0x80);
-
- // If the same bigram occurred recently, don't increment again
- bool pair_used = false;
- if (!RepeatedBigram(destatep, byte1, byte2)) {
- ++counted_otherpairs;
- pair_used = true;
- // Boost both charset= declared encodings, so
- // Nearly-same probability nearby encoding doesn't drift to the top
- if (!FLAGS_demo_nodefault) {
- destatep->enc_prob[destatep->declared_enc_1] += kDeclaredEncBoost >> weightshift;
- destatep->enc_prob[destatep->declared_enc_2] += kDeclaredEncBoost >> weightshift;
- }
- bool was_bad_pair = false;
- for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
- int incr_shift = 0;
- int rankedencoding = destatep->rankedencoding_list[j];
- Encoding enc = kMapToEncoding[rankedencoding];
-
- // For binary, Skip over repeated marker bytes, such as 02, FF, etc.
- if ((rankedencoding == F_BINARY) &&
- RepeatedBinary(destatep, byte1, byte2)) {
- incr_shift = 2; // count 1/4 as much if repeated
- }
-
- // If byte 1x2x for this encoding is exactly zero, illegal byte pair
- // Don't increment, but instead penalize
- const UnigramEntry* ue = &unigram_table[rankedencoding];
- if (ue->b12[byte1x2x] == 0) {
- // Don't whack consecutive duplicate bad pairs -- overkill
- if ((byte1 != prior_bad_byte1) || (byte2 != prior_bad_byte2)) {
- // Extra whack for illegal pair in this encoding
- Whack(destatep, rankedencoding, kBadPairWhack >> weightshift);
- was_bad_pair = true;
- }
- } else {
- // OK to do the real increment
- int incr = ue->b1[byte1f] + ue->b2[byte2] + ue->b12[byte1x2x];
- if ((ue->b12[byte1x2x] & 0x01) != 0) {
- // Use a more-precise table
- int byte32x32 = ((byte1 & 0x1f) << 5) | (byte2 & 0x1f);
- int hiressub = (byte2 & 0x60) >> 5; // select w/bits 5&6 of byte 2
- DCHECK(ue->hires[hiressub] != NULL);
- incr += ue->hires[hiressub][byte32x32];
- } else {
- // Default final offset
- incr += ue->so;
- }
- incr >>= incr_shift;
-
- incr >>= weightshift;
- destatep->enc_prob[rankedencoding] += incr; // The actual increment
-
- if (FLAGS_enc_detect_detail2) {
- if (watch1_rankedenc == rankedencoding) {watch1_incr = incr;}
- if (watch2_rankedenc == rankedencoding) {watch2_incr = incr;}
- }
- }
-
-
- // If consecutive pair of high bytes, give slight boost to one-byte
- // encodings that have a full alphabet in the high bytes
- if (next_pair_consec_hi && HighAlphaEncoding(enc)) {
- Boost(destatep, rankedencoding, kDeclaredEncBoost >> weightshift);
- }
- } // End for j < rankedencoding_list_len
-
- if (was_bad_pair) {
- prior_bad_byte1 = byte1;
- prior_bad_byte2 = byte2;
- }
-
- // Fold in per-bigram most likely encoding for first N bigrams
- if (next_pair < kBestPairsCount) {
- int best_enc = kMostLikelyEncoding[(byte1 << 8) + byte2];
- Boost(destatep, best_enc, kBestEncBoost >> weightshift);
- }
-
- // Possibly score 32 trigrams around a bigram to better separate
- // Latin1 from Latin2 and Latin7. Especially helpful for detecting
- // mis-labelled Hungarian latin2.
- // If looking and at bigram 0,8,16,... do full scoring, else just 1 tri
- if (destatep->do_latin_trigrams ||
- destatep->looking_for_latin_trigrams) {
- // If just looking, do full scan every 8 times
- // Just look up one trigram the other 7 and do full scan if Latin2,7
- bool scan32 = false;
- const uint8* trisrc = &destatep->initial_src[offset_byte12 - 1];
- if (!destatep->do_latin_trigrams) {
- if ((i & 7) == 0 || trisrc + 3 > destatep->limit_src) {
- scan32 = true;
- } else {
- scan32 = (kTriLatin1Likely < TrigramValue(trisrc));
- }
- }
- if (destatep->do_latin_trigrams || scan32) {
- // Just score each block of 32 bytes once
- int tri_block_offset = offset_byte12 & ~0x1f;
- if (destatep->trigram_highwater_mark <= tri_block_offset) {
- bool turnon = BoostLatin127Trigrams(tri_block_offset, destatep);
- if (FLAGS_counts && !destatep->do_latin_trigrams && turnon) {
- ++doing_used; // First time
- }
- if (FLAGS_enc_detect_source) {
- if (!destatep->do_latin_trigrams && turnon) {
- // First time
- PsHighlight(trisrc, destatep->initial_src, 0, 2);
- }
- }
- destatep->do_latin_trigrams |= turnon;
- destatep->trigram_highwater_mark = tri_block_offset + 32;
- }
- }
- }
-
- } // end if RepeatedBigram()
-
- // Keep track of initial byte high 3 bits
- ++destatep->byte32_count[byte1 >> 5];
-
-
- // TODO: boost subset/superset also
- // Boost(destatep, kRelatedEncoding[best_enc], kBestEncBoost);
-
- if (destatep->debug_data != NULL) {
- // Show detail entry for this bigram
- char buff[16];
- snprintf(buff, sizeof(buff), "%c%02x%02x%c%c",
- pair_used ? ' ' : '[',
- byte1,
- byte2,
- pair_used ? ' ' : ']',
- (weightshift == 0) ? ' ' : '-');
-
- SetDetailsEncProb(destatep,
- destatep->interesting_offsets[OtherPair][next_pair],
- kMostLikelyEncoding[(byte1 << 8) + byte2],
- buff);
- }
- if (FLAGS_enc_detect_detail2) {
- if ((watch1_incr != 0) || (watch2_incr != 0)) {
- // Show increment detail for this encoding
- char buff[32];
- snprintf(buff, sizeof(buff), "%c%d %c%d",
- (watch1_incr < 0) ? '-' : '+', watch1_incr,
- (watch2_incr < 0) ? '-' : '+', watch2_incr);
- SetDetailsEncLabel(destatep, buff);
- }
- }
- } // End for i
-
-
- // If no high bit on, demote all the two-byte codes
- // WAS BUG. This was inside the loop above and should be outside
- if ((counted_otherpairs > 0) && ((or_byte1 & 0x80) == 0)) {
- // No high bit in this group (just 02xx, etc.). Whack 2-byte codes
- // This keeps SJS from creeping past Latin1 on illegal C0 bytes
- for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
- int rankedencoding = destatep->rankedencoding_list[j];
- Encoding enc = kMapToEncoding[rankedencoding];
- if (TwoByteEncoding(enc)) {
- Whack(destatep, rankedencoding, kGentlePairWhack * counted_otherpairs);
- }
- }
- }
-
-
- // BOOST
- // ====================
- if (AnyActive(destatep)) {
- ActiveSpecialBoostWhack(src, destatep);
- }
-
- // Update for next time
- destatep->prior_src = src;
- destatep->prior_interesting_pair[AsciiPair] =
- destatep->next_interesting_pair[AsciiPair];
- destatep->prior_interesting_pair[OtherPair] =
- destatep->next_interesting_pair[OtherPair];
-
-
- // Do any pre-prune final adjustments
- // ====================
- if (prunereason == PRUNE_FINAL) {
- // If UTF8 not in base state, whack
- if (destatep->next_utf8_ministate != 0) {
- Whack(destatep, F_UTF8, kGentlePairWhack * 2 * 1);
- }
- // If UTF8UTF8 not in base state, whack
- if (destatep->next_utf8utf8_ministate != 0) {
- Whack(destatep, F_UTF8UTF8, kGentlePairWhack * 2 * 1);
- }
-
- // If no valid UTF-8 char ever seen, whack
- if (destatep->utf8_minicount[5] == 0) {
- Whack(destatep, F_UTF8, kBadPairWhack * 8); // No sequence
- Whack(destatep, F_UTF8UTF8, kBadPairWhack * 8); // No sequence
- }
-
- // If no valid UTF8UTF8 char ever seen, whack
- if (destatep->utf8utf8_minicount[5] == 0) {
- Whack(destatep, F_UTF8UTF8, kBadPairWhack * 8); // No sequence
- }
-
- // If not all four binary quadrants, whack BINARY;
- // worth 2 pair if 3 quads, 4 pair if 1 or 2 quads
- if (destatep->binary_quadrants_count < 4) {
- if (destatep->binary_quadrants_count == 3) {
- Whack(destatep, F_BINARY, kBadPairWhack * 2);
- } else {
- Whack(destatep, F_BINARY, kBadPairWhack * 4);
- }
- }
-
- // If 1st pair is 1b24, choose between ISO-2022-xx
- // <esc> $ ) C ISO-2022-KR [1b 24 29 43]
- // <esc> $ ) A ISO-2022-CN [1b 24 29 41]
- // <esc> $ ) G ISO-2022-CN [1b 24 29 47]
- // <esc> $ * H ISO-2022-CN [1b 24 2a 48]
- // <esc> ( B ISO-2022-JP [1b 28 42] to ASCII
- // <esc> ( J ISO-2022-JP [1b 28 4a] to X0201
- // <esc> $ @ ISO-2022-JP [1b 24 40] to X0208-78 twobyte
- // <esc> $ B ISO-2022-JP [1b 24 42] to X0208-83 twobyte
- if ((destatep->next_interesting_pair[OtherPair] >= 1) &&
- Iso2022Active(destatep)) {
- if ((destatep->interesting_pairs[OtherPair][0] == 0x1b) &&
- (destatep->interesting_pairs[OtherPair][1] == 0x24)) {
- int offset = destatep->interesting_offsets[OtherPair][0];
- const uint8* esc_src = destatep->initial_src + offset;
- if ((destatep->initial_src + offset) < (destatep->limit_src - 3)) {
- if ((esc_src[2] == ')') && (esc_src[3] == 'C')) {
- Boost(destatep, F_ISO_2022_KR, kBoostOnePair);
- Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
- Whack(destatep, F_JIS, kBadPairWhack);
- } else if ((esc_src[2] == ')') && ((esc_src[3] == 'A') ||
- (esc_src[3] == 'G'))) {
- Boost(destatep, F_ISO_2022_CN, kBoostOnePair);
- Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
- Whack(destatep, F_JIS, kBadPairWhack);
- } else if ((esc_src[2] == '@') || (esc_src[2] == 'B')) {
- Boost(destatep, F_JIS, kBoostOnePair);
- Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
- Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
- }
- } else {
- // Incomplete escape sequence. Whack them all
- Whack(destatep, F_JIS, kBadPairWhack);
- Whack(destatep, F_ISO_2022_CN, kBadPairWhack);
- Whack(destatep, F_ISO_2022_KR, kBadPairWhack);
- }
- }
- }
- if (destatep->debug_data != NULL) {
- SetDetailsEncLabel(destatep, "pre-final");
- }
- }
-
- // PRUNE
- // ====================
- // Find current top two rankedencoding probabilities
- ReRank(destatep);
-
- if (prunereason == PRUNE_SLOWEND) {
- if (destatep->debug_data != NULL) {
- SetDetailsEncLabel(destatep, "slow-end");
- }
- }
-
- // Keep every rankedencoding with probablity >= top_prob - prune_difference
- int prune_diff = destatep->prune_difference;
- // If the top encoding is BINARY, it might be overstated, and we might
- // therefore prune away the real encoding. Make the pruning delta
- // twice as big.
- if (destatep->top_rankedencoding == F_BINARY) {
- prune_diff *= 2;
- }
- int keep_prob = destatep->top_prob - prune_diff;
-
- // Tighten pruning difference (we start wide) for next time
- if (destatep->prune_difference > kFinalPruneDifference) {
- int decrement = kPruneDiffDecrement;
- // If only ASCII pairs, small tighten; if some non-ASCII, full tighten
- if (counted_otherpairs == 0) {
- decrement >>= 1;
- }
- destatep->prune_difference -= decrement;
- }
-
- // Prune the list of active encoding families
- destatep->active_special = 0;
- int k = 0;
- for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
- bool keep = true;
- int rankedencoding = destatep->rankedencoding_list[j];
-
- // If count is too low, ditch it
- if (destatep->enc_prob[rankedencoding] < keep_prob) {
- keep = false;
- }
-
- // If at end of slow section, ditch any 7-bit with zero evidence so far
- if ((prunereason == PRUNE_SLOWEND) &&
- SevenBitEncoding(kMapToEncoding[rankedencoding]) &&
- (destatep->enc_prob[rankedencoding] <= 0) &&
- (rankedencoding != destatep->top_rankedencoding)) {
- keep = false;
- }
-
- // Keep it. This will always keep at least top_prob rankedencoding
- if (keep) {
- destatep->active_special |= kSpecialMask[kMapToEncoding[rankedencoding]];
- destatep->rankedencoding_list[k++] = rankedencoding;
- }
- }
-
- if (destatep->debug_data != NULL) {
- char buff[32];
- snprintf(buff, sizeof(buff), "%d prune", prune_diff / XLOG2);
- SetDetailsEncLabel(destatep, buff);
- }
- destatep->rankedencoding_list_len = k;
-
-
-
- // Force final result in some cases
- // Do any post-prune final adjustments
- if (prunereason == PRUNE_FINAL) {
- // If no high-byte pairs, result is ASCII7, BINARY, UTF7, 2022, or HZ
- if (destatep->next_interesting_pair[OtherPair] == 0) {
- if ((destatep->top_rankedencoding != F_BINARY) &&
- (destatep->top_rankedencoding != F_UTF7) &&
- (destatep->top_rankedencoding != F_ISO_2022_CN) &&
- (destatep->top_rankedencoding != F_ISO_2022_KR) &&
- (destatep->top_rankedencoding != F_JIS) &&
- (destatep->top_rankedencoding != F_HZ_GB_2312)) {
- destatep->top_rankedencoding = F_ASCII_7_bit;
- Boost(destatep, F_ASCII_7_bit, kBoostOnePair * 2);
- }
- }
-
- // If some 89 pairs, not ISO_8859_x and vice versa
- if (destatep->byte32_count[4] > 0) {
- switch (destatep->top_rankedencoding) {
- case F_ASCII: // ISO-8859-1
- destatep->top_rankedencoding = F_CP1252;
- // Better: destatep->enc_prob[F_ASCII] <==> destatep->enc_prob[F_CP1252]
- Boost(destatep, F_CP1252, kBoostOnePair * 2);
- break;
- case F_Latin2: // ISO-8859-2
- // Don't swap back; not superset
- //destatep->top_rankedencoding = F_CP1250;
- //Boost(destatep, F_CP1250, kBoostOnePair * 2);
- break;
- case F_Arabic: // ISO-8859-6
- destatep->top_rankedencoding = F_CP1256;
- Boost(destatep, F_CP1256, kBoostOnePair * 2);
- break;
- case F_Greek: // ISO-8859-7
- // Don't swap -- not proper superset
- // Capital Alpha tonos at 0xB6 in ISO-8859-7, 0xA2 in CP1253
- //destatep->top_rankedencoding = F_CP1253;
- //Boost(destatep, F_CP1253, kBoostOnePair * 2);
- break;
- case F_Hebrew: // ISO-8859-8
- // Don't swap -- visual vs. logical
- //destatep->top_rankedencoding = F_CP1255;
- //Boost(destatep, F_CP1255, kBoostOnePair * 2);
- break;
- case F_Latin5: // ISO-8859-9
- destatep->top_rankedencoding = F_CP1254;
- Boost(destatep, F_CP1254, kBoostOnePair * 2);
- break;
- case F_ISO_8859_11: // ISO-8859-11
- destatep->top_rankedencoding = F_CP874;
- Boost(destatep, F_CP874, kBoostOnePair * 2);
- break;
- }
- } else {
- switch (destatep->top_rankedencoding) {
- case F_CP1252: // ISO-8859-1
- destatep->top_rankedencoding = F_ASCII;
- Boost(destatep, F_ASCII, kBoostOnePair * 2);
- break;
- case F_CP1250: // ISO-8859-2
- // Don't swap back; not superset
- //destatep->top_rankedencoding = F_Latin2;
- //Boost(destatep, F_Latin2, kBoostOnePair * 2);
- break;
- case F_CP1256: // ISO-8859-6
- // Don't swap back -- not proper superset
- //destatep->top_rankedencoding = F_Arabic;
- //Boost(destatep, F_Arabic, kBoostOnePair * 2);
- break;
- case F_CP1253: // ISO-8859-7
- // Don't swap back -- not proper superset
- //destatep->top_rankedencoding = F_Greek;
- //Boost(destatep, F_Greek, kBoostOnePair * 2);
- break;
- case F_CP1255: // ISO-8859-8
- // Don't swap back -- not proper superset
- //destatep->top_rankedencoding = F_Hebrew;
- //Boost(destatep, F_Hebrew, kBoostOnePair * 2);
- break;
- case F_CP1254: // ISO-8859-9
- destatep->top_rankedencoding = F_Latin5;
- Boost(destatep, F_Latin5, kBoostOnePair * 2);
- break;
- case F_CP874: // ISO-8859-11
- destatep->top_rankedencoding = F_ISO_8859_11;
- Boost(destatep, F_ISO_8859_11, kBoostOnePair * 2);
- break;
- }
- }
-
- if (destatep->debug_data != NULL) {
- char buff[32];
- snprintf(buff, sizeof(buff), "final %d",
- static_cast<int>(src - destatep->initial_src));
- SetDetailsEncLabel(destatep, buff);
-
- // Show winning encoding and its delta log base2 from 2nd-best
- // Divide delta by XLOG2 to get log base 2
- int delta = destatep->top_prob - destatep->second_top_prob;
- if (delta < (2 * XLOG2)) {
- delta /= XDECILOG2;
- snprintf(buff, sizeof(buff), "+%d.%d %s ",
- delta / 10, delta % 10,
- MyEncodingName(kMapToEncoding[destatep->top_rankedencoding]));
- } else if (delta < (50 * XLOG2)) {
- delta /= XLOG2;
- snprintf(buff, sizeof(buff), "+%d %s",
- delta,
- MyEncodingName(kMapToEncoding[destatep->top_rankedencoding]));
- } else {
- snprintf(buff, sizeof(buff), "%s",
- MyEncodingName(kMapToEncoding[destatep->top_rankedencoding]));
- }
- SetDetailsEncProbCopyOffset(destatep, destatep->top_rankedencoding, buff);
- }
- }
-
-
- // FINISH
- // ====================
- // Eventual encoding result is reliable if big difference in top two, or if
- // only Ascii7 ever encountered
- // Also reliable if exactly one OtherPair and it's best encoding matches top
- destatep->reliable = false;
- if (destatep->next_interesting_pair[OtherPair] == 0) {
- // Only 7-bit ASCII
- destatep->reliable = true;
- }
- if ((destatep->top_prob - destatep->second_top_prob) >=
- FLAGS_ced_reliable_difference) {
- destatep->reliable = true;
- }
- if (destatep->next_interesting_pair[OtherPair] == 1) {
- uint8 byte1 = destatep->interesting_pairs[OtherPair][0];
- uint8 byte2 = destatep->interesting_pairs[OtherPair][1];
- int best_enc = kMostLikelyEncoding[(byte1 << 8) + byte2];
- if (best_enc == destatep->top_rankedencoding) {
- destatep->reliable = true;
- }
- }
-
- // If we pruned to one encoding, we are done
- if (destatep->rankedencoding_list_len == 1) {
- destatep->reliable = true;
- destatep->done = true;
- }
-
- // If we pruned to two or three encodings in the same *superset/subset
- // rankedencoding* and enough pairs, we are done. Else keep going
- if (destatep->rankedencoding_list_len == 2) {
- Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
- Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
- if (kMapEncToBaseEncoding[enc0] == kMapEncToBaseEncoding[enc1]) {
- if (destatep->prune_count >= 3) {
- destatep->reliable = true;
- destatep->done = true;
- }
- }
- } else if (destatep->rankedencoding_list_len == 3) {
- Encoding enc0 = kMapToEncoding[destatep->rankedencoding_list[0]];
- Encoding enc1 = kMapToEncoding[destatep->rankedencoding_list[1]];
- Encoding enc2 = kMapToEncoding[destatep->rankedencoding_list[2]];
- Encoding base0 = kMapEncToBaseEncoding[enc0];
- Encoding base1 = kMapEncToBaseEncoding[enc1];
- Encoding base2 = kMapEncToBaseEncoding[enc2];
-
- if ((base0 == base1) && (base0 == base2)) {
- if (destatep->prune_count >= 3) {
- destatep->reliable = true;
- destatep->done = true;
- }
- }
- }
- }
-
-
- // Accumulate aligned byte-pair at src
- // Occasionally, calc boost for some encodings and then prune the active list
- // weightshift is used to give low weight some text, such as inside tags
- // Returns true if pruning occurred
- bool IncrementAndBoostPrune(const uint8* src,
- int remaining_length,
- DetectEncodingState* destatep,
- int weightshift,
- int exit_reason) {
- destatep->last_pair = src;
- // Pick up byte pair, or very last byte plus 0x20
- uint8 byte1 = src[0];
- uint8 byte2 = 0x20;
- if (1 < remaining_length) {byte2 = src[1];}
-
- // whatset=0 for Ascii + ~, 1 for all others; see kTestPrintableAsciiTildePlus
- int whatset = exit_reason - 1;
- int next_pair = destatep->next_interesting_pair[whatset];
-
- if (next_pair > 16) {
- // If not clear by 16 bigrams, stop accumulating + ~ 00
- if (byte1 == '+') {return false;}
- if (byte1 == '~') {return false;}
- if (byte1 == 0x00) {return false;}
- }
-
- // Remember pair in appropriate list
- if (next_pair >= kMaxPairs) {
- // We have filled up our alloted space for interesting pairs with no
- // decision. If ASCII pairs full, just skip until end of slow loop; if
- // non-Ascii pairs full, force done
- if (whatset == OtherPair) {
- destatep->done = true;
- }
- } else {
- int offset = static_cast<int>(src - destatep->initial_src);
- destatep->interesting_pairs[whatset][next_pair * 2 + 0] = byte1;
- destatep->interesting_pairs[whatset][next_pair * 2 + 1] = byte2;
- destatep->interesting_offsets[whatset][next_pair] = offset;
- destatep->interesting_weightshift[whatset][next_pair] = weightshift;
- ++destatep->next_interesting_pair[whatset];
- ++next_pair;
- }
-
- // Prune now and then , but always if forced to be done
- if (destatep->done || ((next_pair & kPruneMask) == 0)) { // Prune every M
- BoostPrune(src + 2, destatep, PRUNE_NORMAL); // src+2 first unscanned byte
- // may be off end of input
- return true;
- }
- return false;
- }
-
- void DumpSummary(DetectEncodingState* destatep, int whatset, int n) {
- printf(" %sSummary[%2d]: ", kWhatSetName[whatset],
- destatep->next_interesting_pair[whatset]);
- int limit = minint(n, destatep->next_interesting_pair[whatset]);
- for (int i = 0; i < limit; ++i) {
- printf("%02x%02x ",
- destatep->interesting_pairs[whatset][i * 2 + 0],
- destatep->interesting_pairs[whatset][i * 2 + 1]);
- if ((i & 7) == 7) {printf(" ");}
- }
- printf("\n");
- }
-
- void BeginDetail(DetectEncodingState* destatep) {
- fprintf(stderr, "%d [", NUM_RANKEDENCODING);
- for (int e = 0; e < NUM_RANKEDENCODING; ++e) {
- fprintf(stderr, "(%s)", MyRankedEncName(e));
- if ((e % 10) == 9) {fprintf(stderr, "\n ");}
- }
- fprintf(stderr, "] size-detail\n");
- destatep->next_detail_entry = 0;
- }
-
- // Single character to represent (printable ASCII) gap between bigrams
- char DetailOffsetChar(int delta) {
- if (delta == 0) {return ' ';}
- if (delta <= 2) {return '=';}
- if (delta <= 15) {return '_';}
- if (delta <= 31) {return '+';}
- {return ' ';}
- }
-
- void DumpDetail(DetectEncodingState* destatep) {
- // Turn all counts into delta from previous entry
- fprintf(stderr, "%d count-detail\n", destatep->next_detail_entry);
- // Rewrite, recording deltas
- for (int z = destatep->next_detail_entry - 1; z > 0; --z) {
- destatep->debug_data[z].offset -= destatep->debug_data[z - 1].offset;
- for (int e = 0; e < NUM_RANKEDENCODING; ++e) {
- destatep->debug_data[z].detail_enc_prob[e] -=
- destatep->debug_data[z - 1].detail_enc_prob[e];
- }
- }
- // Now print
- for (int z = 0; z < destatep->next_detail_entry; ++z) {
- // Highlight some entries ending in '!' with light red underbar
- int len = destatep->debug_data[z].label.size();
- if (destatep->debug_data[z].label[len - 1] == '!') {
- fprintf(stderr, "1 0.9 0.9 do-flag\n");
- }
- fprintf(stderr, "(%c%s) %d [",
- DetailOffsetChar(destatep->debug_data[z].offset),
- destatep->debug_data[z].label.c_str(),
- destatep->debug_data[z].best_enc);
- for (int e = 0; e < NUM_RANKEDENCODING; ++e) {
- fprintf(stderr, "%d ", destatep->debug_data[z].detail_enc_prob[e]);
- if ((e % 10) == 9) {fprintf(stderr, " ");}
- }
- fprintf(stderr, "] do-detail-e\n");
- }
- // Get ready for next time,if any
- destatep->next_detail_entry = 0;
- }
-
- void PsRecurse(const char* buff) {
- fprintf(stderr, "() end-detail (%s) start-detail\n\n", buff);
- }
-
- void DumpReliable(DetectEncodingState* destatep) {
- printf("Not reliable: ");
-
- // Find center of gravity of OtherPair list
- int x_sum = 0;
- int y_sum = 0;
- int count = destatep->next_interesting_pair[OtherPair];
- for (int i = 0; i < count; ++i) {
- uint8 byte1 = destatep->interesting_pairs[OtherPair][i * 2 + 0];
- uint8 byte2 = destatep->interesting_pairs[OtherPair][i * 2 + 1];
- x_sum += byte2;
- y_sum += byte1;
- }
- if (count == 0) {count = 1;} // adoid zdiv
- int x_bar = x_sum / count;
- int y_bar = y_sum / count;
- printf("center %02X,%02X\n", x_bar, y_bar);
-
- double closest_dist = 999.0;
- int closest = 0;
- for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
- int rankedencoding = destatep->rankedencoding_list[j];
- const UnigramEntry* ue = &unigram_table[rankedencoding];
- printf(" %8s = %4d at %02x,%02x +/- %02X,%02X ",
- MyEncodingName(kMapToEncoding[rankedencoding]),
- destatep->enc_prob[rankedencoding],
- ue->x_bar, ue->y_bar,
- ue->x_stddev, ue->y_stddev);
- double x_diff = x_bar - ue->x_bar;
- double y_diff = y_bar - ue->y_bar;
- double dist = sqrt((x_diff * x_diff) + (y_diff * y_diff));
- printf("(%3.1f)\n", dist);
-
- if (closest_dist > dist) {
- closest_dist = dist;
- closest = rankedencoding;
- }
- }
- printf("Closest=%s (%3.1f)\n",
- MyEncodingName(kMapToEncoding[closest]), closest_dist);
-
- for (int i = 0; i < 8; ++i) {
- // Demote by distance to CG and see if that helps, or just quit
- }
- }
-
- // Scan short single lines quickly for all printable ASCII
- // Return true if all bytes are in [20..7F], false otherwise
- bool QuickPrintableAsciiScan(const char* text, int text_length) {
- const uint8* src = reinterpret_cast<const uint8*>(text);
- const uint8* srclimit = src + text_length;
- const uint8* srclimit8 = srclimit - 7;
- while (src < srclimit8) {
- // Exits on any byte outside [0x20..0x7E] range (HT LF CR exit)
- uint8 mask = 0;
- for (int i = 0; i < 8; ++i) mask |= (src[i]-0x20)|(src[i]+0x01);
- if ((mask & 0x80) != 0) break;
- src += 8;
- }
- while (src < srclimit) {
- uint8 uc = *src++;
- if (kIsPrintableAscii[uc] == 0) {return false;}
- }
- return true;
- }
-
- static const int kMaxScanBack = 192;
-
- // Return true if text is inside a tag or JS comment
- bool TextInsideTag(const uint8* isrc, const uint8* src, const uint8* srclimit) {
- const uint8* srcbacklimit = src - kMaxScanBack;
- if (srcbacklimit < isrc) {
- srcbacklimit = isrc;
- }
- const uint8* ss = src - 1;
- while (srcbacklimit <= ss) {
- uint8 c = *ss--;
- if ((c & ~0x02) == '<') {
- // We found preceding < 3C or > 3E nearby
- // Even cheaper: if inside a tag, we don't care what tag; return true
- if (c == '<') {
- return true;
- }
- // See if we are just after <title>...
- if ((c == '>') && (isrc <= (ss - 5)) &&
- (ss[-5] == '<') &&
- ((ss[-4] | 0x20) == 't') &&
- ((ss[-3] | 0x20) == 'i') &&
- ((ss[-2] | 0x20) == 't') &&
- ((ss[-1] | 0x20) == 'l') &&
- ((ss[-0] | 0x20) == 'e')) {
- return true;
- }
- // See if we are just after <SCRIPT language=javascript>...
- if ((c == '>') && (isrc <= (ss - 5)) &&
- (ss[-5] == 's') &&
- ((ss[-4] | 0x20) == 'c') &&
- ((ss[-3] | 0x20) == 'r') &&
- ((ss[-2] | 0x20) == 'i') &&
- ((ss[-1] | 0x20) == 'p') &&
- ((ss[-0] | 0x20) == 't')) {
- return true;
- }
- // Not in a tag
- return false;
- // See if we are just after JavaScript comment /* ...
- } else if (c == '/') {
- if (((ss + 2) < srclimit) && (ss[2] == '*')) {
- // We backscanned to /*
- return true;
- }
- }
- }
-
- return false;
- }
-
- const uint8* SkipToTagEnd(const uint8* src, const uint8* srclimit) {
- const uint8* ss = src + 1;
- while (ss <= srclimit) {
- uint8 c = *ss++;
- if ((c == '<') || (c == '>')) {
- return ss;
- }
- }
- return src + 2; // Always make progress, Otherwise we get an infinite loop
- }
-
-
- // Take a watch string and map to a ranked encoding. If no match, return -1
- int LookupWatchEnc(const string& watch_str) {
- int watchval = -1;
- // Mixed encoding maps to enc=UTF8UTF8
- if (watch_str == "UTF8UTF8") {
- watchval = F_UTF8UTF8;
- } else {
- Encoding enc;
- if (EncodingFromName(watch_str.c_str(), &enc)) {
- watchval = CompactEncDet::BackmapEncodingToRankedEncoding(enc);
- }
- }
- return watchval;
- }
-
- // Return true if enc and enc2 are equal or one is a subset of the other
- // or either is UNKNOWN
- // also UTF8UTF8 is compatible with both Latin1 and UTF8
- bool CompatibleEnc(Encoding enc, Encoding enc2) {
- if (enc < 0) {return false;}
- if (NUM_ENCODINGS <= enc) {return false;}
- if (enc2 < 0) {return false;}
- if (NUM_ENCODINGS <= enc2) {return false;}
- if (enc == enc2) {return true;}
- if (kMapEncToBaseEncoding[enc] == kMapEncToBaseEncoding[enc2]) {return true;}
-
- if (enc == ASCII_7BIT) {return true;}
- if (enc2 == ASCII_7BIT) {return true;}
- if (enc == UNKNOWN_ENCODING) {return true;}
- if (enc2 == UNKNOWN_ENCODING) {return true;}
- if (enc == UTF8UTF8) {
- if (enc2 == UTF8) {return true;}
- if (kMapEncToBaseEncoding[enc2] == ISO_8859_1) {return true;}
- }
- if (enc2 == UTF8UTF8) {
- if (enc == UTF8) {return true;}
- if (kMapEncToBaseEncoding[enc] == ISO_8859_1) {return true;}
- }
-
- return false;
- }
-
- // Return superset of enc and enc2, which must be compatible
- Encoding SupersetEnc(Encoding enc, Encoding enc2) {
- //printf(" SupersetEnc (%s, ", MyEncodingName(enc)); // TEMP
- //printf("%s) ", MyEncodingName(enc2));
- //printf("= %s\n",
- // MyEncodingName(kMapEncToSuperLevel[enc] >= kMapEncToSuperLevel[enc2] ?
- // enc :enc2));
- if (kMapEncToSuperLevel[enc] >= kMapEncToSuperLevel[enc2]) {
- return enc;
- }
- return enc2;
- }
-
-
- // If unreliable, try rescoring to separate some encodings
- Encoding Rescore(Encoding enc, const uint8* isrc,
- const uint8* srctextlimit, DetectEncodingState* destatep) {
- if (FLAGS_counts) {++rescore_used;}
- Encoding new_enc = enc;
-
- bool rescore_change = false;
-
- int count = destatep->next_interesting_pair[OtherPair];
- int text_length = srctextlimit - isrc;
- for (int i = 0; i < count; ++i) {
- int bigram_offset = destatep->interesting_offsets[OtherPair][i];
- uint8 byte0 = (0 < bigram_offset) ?
- isrc[bigram_offset - 1] : 0x20;
- uint8 byte1 = isrc[bigram_offset + 0]; // Known to have high bit on
- uint8 byte2 = ((bigram_offset + 1) < text_length) ?
- isrc[bigram_offset + 1] : 0x20;
- uint8 byte3 = ((bigram_offset + 2) < text_length) ?
- isrc[bigram_offset + 2] : 0x20;
- int high_hash = ((byte0 & 0xc0) >> 0) |
- ((byte1 & 0xc0) >> 1) |
- ((byte2 & 0xc0) >> 4) |
- ((byte3 & 0xc0) >> 6); // 00112233
-
- // Boost HighAccent encodings for Ascii bit patterns
- // 0x1x 0x0x
- // 1010 1010
- // 0010 0000
- //
- if ((high_hash & 0xaa) == 0x20) {
- for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
- int rankedencoding = destatep->rankedencoding_list[j];
- if (HighAccentEncoding(kMapToEncoding[rankedencoding])) {
- // TODO: also want to boost Shift-JIS here if byte1 is Ax..Dx
- // TEMP
- //printf(" Rescore[%02x] %s +%d\n",
- // high_hash, MyRankedEncName(rankedencoding), kGentlePairBoost);
- Boost(destatep, rankedencoding, kGentlePairBoost);
- rescore_change = true;
- }
- }
- }
-
- // Whack HighAccent encodings for high bit patterns
- // 1x1x 1x1x
- // 1010 1010
- // 1010 1010
- //
- if ((high_hash & 0xaa) == 0xaa) {
- for (int j = 0; j < destatep->rankedencoding_list_len; j++) {
- int rankedencoding = destatep->rankedencoding_list[j];
- if (HighAccentEncoding(kMapToEncoding[rankedencoding])) {
- // TEMP
- //printf(" Rescore[%02x] %s -%d\n",
- // high_hash, MyRankedEncName(rankedencoding), kGentlePairBoost);
- Whack(destatep, rankedencoding, kGentlePairBoost);
- rescore_change = true;
- }
- }
- }
-
- }
-
- if (rescore_change) {
- ReRank(destatep);
- new_enc = kMapToEncoding[destatep->top_rankedencoding];
-
- if (destatep->debug_data != NULL) {
- char buff[32];
- snprintf(buff, sizeof(buff), "=Rescore %s", MyEncodingName(new_enc));
- SetDetailsEncProb(destatep,
- 0,
- CompactEncDet::BackmapEncodingToRankedEncoding(new_enc),
- buff);
- //// DumpDetail(destatep);
- }
-
- SimplePrune(destatep, kFinalPruneDifference);
- CalcReliable(destatep);
- }
-
- //if (new_enc != enc) {
- // // TEMP
- // printf(" Rescore new top encoding = %s\n",
- // MyRankedEncName(destatep->top_rankedencoding));
- //}
-
- return new_enc;
- }
-
-
- // Given an encoding, add its corresponding ranked encoding to the set
- void AddToSet(Encoding enc, int* list_len, int* list) {
- // TEMP print
- int item = CompactEncDet::BackmapEncodingToRankedEncoding(enc);
- for (int i = 0; i < *list_len; ++i) {
- if (list[i] == item) {
- return; // Already in the set; don't add again
- }
- }
- list[(*list_len)++] = item;
- }
-
-
- static const int kMinRobustBigramCount = 1000;
- static const int kMinKBToRobustScan = 64;
- static const int kMaxKBToRobustScan = 256;
-
- // Scan the first 64K or so, just doing raw bigram increments on given
- // probability list.
- // No fancy duplicate filtering or anything else here.
- // Returns number of bigrams counted
- int RobustScan(const char* text,
- int text_length,
- int robust_renc_list_len,
- int* robust_renc_list,
- int* robust_renc_probs) {
- if (FLAGS_counts) {++robust_used;}
- // Zero all the result probabilities
- for (int i = 0; i < robust_renc_list_len; ++i) {
- robust_renc_probs[i] = 0;
- }
- int max_fast_len = minint(text_length, (kMaxKBToRobustScan << 10));
- const uint8* isrc = reinterpret_cast<const uint8*>(text);
- const uint8* src = isrc;
- const uint8* srclimitfast2 = isrc + max_fast_len - 1;
- const uint8* srclimitfast4 = isrc + max_fast_len - 3;
-
- int min_fast_len = minint(text_length, (kMinKBToRobustScan << 10));
- const uint8* srclimitmin = isrc + min_fast_len - 1;
-
- int bigram_count = 0;
-
- if (FLAGS_enc_detect_source) {
- PsSourceInit(kPsSourceWidth);
- fprintf(stderr, "(RobustScan) do-src\n");
- }
-
- // Sum over a big chunk of the input
- // Faster loop, no 7-bit-encodings possible, approx 3000 GB/sec
- //====================================
- while (src < srclimitfast2) {
- // Skip to next interesting bigram
-
- while (src < srclimitfast4) {
- if (((src[0] | src[1] | src[2] | src[3]) & 0x80) != 0) break;
- src += 4;
- }
-
- while (src < srclimitfast2) {
- if ((src[0] & 0x80) != 0) break;
- src++;
- }
-
- if (src < srclimitfast2) {
- // We found a bigram with high bit on
- // Next 5 lines commented out so we don't show all the source.
- //const uint8* srctextlimit = isrc + text_length;
- //if (FLAGS_enc_detect_source) {
- // PsSource(src, isrc, srctextlimit);
- // PsMark(src, 2, isrc, 0);
- //}
-
- uint8 byte1 = src[0];
- uint8 byte2 = src[1];
- uint8 byte1x2x = (byte1 & 0xf0) | ((byte2 >> 4) & 0x0f);
- uint8 byte1f = byte1;
- // Flip top bit of subscript to better separate quadrant 4 (esp. for Hebrew)
- byte1f ^= (byte2 & 0x80);
-
- // The real increments
- for (int j = 0; j < robust_renc_list_len; ++j) {
- int rankedencoding = robust_renc_list[j];
- const UnigramEntry* ue = &unigram_table[rankedencoding];
- int incr = ue->b1[byte1f] + ue->b2[byte2] + ue->b12[byte1x2x];
- if ((ue->b12[byte1x2x] & 0x01) != 0) {
- // Use a more-precise table
- int byte32x32 = ((byte1 & 0x1f) << 5) | (byte2 & 0x1f);
- int hiressub = (byte2 & 0x60) >> 5; // select w/bits 5&6 of byte 2
- DCHECK(ue->hires[hiressub] != NULL);
- incr += ue->hires[hiressub][byte32x32];
- } else {
- // Default final offset
- incr += ue->so;
- }
- robust_renc_probs[j] += incr;
- }
-
- src += 2; // Continue after this bigram
- ++bigram_count;
-
- // Stop after 1000 bigrams reached, if at least 64KB scanned
- if ((bigram_count > kMinRobustBigramCount) && (src > srclimitmin)) {
- break;
- }
-
- }
- }
-
- if (FLAGS_enc_detect_source) {
- fprintf(stderr, "( bigram_count = %d) do-src\n", bigram_count);
- if (bigram_count == 0) {bigram_count = 1;} // zdiv
- for (int i = 0; i < robust_renc_list_len; ++i) {
- fprintf(stderr, "( enc[%-12.12s] = %7d (avg %d)) do-src\n",
- MyRankedEncName(robust_renc_list[i]), robust_renc_probs[i],
- robust_renc_probs[i] / bigram_count);
- }
- PsSourceFinish();
- }
-
- return bigram_count;
- }
-
- // If unreliable, rescan middle of document to see if we can get a better
- // answer. Rescan is only worthwhile if there are ~200 bytes or more left,
- // since the detector takes as much as 96 bytes of bigrams to decide.
- Encoding Rescan(Encoding enc,
- const uint8* isrc,
- const uint8* src,
- const uint8* srctextlimit,
- const char* url_hint,
- const char* http_charset_hint,
- const char* meta_charset_hint,
- const int encoding_hint,
- const Language language_hint,
- const CompactEncDet::TextCorpusType corpus_type,
- bool ignore_7bit_mail_encodings,
- DetectEncodingState* destatep) {
- bool enc_is_reliable = destatep->reliable;
- Encoding new_enc = enc;
- Encoding second_best_enc =
- kMapToEncoding[destatep->second_top_rankedencoding];
-
- if (FLAGS_counts) {++rescan_used;}
-
- int scanned_bytes = src - isrc;
- int unscanned_bytes = srctextlimit - src;
- int text_length = srctextlimit - isrc;
- bool empty_rescan = true;
-
- // See if enough bytes left to bother doing rescan
- if (kMinRescanLength < unscanned_bytes) {
- const char* text = reinterpret_cast<const char*>(isrc);
-
- Encoding one_hint = destatep->http_hint;
- if ((one_hint == UNKNOWN_ENCODING) &&
- (destatep->meta_hint != UNKNOWN_ENCODING)) {
- one_hint = destatep->meta_hint;
- }
- if ((one_hint == UNKNOWN_ENCODING) &&
- (destatep->bom_hint != UNKNOWN_ENCODING)) {
- one_hint = destatep->bom_hint;
- }
-
- // Go to an even offset to keep UTF-16 in synch
- int middle_offset = (scanned_bytes + (unscanned_bytes / 2)) & ~1;
- CHECK(middle_offset <= text_length);
-
- // Look back a bit for a low byte to synchronize, else hope for the best.
- const uint8* srcbacklimit = isrc + middle_offset - kMaxScanBack;
- if (srcbacklimit < src) {
- srcbacklimit = src;
- }
- const uint8* ss = isrc + middle_offset - 1;
- while (srcbacklimit <= ss) {
- if ((*ss & 0x80) == 0) {break;}
- --ss;
- }
- // Leave middle offset unchanged unless we found a low byte
- if (srcbacklimit <= ss) {
- // Align to low byte or high byte just after it, whichever is even
- middle_offset = (ss - isrc + 1) & ~1; // Even to keep UTF-16 in sync
- }
- CHECK(middle_offset <= text_length);
-
- if (destatep->debug_data != NULL) {
- SetDetailsEncLabel(destatep, ">> Rescan");
- // Print the current chart before recursive call
- DumpDetail(destatep);
-
- char buff[32];
- snprintf(buff, sizeof(buff), ">> Rescan[%d..%d]",
- middle_offset, text_length);
- PsRecurse(buff);
- }
-
- int mid_bytes_consumed;
- bool mid_is_reliable;
- Encoding mid_second_best_enc;
- CEDInternalFlags newflags = static_cast<CEDInternalFlags>(
- kCEDRescanning + kCEDForceTags);
- // Recursive call for rescan of half of remaining
- Encoding mid_enc = InternalDetectEncoding(
- newflags,
- text + middle_offset,
- text_length - middle_offset,
- url_hint,
- http_charset_hint,
- meta_charset_hint,
- encoding_hint,
- language_hint, // User interface lang
- corpus_type,
- ignore_7bit_mail_encodings,
- &mid_bytes_consumed,
- &mid_is_reliable,
- &mid_second_best_enc);
- destatep->reliable = mid_is_reliable;
-
- empty_rescan = (mid_enc == ASCII_7BIT);
-
- // Not the right decision if, e.g. enc=Greek, mid=ASCII7, one=KSC
- // hence the !empty_rescan term
- if (!empty_rescan && CompatibleEnc(one_hint, mid_enc)) {
- // Encoding we just found is compatible with the
- // single hint (if any); return superset
- new_enc = SupersetEnc(one_hint, mid_enc);
- }
-
- // If original and mid are compatible, and both reliable,
- // return new_enc = SupersetEnc(enc, mid_enc)
- //
- // This avoids too much weight on a bogus hint causing a RobustScan
- // that gets the wrong answer
- if (!empty_rescan && mid_is_reliable && enc_is_reliable &&
- CompatibleEnc(enc, mid_enc)) {
- new_enc = SupersetEnc(enc, mid_enc);
- return new_enc;
- }
-
- // if mid unreliable, robustscan
- // if mid empty, robustscan
- // if original and mid not compatible, robustscan
- // if mid and one_hint not compatible, robustscan
-
- // If we found conflicting data, drop back and do a robust scan of a big
- // chunk of the input over a set of candidate encodings
- //
- if (!mid_is_reliable ||
- empty_rescan ||
- !CompatibleEnc(enc, mid_enc) ||
- !CompatibleEnc(one_hint, mid_enc)) {
- int robust_renc_list_len; // Number of active encodings
- int robust_renc_list[NUM_RANKEDENCODING]; // List of ranked encodings
- int robust_renc_probs[NUM_RANKEDENCODING]; // List of matching probs
-
- robust_renc_list_len = 0;
- AddToSet(enc, &robust_renc_list_len, robust_renc_list);
- AddToSet(second_best_enc, &robust_renc_list_len, robust_renc_list);
- AddToSet(mid_enc, &robust_renc_list_len, robust_renc_list);
- AddToSet(mid_second_best_enc, &robust_renc_list_len, robust_renc_list);
- if (destatep->http_hint != UNKNOWN_ENCODING) {
- AddToSet(destatep->http_hint, &robust_renc_list_len, robust_renc_list);
- }
- if (destatep->meta_hint != UNKNOWN_ENCODING) {
- AddToSet(destatep->meta_hint, &robust_renc_list_len, robust_renc_list);
- }
- if (destatep->bom_hint != UNKNOWN_ENCODING) {
- AddToSet(destatep->bom_hint, &robust_renc_list_len, robust_renc_list);
- }
- if (destatep->tld_hint != UNKNOWN_ENCODING) {
- AddToSet(destatep->tld_hint, &robust_renc_list_len, robust_renc_list);
- }
-
- // Separate simple scan
- // =====================
- if (destatep->debug_data != NULL) {
- SetDetailsEncLabel(destatep, ">> RobustScan");
- // Print the current chart before recursive call
- DumpDetail(destatep);
-
- char buff[32];
- snprintf(buff, sizeof(buff), ">> RobustScan[0..%d]", text_length);
- PsRecurse(buff);
- }
-
- int bigram_count = RobustScan(text, text_length,
- robust_renc_list_len, robust_renc_list, robust_renc_probs);
-
- // Default to new_enc and update if something better was found
- int best_prob = -1;
- // TEMP print
- for (int i = 0; i < robust_renc_list_len; ++i) {
- if (best_prob < robust_renc_probs[i]) {
- best_prob = robust_renc_probs[i];
- new_enc = kMapToEncoding[robust_renc_list[i]];
- }
- }
-
- if (destatep->debug_data != NULL) {
- char buff[32];
- snprintf(buff, sizeof(buff), "=Robust[%d] %s",
- bigram_count, MyEncodingName(new_enc));
- SetDetailsEncProb(destatep,
- 0,
- CompactEncDet::BackmapEncodingToRankedEncoding(new_enc),
- buff);
- }
- }
- } // End if enough bytes
-
- return new_enc;
- }
-
- // With no hints at all, and perhaps on rescan, we relax our pickiness
- // and go ahead and accept the top multibyte encodings, even though
- // strictly their web pages should have declared an explicit encoding to
- // avoid the HTML standard's default ISO-8859-1.
- bool NoHintsCloseEnoughCompatible(Encoding top_enc) {
- // First test accepts degenerate cases plus UTF8 and UTF8UTF8
- if (CompatibleEnc(UTF8, top_enc)) {return true;}
-
- // The rest look for exact match of base encoding
- Encoding base_enc = kMapEncToBaseEncoding[top_enc];
- if (base_enc == JAPANESE_EUC_JP) {return true;}
- if (base_enc == JAPANESE_SHIFT_JIS) {return true;}
- if (base_enc == CHINESE_BIG5) {return true;}
- if (base_enc == CHINESE_GB) {return true;}
- if (base_enc == KOREAN_EUC_KR) {return true;}
- return false;
- }
-
-
-
- // Scan raw bytes and detect most likely encoding
- // Design goals:
- // Skip over big initial stretches of seven-bit ASCII bytes very quickly
- // Thread safe
- // Works equally well on
- // 50-byte queries,
- // 5000-byte email and
- // 50000-byte web pages
- // Length 0 input returns ISO_8859_1 (ASCII) encoding
- // Setting ignore_7bit_mail_encodings effectively turns off detection of
- // UTF-7, HZ, and ISO-2022-xx
- Encoding InternalDetectEncoding(
- CEDInternalFlags flags, const char* text, int text_length,
- const char* url_hint, const char* http_charset_hint,
- const char* meta_charset_hint, const int encoding_hint,
- const Language language_hint, // User interface lang
- const CompactEncDet::TextCorpusType corpus_type,
- bool ignore_7bit_mail_encodings, int* bytes_consumed, bool* is_reliable,
- Encoding* second_best_enc) {
- *bytes_consumed = 0;
- *is_reliable = false;
- *second_best_enc = ASCII_7BIT;
-
- if (text_length == 0) {
- // Follow the spec. Text might be NULL.
- *is_reliable = true;
- return ISO_8859_1;
- }
-
- // For very short (20-50 byte) input strings that are highly likely to be
- // all printable ASCII, our startup overhead might dominate. We have to do the
- // full detection if the ISO-2022-xx, HZ, or UTF-7 encodings are possible.
- // Otherwise, we can do a quick scan for printable ASCII.
- if ((text_length <= 500) && ignore_7bit_mail_encodings &&
- QuickPrintableAsciiScan(text, text_length)) {
- *is_reliable = true;
- return ASCII_7BIT;
- }
-
- // Go for the full boat detection
- DetectEncodingState destate;
- InitDetectEncodingState(&destate);
-
- std::unique_ptr<DetailEntry[]> scoped_debug_data;
- if (FLAGS_enc_detect_detail) {
- // Allocate max 10 details per bigram
- scoped_debug_data.reset(new DetailEntry[kMaxPairs * 10]);
- destate.debug_data = scoped_debug_data.get();
- // NOTE: destate and scoped_debug_data have exactly the same scope
- // All other FLAGS_enc_detect_detail tests use destate.debug_data != NULL
- }
-
- // Get text length limits
- // Typically, we scan the first 16KB looking for all encodings, then
- // scan the rest (up to 256KB) a bit faster by no longer looking for
- // interesting bytes below 0x80. This allows us to skip over runs of
- // 7-bit-ASCII much more quickly.
- int slow_len = minint(text_length, (FLAGS_enc_detect_slow_max_kb << 10));
- int fast_len = minint(text_length, (FLAGS_enc_detect_fast_max_kb << 10));
-
- // Initialize pointers.
- // In general, we do not look at last 3 bytes of input in the fast scan
- // We do, however want to look at the last byte or so in the slow scan,
- // especilly in the case of a very short text whose only interesting
- // information is a 3-byte UTF-8 character in the last three bytes.
- // If necessary, we fake a last bigram with 0x20 space as a pad byte.
- const uint8* isrc = reinterpret_cast<const uint8*>(text);
- const uint8* src = isrc;
- const uint8* srctextlimit = isrc + text_length;
- const uint8* srclimitslow2 = isrc + slow_len - 1;
- const uint8* srclimitfast2 = isrc + fast_len - 1;
- const uint8* srclimitfast4 = isrc + fast_len - 3;
- if (srclimitslow2 > srclimitfast2) {
- srclimitslow2 = srclimitfast2;
- }
- destate.initial_src = isrc;
- destate.limit_src = srclimitfast2 + 1; // May include last byte
- destate.prior_src = isrc;
- destate.last_pair = isrc - 2;
-
- const char* scan_table = kTestPrintableAsciiTildePlus;
- if (ignore_7bit_mail_encodings) {
- // Caller wants to ignore UTF-7, HZ, ISO-2022-xx
- // Don't stop on + (for UTF-7), nor on ~ (for HZ)
- scan_table = kTestPrintableAscii;
- }
- int exit_reason = 0;
-
- if (destate.debug_data != NULL) {
- BeginDetail(&destate);
- // Take any incoming watch encoding name and backmap to the corresponding
- // ranked enum value
- watch1_rankedenc = LookupWatchEnc(FLAGS_enc_detect_watch1);
- if (watch1_rankedenc >= 0) {
- fprintf(stderr, "/track-me %d def\n", watch1_rankedenc);
- }
-
- watch2_rankedenc = LookupWatchEnc(FLAGS_enc_detect_watch2);
- if (watch2_rankedenc >= 0) {
- fprintf(stderr, "/track-me2 %d def\n", watch2_rankedenc);
- }
-
- fprintf(stderr, "%% kDerateHintsBelow = %d\n", kDerateHintsBelow);
- }
- if (FLAGS_enc_detect_source) {
- PsSourceInit(kPsSourceWidth);
- PsSource(src, isrc, srctextlimit);
- PsMark(src, 4, isrc, 0);
- }
-
- // Apply hints, if any, to probabilities
- // NOTE: Encoding probabilites are all zero at this point
- ApplyHints(url_hint,
- http_charset_hint,
- meta_charset_hint,
- encoding_hint,
- language_hint,
- corpus_type,
- &destate);
-
- // NOTE: probabilities up to this point are subject to derating for
- // small numbers of bigrams.
- // Probability changes after this point are not derated.
-
- // Do first 4 bytes to pick off strong markers
- InitialBytesBoost(isrc, text_length, &destate);
-
- bool ignored_some_tag_text = false;
- int tag_text_bigram_count = 0;
-
- // Slower loop, approx 500 MB/sec (2.8 GHz P4)
- // ASSERT(srclimitslow2 <= srclimitfast2);
- //====================================
- DoMoreSlowLoop:
- while (src < srclimitslow2) {
- // Skip to next interesting byte (this is the slower part)
- while (src < srclimitslow2) {
- uint8 uc = *src++;
- if (scan_table[uc] != 0) {exit_reason = scan_table[uc]; src--; break;}
- }
-
- if (src < srclimitslow2) {
- if (FLAGS_enc_detect_source) {
- PsSource(src, isrc, srctextlimit); // don't mark yet
- }
-
- int weightshift = 0;
- // In the first 16KB, derate new text run inside <title>...</title> and
- // inside <!-- ... -->
- if (////((destate.last_pair + 6) <= src) && // if beyond last one
- ////(tag_text_bigram_count < kMaxBigramsTagTitleText) &&
- (corpus_type == CompactEncDet::WEB_CORPUS) && // and web page
- !CEDFlagForceTags(flags)) { // and OK to skip
- ////if (TextInsideTag(destate.last_pair + 2, src, srclimitslow2)) {
- if (TextInsideTag(isrc, src, srclimitslow2)) {
- if (tag_text_bigram_count >= kMaxBigramsTagTitleText) {
- ignored_some_tag_text = true;
- src = SkipToTagEnd(src, srclimitslow2);
- continue;
- } else {
- weightshift = kWeightshiftForTagTitleText;
- ++tag_text_bigram_count;
- }
- }
- }
- if (FLAGS_enc_detect_source) {
- PsMark(src, 2, isrc, weightshift);
- }
- // Saves byte pair and offset
- bool pruned = IncrementAndBoostPrune(src, srctextlimit - src,
- &destate, weightshift, exit_reason);
- // Advance; if inside tag, advance to end of tag
- if (weightshift == 0) {
- src += exit_reason; // 1 Ascii, 2 other
- } else {
- src += exit_reason; // 1 Ascii, 2 other
- //// src = SkipToTagEnd(src, srclimitslow2);
- }
-
- if (pruned) {
- // Scoring and active encodings have been updated
- if (destate.done) {break;}
- // Check if all the reasons for the slow loop have been pruned
- // If so, go to fast loop
- if (!SevenBitActive(&destate)) {break;}
- }
- }
- }
- //====================================
-
- // We reached the end of a slow scan, possibly because no more SevenBitActive,
- // or possibly are at end of source.
- // If we are exactly at the end of the source, make sure we look at the very
- // last byte.
- bool very_last_byte_incremented = false;
- if (src == (srctextlimit - 1)) {
- exit_reason = scan_table[*src];
- if (exit_reason != 0) {
- // The very last byte is an interesting byte
- // Saves byte pair and offset
- //printf("Interesting very last slow byte = 0x%02x\n", *src);
- IncrementAndBoostPrune(src, srctextlimit - src, &destate, 0, exit_reason);
- very_last_byte_incremented = true;
- }
- }
-
- if (FLAGS_enc_detect_source) {
- PsSource(src, isrc, srctextlimit);
- PsMark(src, 2, isrc, 0);
- }
- // Force a pruning based on whatever we have
- // Delete the seven-bit encodings if there is no evidence of them so far
- BoostPrune(src, &destate, PRUNE_SLOWEND);
-
- if (!destate.done) {
- // If not clear yet on 7-bit-encodings and more bytes, do more slow
- if (SevenBitActive(&destate) && (src < srclimitfast2)) {
- // Increment limit by another xxxK
- slow_len += (FLAGS_enc_detect_slow_max_kb << 10);
- srclimitslow2 = isrc + slow_len - 1;
- if (srclimitslow2 > srclimitfast2) {
- srclimitslow2 = srclimitfast2;
- }
- if (!UTF7OrHzActive(&destate)) {
- // We can switch to table that does not stop on + ~
- scan_table = kTestPrintableAscii;
- }
- goto DoMoreSlowLoop;
- }
-
-
- exit_reason = 2;
- // Faster loop, no 7-bit-encodings possible, approx 3000 GB/sec
- //====================================
- while (src < srclimitfast2) {
- // Skip to next interesting byte (this is the faster part)
- while (src < srclimitfast4) {
- if (((src[0] | src[1] | src[2] | src[3]) & 0x80) != 0) break;
- src += 4;
- }
-
- while (src < srclimitfast2) {
- if ((src[0] & 0x80) != 0) break;
- src++;
- }
-
- if (src < srclimitfast2) {
- if (FLAGS_enc_detect_source) {
- PsSource(src, isrc, srctextlimit);
- PsMark(src, 2, isrc, 0);
- }
- // saves byte pair and offset
- bool pruned = IncrementAndBoostPrune(src, srctextlimit - src,
- &destate, 0, exit_reason);
- src += exit_reason; // 1 Ascii, 2 other
- if (pruned) {
- // Scoring and active encodings have been updated
- if (destate.done) {break;}
- }
- }
- }
- //====================================
- // We reached the end of fast scan
-
- // If we are exactly at the end of the source, make sure we look at the very
- // last byte.
- if (src == (srctextlimit - 1) && !very_last_byte_incremented) {
- exit_reason = scan_table[*src];
- if (exit_reason != 0) {
- // The very last byte is an interesting byte
- // Saves byte pair and offset
- //printf("Interesting very last fast byte = 0x%02x\n", *src);
- IncrementAndBoostPrune(src, srctextlimit - src, &destate, 0, exit_reason);
- very_last_byte_incremented = true;
- }
- }
-
- } // End if !done
-
- if (FLAGS_enc_detect_source) {
- PsSource(src, isrc, srctextlimit);
- PsMark(src, 2, isrc, 0);
- }
- // Force a pruning based on whatever we have
- BoostPrune(src, &destate, PRUNE_FINAL);
-
- if (FLAGS_enc_detect_summary) {
- DumpSummary(&destate, AsciiPair, 32);
- DumpSummary(&destate, OtherPair, 32);
- }
- if (FLAGS_enc_detect_source) {
- PsSourceFinish();
- }
- if (destate.debug_data != NULL) {
- //// DumpDetail(&destate);
- }
-
-
- if (ignored_some_tag_text &&
- (kMapToEncoding[destate.top_rankedencoding] == ASCII_7BIT)) {
- // There were some interesting bytes, but only in tag text.
- // Recursive call to reprocess looking at the tags this time.
-
- if (destate.debug_data != NULL) {
- SetDetailsEncLabel(&destate, ">> Recurse/tags");
- // Print the current chart before recursive call
- DumpDetail(&destate);
-
- char buff[32];
- snprintf(buff, sizeof(buff), ">> Recurse for tags");
- PsRecurse(buff);
- }
-
- // Recursive call for high bytes in tags [no longer used, 1/16 tag score]
- Encoding enc2 = InternalDetectEncoding(
- kCEDForceTags, // force
- text,
- text_length,
- url_hint,
- http_charset_hint,
- meta_charset_hint,
- encoding_hint,
- language_hint,
- corpus_type,
- ignore_7bit_mail_encodings,
- bytes_consumed,
- is_reliable,
- second_best_enc);
-
- if (destate.debug_data != NULL) {
- // Show winning encoding and dump PostScript
- char buff[32];
- snprintf(buff, sizeof(buff), "=2 %s", MyEncodingName(enc2));
- SetDetailsEncProb(&destate,
- 0,
- CompactEncDet::BackmapEncodingToRankedEncoding(enc2),
- buff);
- DumpDetail(&destate);
- }
-
- return enc2;
- }
-
-
- // If the detected encoding does not match default/hints, or if the hints
- // conflict with each other, mark as unreliable. This can be used to trigger
- // further scoring.
- // Three buckets of input documents;
- // ~19% of the web no hints, and top == 7bit, Latin1, or CP1252
- // ~79% of the web one or more hints, all same encoding X and top == X
- // ~ 2% of the web one or more hints that are inconsistent
-
- Encoding top_enc = kMapToEncoding[destate.top_rankedencoding];
- Encoding one_hint = destate.http_hint;
- if ((one_hint == UNKNOWN_ENCODING) &&
- (destate.meta_hint != UNKNOWN_ENCODING)) {
- one_hint = destate.meta_hint;
- }
- if ((one_hint == UNKNOWN_ENCODING) &&
- (destate.bom_hint != UNKNOWN_ENCODING)) {
- one_hint = destate.bom_hint;
- }
-
- bool found_compatible_encoding = true;
- if (one_hint == UNKNOWN_ENCODING) {
- // [~14% of the web] No hints, and top == 7bit, Latin1, or CP1252
- if (!CompatibleEnc(ISO_8859_1, top_enc)) {
- found_compatible_encoding = false;
- // If there is nothing but a TLD hint and its top encoding matches, OK
- if ((destate.tld_hint != UNKNOWN_ENCODING) &&
- CompatibleEnc(destate.tld_hint, top_enc)) {
- found_compatible_encoding = true;
- }
- }
- } else if (CompatibleEnc(one_hint, destate.http_hint) &&
- CompatibleEnc(one_hint, destate.meta_hint) &&
- CompatibleEnc(one_hint, destate.bom_hint)) {
- // [~83% of the web] One or more hints, all same encoding X and top == X
- if (!CompatibleEnc(one_hint, top_enc)) {
- // [~ 2% of the web] Oops, not the declared encoding
- found_compatible_encoding = false;
- }
- } else {
- // [~ 3% of the web] Two or more hints that are inconsistent
- one_hint = UNKNOWN_ENCODING;
- found_compatible_encoding = false;
- }
-
- // If we turned Latin1 into Latin2 or 7 via trigrams, don't fail it here
- if (destate.do_latin_trigrams) {
- if (CompatibleEnc(kMapToEncoding[F_Latin1], top_enc) ||
- CompatibleEnc(kMapToEncoding[F_Latin2], top_enc) ||
- CompatibleEnc(kMapToEncoding[F_CP1250], top_enc) ||
- CompatibleEnc(kMapToEncoding[F_ISO_8859_13], top_enc)) {
- found_compatible_encoding = true;
- destate.reliable = true;
- }
- }
-
- // If top encoding is not compatible with the hints, but it is reliably
- // UTF-8, accept it anyway.
- // This will perform badly with mixed UTF-8 prefix plus another encoding in
- // the body if done too early, so we want to be rescanning.
- if (!found_compatible_encoding &&
- destate.reliable &&
- NoHintsCloseEnoughCompatible(top_enc) &&
- (destate.next_interesting_pair[OtherPair] >= kStrongPairs) &&
- CEDFlagRescanning(flags)) {
- found_compatible_encoding = true;
- }
-
- // Hold off on this so Rescan() can see if the original encoding was reliable
- //if (!found_compatible_encoding) {
- // destate.reliable = false;
- //}
-
- // If unreliable, try rescoring to separate some encodings
- if (!destate.reliable || !found_compatible_encoding) {
- top_enc = Rescore(top_enc, isrc, srctextlimit, &destate);
- }
-
- *second_best_enc = kMapToEncoding[destate.second_top_rankedencoding];
-
- // If unreliable, and not already rescanning,
- // rescan middle of document to see if we can get a better
- // answer. Rescan is only worthwhile if there are ~200 bytes or more left,
- // since the detector takes as much as 96 bytes of bigrams to decide.
- //
- // CANNOT retry ISO-2022-xx HZ etc. because no declaration escape at the front
- // or we may land in the middle of some partial state. Skip them all.
- //
- if ((!destate.reliable || !found_compatible_encoding) &&
- !CEDFlagRescanning(flags) &&
- !SevenBitEncoding(top_enc)) {
- top_enc = Rescan(top_enc,
- isrc,
- src,
- srctextlimit,
- url_hint,
- http_charset_hint,
- meta_charset_hint,
- encoding_hint,
- language_hint,
- corpus_type,
- ignore_7bit_mail_encodings,
- &destate);
- } else {
- if (!found_compatible_encoding) {
- destate.reliable = false;
- }
- }
-
- if (destate.debug_data != NULL) {
- // Dump PostScript
- DumpDetail(&destate);
- }
-
- *bytes_consumed = src - isrc + 1; // We looked 1 byte beyond src
- *is_reliable = destate.reliable;
- return top_enc;
- }
-
- Encoding CompactEncDet::DetectEncoding(
- const char* text, int text_length, const char* url_hint,
- const char* http_charset_hint, const char* meta_charset_hint,
- const int encoding_hint,
- const Language language_hint, // User interface lang
- const TextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
- int* bytes_consumed, bool* is_reliable) {
- if (FLAGS_ced_echo_input) {
- string temp(text, text_length);
- fprintf(stderr, "CompactEncDet::DetectEncoding()\n%s\n\n", temp.c_str());
- }
-
- if (FLAGS_counts) {
- encdet_used = 0;
- rescore_used = 0;
- rescan_used = 0;
- robust_used = 0;
- looking_used = 0;
- doing_used = 0;
- ++encdet_used;
- }
- if (FLAGS_dirtsimple) {
- // Just count first 64KB bigram encoding probabilities for each encoding
- int robust_renc_list_len; // Number of active encodings
- int robust_renc_list[NUM_RANKEDENCODING]; // List of ranked encodings
- int robust_renc_probs[NUM_RANKEDENCODING]; // List of matching probs
-
- for (int i = 0; i < NUM_RANKEDENCODING; ++i) {
- robust_renc_list[i] = i;
- }
- robust_renc_list_len = NUM_RANKEDENCODING;
-
- RobustScan(text, text_length,
- robust_renc_list_len, robust_renc_list, robust_renc_probs);
-
- // Pick off best encoding
- int best_prob = -1;
- Encoding enc = UNKNOWN_ENCODING;
- for (int i = 0; i < robust_renc_list_len; ++i) {
- if (best_prob < robust_renc_probs[i]) {
- best_prob = robust_renc_probs[i];
- enc = kMapToEncoding[robust_renc_list[i]];
- }
- }
-
- *bytes_consumed = minint(text_length, (kMaxKBToRobustScan << 10));
- *is_reliable = true;
- if (FLAGS_counts) {
- printf("CEDcounts ");
- while (encdet_used--) {printf("encdet ");}
- while (rescore_used--) {printf("rescore ");}
- while (rescan_used--) {printf("rescan ");}
- while (robust_used--) {printf("robust ");}
- while (looking_used--) {printf("looking ");}
- while (doing_used--) {printf("doing ");}
- printf("\n");
- }
-
- return enc;
- }
-
- Encoding second_best_enc;
- Encoding enc = InternalDetectEncoding(kCEDNone,
- text,
- text_length,
- url_hint,
- http_charset_hint,
- meta_charset_hint,
- encoding_hint,
- language_hint, // User interface lang
- corpus_type,
- ignore_7bit_mail_encodings,
- bytes_consumed,
- is_reliable,
- &second_best_enc);
- if (FLAGS_counts) {
- printf("CEDcounts ");
- while (encdet_used--) {printf("encdet ");}
- while (rescore_used--) {printf("rescore ");}
- while (rescan_used--) {printf("rescan ");}
- while (robust_used--) {printf("robust ");}
- while (looking_used--) {printf("looking ");}
- while (doing_used--) {printf("doing ");}
- printf("\n");
- }
-
- #if defined(HTML5_MODE)
- // Map all the Shift-JIS variants to Shift-JIS when used in Japanese locale.
- if (language_hint == JAPANESE && IsShiftJisOrVariant(enc)) {
- enc = JAPANESE_SHIFT_JIS;
- }
-
- // 7-bit encodings (except ISO-2022-JP), and some obscure encodings not
- // supported in WHATWG encoding standard are marked as ASCII to keep the raw
- // bytes intact.
- switch (enc) {
- case ISO_2022_KR:
- case ISO_2022_CN:
- case HZ_GB_2312:
- case UTF7:
- case UTF16LE:
- case UTF16BE:
-
- case CHINESE_EUC_DEC:
- case CHINESE_CNS:
- case CHINESE_BIG5_CP950:
- case JAPANESE_CP932:
- case MSFT_CP874:
- case TSCII:
- case TAMIL_MONO:
- case TAMIL_BI:
- case JAGRAN:
- case BHASKAR:
- case HTCHANAKYA:
- case BINARYENC:
- case UTF8UTF8:
- case TAM_ELANGO:
- case TAM_LTTMBARANI:
- case TAM_SHREE:
- case TAM_TBOOMIS:
- case TAM_TMNEWS:
- case TAM_WEBTAMIL:
- case KDDI_SHIFT_JIS:
- case DOCOMO_SHIFT_JIS:
- case SOFTBANK_SHIFT_JIS:
- case KDDI_ISO_2022_JP:
- case SOFTBANK_ISO_2022_JP:
- enc = ASCII_7BIT;
- break;
- default:
- break;
- }
- #endif
-
- return enc;
- }
-
-
- // Return top encoding hint for given string
- Encoding CompactEncDet::TopEncodingOfLangHint(const char* name) {
- string normalized_lang = MakeChar8(string(name));
- int n = HintBinaryLookup8(kLangHintProbs, kLangHintProbsSize,
- normalized_lang.c_str());
- if (n < 0) {return UNKNOWN_ENCODING;}
-
- // Charset is eight bytes, probability table is eight bytes
- int toprankenc =
- TopCompressedProb((const char *)&kLangHintProbs[n].key_prob[kMaxLangKey],
- kMaxLangVector);
- return kMapToEncoding[toprankenc];
- }
-
- // Return top encoding hint for given string
- Encoding CompactEncDet::TopEncodingOfTLDHint(const char* name) {
- string normalized_tld = MakeChar4(string(name));
- int n = HintBinaryLookup4(kTLDHintProbs, kTLDHintProbsSize,
- normalized_tld.c_str());
- if (n < 0) {return UNKNOWN_ENCODING;}
-
- // TLD is four bytes, probability table is 12 bytes
- int toprankenc =
- TopCompressedProb((const char *)&kTLDHintProbs[n].key_prob[kMaxTldKey],
- kMaxTldVector);
- return kMapToEncoding[toprankenc];
- }
-
- // Return top encoding hint for given string
- Encoding CompactEncDet::TopEncodingOfCharsetHint(const char* name) {
- string normalized_charset = MakeChar44(string(name));
- int n = HintBinaryLookup8(kCharsetHintProbs, kCharsetHintProbsSize,
- normalized_charset.c_str());
- if (n < 0) {return UNKNOWN_ENCODING;}
-
- // Charset is eight bytes, probability table is eight bytes
- int toprankenc =
- TopCompressedProb((const char *)&kCharsetHintProbs[n].key_prob[kMaxCharsetKey],
- kMaxCharsetVector);
- return kMapToEncoding[toprankenc];
- }
-
- const char* CompactEncDet::Version(void) {
- return kVersion;
- }
|