[Rework] Use google-ced instead of libicu chardet as the former sucks

CMakeLists.txt

@@ -623,6 +623,8 @@ ADD_SUBDIRECTORY(contrib/t1ha)
 ADD_SUBDIRECTORY(contrib/libev)
 ADD_SUBDIRECTORY(contrib/kann)
 ADD_SUBDIRECTORY(contrib/fastutf8)
+ADD_SUBDIRECTORY(contrib/google-ced)
+
 
 IF (NOT WITH_LUAJIT)
 	ADD_SUBDIRECTORY(contrib/lua-bit)
@@ -643,6 +645,7 @@ LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-actrie)
 LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-t1ha)
 LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-ev)
 LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-kann)
+LIST(APPEND RSPAMD_REQUIRED_LIBRARIES rspamd-ced)
 
 IF(ENABLE_CLANG_PLUGIN MATCHES "ON")
 	ADD_SUBDIRECTORY(clang-plugin)

contrib/google-ced/CMakeLists.txt

@@ -0,0 +1,26 @@
+# Copyright 2016 Google Inc.  All Rights Reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+project(CED CXX)
+set(CMAKE_SUPPRESS_DEVELOPER_WARNINGS 1 CACHE INTERNAL "No dev warnings")
+
+option(BUILD_SHARED_LIBS "Build shared libraries" OFF)
+
+if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wno-narrowing")
+elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wno-c++11-narrowing")
+endif()
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+set(CED_LIBRARY_SOURCES
+    compact_enc_det.cc
+    compact_enc_det_hint_code.cc
+    util/encodings/encodings.cc
+    util/languages/languages.cc
+    ced_c.cc
+    )
+
+add_library(rspamd-ced STATIC ${CED_LIBRARY_SOURCES})

contrib/google-ced/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

contrib/google-ced/ced_c.cc

@@ -0,0 +1,25 @@
+#include "ced_c.h"
+#include "compact_enc_det.h"
+
+const char* ced_encoding_detect(const char* text, int text_length,
+								const char* url_hint,
+								const char* http_charset_hint,
+								const char* meta_charset_hint,
+								const int encoding_hint,
+								CedTextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
+								int* bytes_consumed, bool* is_reliable)
+{
+	CompactEncDet::TextCorpusType ct = CompactEncDet::NUM_CORPA;
+
+	ct = static_cast<CompactEncDet::TextCorpusType>(corpus_type);
+
+	auto enc = CompactEncDet::DetectEncoding(text, text_length, url_hint,
+			http_charset_hint, meta_charset_hint, encoding_hint, default_language(),
+			ct, ignore_7bit_mail_encodings, bytes_consumed, is_reliable);
+
+	if (IsValidEncoding(enc)) {
+		return EncodingName(enc);
+	}
+
+	return nullptr;
+}

contrib/google-ced/ced_c.h

@@ -0,0 +1,29 @@
+#ifndef RSPAMD_CED_C_H
+#define RSPAMD_CED_C_H
+
+#include <stdbool.h>
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+enum CedTextCorpusType {
+	CED_WEB_CORPUS,
+	CED_XML_CORPUS,
+	CED_QUERY_CORPUS,
+	CED_EMAIL_CORPUS,
+	CED_NUM_CORPA,
+};
+
+const char *ced_encoding_detect (const char *text, int text_length,
+								 const char *url_hint,
+								 const char *http_charset_hint,
+								 const char *meta_charset_hint,
+								 const int encoding_hint,
+								 enum CedTextCorpusType corpus_type,
+								 bool ignore_7bit_mail_encodings,
+								 int *bytes_consumed, bool *is_reliable);
+
+#ifdef  __cplusplus
+}
+#endif
+#endif

contrib/google-ced/compact_enc_det.h

@@ -0,0 +1,83 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_H_
+#define COMPACT_ENC_DET_COMPACT_ENC_DET_H_
+
+#include "util/encodings/encodings.h"  // for Encoding
+#include "util/languages/languages.h"  // for Language
+
+#include <string.h>
+
+namespace CompactEncDet {
+  // We may want different statistics, depending on whether the text being
+  // identfied is from the web, from email, etc.  This is currently ignored,
+  // except WEB_CORPUS enables ignoring chars inside tags.
+  enum TextCorpusType {
+    WEB_CORPUS,
+    XML_CORPUS,
+    QUERY_CORPUS,       // Use this for vanilla plaintext
+    EMAIL_CORPUS,
+    NUM_CORPA,          // always last
+  };
+
+  // Scan raw bytes and detect most likely encoding
+  // Design goals:
+  //   Skip over big initial stretches of seven-bit ASCII bytes very quickly
+  //   Thread safe
+  //   Works equally well on
+  //    50-byte queries,
+  //    5000-byte email and
+  //    50000-byte web pages
+  // Length 0 input returns ASCII (aka ISO-8859-1 or Latin1)
+  //
+  // Inputs: text and text_length
+  //  web page's url (preferred) or just
+  //    top-level domain name (e.g. "com") or NULL as a hint
+  //  web page's HTTPheader charset= string (e.g. "Latin1") or NULL as a hint
+  //  web page's <meta> tag charset= string (e.g. "utf-8") or NULL as a hint
+  //  an Encoding or UNKNOWN_ENCODING as a hint
+  //  a Language or UNKNOWN_LANGUAGE as a hint
+  //  corpus type from the list above. Currently ignored; may select
+  //    different probability tables in the future
+  //  ignore_7bit if true says to NOT return the pure seven-bit encodings
+  //    ISO-2022-JP (aka JIS), ISO-2022-CN, ISO-2022-KR, HZ, and UTF-7.
+  //    This may save a little scoring time on pure printable ASCII input text
+  // Outputs: bytes_consumed says how much of text_length was actually examined
+  //  is_reliable set true if the returned encoding is at least 2**10 time more
+  //  probable then the second-best encoding
+  // Return value: the most likely encoding for the input text
+  //
+  // Setting ignore_7bit_mail_encodings effectively turns off detection of
+  // UTF-7, HZ, and ISO-2022-xx. It is recommended that this flag be true
+  // when corpus_type is QUERY_CORPUS.
+  Encoding DetectEncoding(
+      const char* text, int text_length, const char* url_hint,
+      const char* http_charset_hint, const char* meta_charset_hint,
+      const int encoding_hint,
+      const Language language_hint,  // User interface lang
+      const TextCorpusType corpus_type, bool ignore_7bit_mail_encodings,
+      int* bytes_consumed, bool* is_reliable);
+
+  // Support functions for unit test program
+  int BackmapEncodingToRankedEncoding(Encoding enc);
+  Encoding TopEncodingOfLangHint(const char* name);
+  Encoding TopEncodingOfTLDHint(const char* name);
+  Encoding TopEncodingOfCharsetHint(const char* name);
+  const char* Version(void);
+}      // End namespace CompactEncDet
+
+#endif  // COMPACT_ENC_DET_COMPACT_ENC_DET_H_

contrib/google-ced/compact_enc_det_generated_tables2.h

@@ -0,0 +1,856 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/basictypes.h"
+
+static const uint8 ced_hires_0[1024] = {
+  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  0,135,134,135,89,111,136,129,  81,121,130,40,64,74,78,13,  90,87,103,56,51,37,63,44,  0,97,60,55,143,97,70,100,
+  29,126,116,149,131,156,108,153,  147,119,135,111,133,135,115,0,  2,13,20,4,12,5,17,5,  4,0,58,50,2,30,80,64,
+  35,0,0,0,0,22,0,9,  14,12,4,6,7,8,16,3,  147,160,163,141,144,139,146,127,  131,124,23,17,0,11,16,6,
+  0,68,108,52,129,66,109,65,  113,58,116,124,126,117,88,114,  85,122,102,126,116,122,96,131,  102,126,107,117,49,113,80,128,
+  0,85,119,107,127,75,102,105,  118,101,108,118,101,117,98,128,  125,114,115,125,115,121,116,121,  125,130,122,119,64,102,78,124,
+  93,87,64,93,105,92,76,80,  113,82,75,88,89,68,88,97,  87,102,145,90,90,130,106,116,  129,61,81,75,47,69,74,64,
+  92,98,81,95,81,161,75,81,  101,95,93,75,100,87,108,91,  113,110,118,114,105,105,123,94,  89,92,104,90,62,85,81,84,
+
+  47,141,155,88,85,108,121,141,  94,73,82,81,163,125,111,89,  106,133,156,95,80,96,88,61,  65,47,74,55,33,54,59,19,
+  128,129,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  128,131,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  128,128,128,128,135,128,128,128,  128,128,161,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  141,115,118,118,118,128,115,127,  136,136,122,124,125,126,128,121,  147,128,128,128,128,128,128,123,  128,121,128,128,107,154,128,124,
+  82,141,145,136,128,144,115,123,  124,117,107,104,103,99,107,90,  92,98,102,82,88,134,161,126,  115,104,114,106,73,91,111,84,
+  128,93,94,94,94,116,91,103,  108,106,98,100,101,102,110,97,  137,133,120,132,146,114,136,131,  144,145,155,131,107,151,144,133,
+  128,103,106,106,106,128,103,115,  120,118,110,112,113,114,122,109,  157,138,128,117,136,123,136,128,  137,123,152,126,104,136,137,134,
+
+  32,94,46,28,98,102,130,97,  59,91,81,73,26,125,102,48,  100,66,105,57,87,121,71,120,  86,64,64,83,68,105,91,70,
+  32,130,98,103,64,30,123,122,  97,123,58,127,70,74,55,59,  53,99,95,99,105,52,48,73,  52,68,138,89,29,104,97,142,
+  20,99,54,114,87,86,108,55,  46,27,42,99,130,118,83,95,  138,115,69,67,96,95,125,86,  82,121,92,145,131,142,115,134,
+  27,53,57,104,139,102,124,124,  135,81,79,148,118,90,71,25,  151,83,146,71,74,129,109,35,  84,115,121,77,48,24,58,63,
+  29,55,85,51,70,123,21,95,  116,128,90,107,124,74,98,80,  132,125,112,114,118,109,167,103,  57,127,95,84,13,65,84,109,
+  27,137,123,69,151,81,84,95,  124,80,115,38,140,164,121,122,  86,116,83,102,117,110,111,92,  72,94,114,144,62,90,84,109,
+  31,128,63,37,14,138,126,79,  132,63,54,95,60,132,89,132,  25,77,134,94,93,154,147,125,  104,98,113,119,70,130,98,111,
+  24,78,61,30,100,41,65,115,  80,28,93,97,86,38,74,112,  85,105,135,113,132,45,65,30,  126,91,113,114,73,89,65,130,
+
+  21,150,114,47,106,125,121,37,  79,158,90,155,73,92,94,107,  87,137,97,135,91,139,152,102,  98,85,126,121,80,140,42,25,
+  23,47,75,42,64,150,67,93,  100,140,70,62,117,140,54,122,  100,98,95,105,88,100,132,41,  81,137,76,144,60,131,128,66,
+  23,59,107,144,98,44,20,119,  70,80,58,85,119,93,113,59,  83,28,117,67,135,65,77,60,  130,114,138,111,31,109,92,87,
+  18,93,79,109,57,128,94,69,  135,54,72,38,53,55,94,56,  133,98,153,147,82,58,124,55,  48,50,153,128,24,54,80,89,
+  29,130,95,117,49,47,46,91,  164,58,99,148,64,113,86,110,  144,134,49,60,102,62,103,107,  69,89,149,71,113,31,107,71,
+  25,90,106,113,119,82,75,91,  111,128,139,79,130,101,90,31,  97,103,87,84,53,138,43,87,  155,57,75,150,93,128,120,43,
+  24,118,82,52,79,56,143,92,  47,103,22,46,70,106,155,129,  93,104,125,42,96,124,137,93,  73,43,134,98,100,111,139,56,
+  22,65,103,76,38,122,121,155,  111,147,47,76,96,103,153,78,  71,103,137,104,150,89,139,159,  77,120,71,98,98,125,78,143,
+};
+
+static const uint8 ced_hires_1[1024] = {
+  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  62,124,71,116,117,77,46,71,  78,98,127,132,69,67,72,79,  34,37,88,95,81,88,135,138,  116,123,132,136,97,128,45,101,
+  68,71,36,0,24,19,3,1,  0,9,60,40,64,146,70,113,  20,16,20,26,13,20,9,12,  14,21,15,19,55,50,99,61,
+  17,127,133,119,128,134,113,109,  106,124,100,107,119,131,111,122,  138,113,136,154,132,125,120,114,  115,119,101,25,19,31,19,19,
+  122,112,48,123,115,89,133,132,  124,114,123,130,73,103,139,130,  106,90,103,114,71,97,92,74,  115,112,59,105,83,90,140,121,
+  118,113,51,133,112,43,123,112,  133,130,102,109,71,110,89,102,  128,125,104,139,115,139,129,134,  112,120,123,117,111,132,129,118,
+  73,151,165,130,127,146,100,107,  108,121,103,114,156,112,107,111,  125,129,150,135,107,155,118,116,  168,85,79,83,77,89,77,77,
+  98,101,81,63,71,82,66,64,  60,72,57,62,72,70,60,67,  75,153,138,154,129,161,146,125,  125,139,151,134,139,151,135,146,
+
+  59,21,36,18,26,37,21,19,  15,27,12,17,27,25,15,22,  30,34,38,44,31,38,27,30,  32,39,33,37,31,43,31,31,
+  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  128,127,128,124,144,128,127,126,  121,128,118,123,128,128,121,128,  128,128,128,128,128,128,128,128,  128,128,128,128,150,128,128,128,
+  90,112,109,86,83,124,57,54,  84,70,53,84,62,78,53,75,  124,134,116,95,118,85,129,65,  67,74,68,72,66,78,66,66,
+  141,134,118,107,135,127,130,134,  118,109,94,99,109,107,97,104,  112,116,120,126,113,120,109,112,  114,121,115,119,113,125,113,113,
+  161,134,128,112,120,139,115,128,  122,121,106,111,121,119,109,116,  124,128,128,128,125,128,121,124,  126,128,127,128,125,128,125,125,
+
+  76,29,158,87,47,122,133,89,  63,101,139,116,147,111,79,122,  60,112,93,68,42,145,73,100,  89,123,46,122,124,112,99,90,
+  118,56,99,61,145,53,131,144,  66,122,110,97,113,82,19,27,  33,134,120,84,79,124,78,108,  138,87,26,133,114,84,47,165,
+  62,148,104,137,139,97,125,86,  134,47,127,93,141,85,119,111,  126,56,42,26,102,89,122,48,  59,116,110,106,46,115,64,113,
+  107,81,49,29,28,64,130,22,  111,53,142,89,89,127,131,86,  122,122,37,143,66,96,82,105,  155,96,154,124,63,121,90,60,
+  115,107,42,30,134,130,119,108,  66,145,122,85,105,19,88,92,  36,134,34,101,97,112,152,73,  150,89,116,144,71,130,131,103,
+  88,123,77,30,132,53,98,51,  116,61,95,57,84,109,31,74,  20,98,135,114,78,114,95,123,  45,136,116,111,130,94,120,99,
+  114,121,64,107,94,104,29,77,  152,133,124,119,135,98,0,78,  118,115,88,102,57,61,51,89,  120,51,115,109,64,111,61,66,
+  110,135,106,110,98,74,86,137,  124,103,134,80,86,43,76,131,  134,105,91,84,73,34,71,133,  60,111,117,53,90,133,106,61,
+
+  147,38,145,28,137,149,110,112,  52,83,99,128,56,129,105,74,  93,60,45,62,73,86,30,82,  80,44,19,97,85,87,108,103,
+  99,114,94,104,97,104,140,51,  98,62,119,74,85,141,51,23,  71,58,119,110,151,65,116,74,  130,81,86,90,7,99,38,116,
+  61,59,140,66,91,133,123,152,  80,82,98,67,112,110,110,89,  121,132,120,62,43,74,45,110,  88,146,113,101,134,135,104,131,
+  110,47,96,67,123,147,69,70,  131,106,98,109,71,118,116,128,  92,100,149,53,144,117,99,136,  127,128,3,43,114,100,85,117,
+  92,134,156,8,88,62,89,98,  6,19,46,96,145,108,88,69,  78,94,152,96,146,108,133,66,  75,102,101,10,79,44,57,61,
+  102,96,123,91,109,149,84,63,  39,134,53,110,111,64,97,103,  155,133,105,105,34,125,93,56,  28,84,65,95,103,58,41,83,
+  48,25,116,120,91,104,124,84,  111,119,105,64,63,97,116,114,  128,103,136,74,59,33,47,93,  54,80,130,39,139,130,119,83,
+  140,68,94,85,77,79,118,109,  120,106,114,103,115,165,94,79,  60,52,84,77,82,78,49,74,  88,49,48,76,102,99,138,71,
+};
+
+static const uint8 ced_hires_2[1024] = {
+  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  44,106,18,108,117,63,65,73,  40,57,43,81,77,63,68,90,  54,16,20,95,85,131,112,100,  58,144,140,135,114,96,110,128,
+  111,93,68,129,135,62,95,32,  82,42,46,10,8,23,10,16,  26,21,70,26,93,84,168,105,  79,71,29,48,15,26,68,128,
+  28,126,105,123,109,126,102,109,  105,122,72,104,108,132,113,126,  118,82,122,110,129,125,114,134,  113,133,103,54,21,32,21,128,
+  113,122,134,118,123,77,97,101,  121,135,128,138,125,120,35,120,  31,24,148,129,0,0,0,0,  0,0,0,5,0,0,0,128,
+  137,137,112,130,105,127,110,121,  86,140,134,139,128,145,17,121,  19,27,60,150,106,62,95,0,  0,0,0,9,0,0,0,128,
+  86,81,78,87,82,79,82,75,  75,78,69,74,72,87,74,80,  90,85,84,72,92,96,88,82,  98,102,93,112,79,90,79,128,
+  162,135,144,153,151,137,127,118,  113,125,105,106,91,139,129,109,  124,140,83,71,91,95,87,81,  97,101,92,111,78,89,78,128,
+
+  40,35,34,43,36,33,36,29,  29,32,23,28,26,41,28,34,  44,39,38,26,46,50,42,36,  52,56,47,66,33,44,33,128,
+  128,128,128,131,128,128,130,128,  147,152,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  128,128,128,138,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  128,128,128,143,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  166,128,128,130,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,146,128,128,  128,128,128,128,148,128,128,128,
+  139,134,133,75,121,105,114,103,  64,68,129,100,62,76,63,69,  79,74,73,114,137,85,77,71,  174,140,82,101,68,79,68,128,
+  122,117,114,122,118,115,118,113,  112,117,105,110,108,123,110,116,  126,121,120,108,128,128,124,118,  128,128,128,128,115,126,115,128,
+  128,128,126,154,128,127,132,123,  137,126,117,122,120,128,122,128,  128,128,128,120,128,128,128,128,  128,128,128,128,127,128,127,128,
+
+  67,115,51,135,110,136,135,45,  132,133,58,84,156,90,77,88,  106,108,86,29,51,132,78,133,  122,67,145,151,77,36,57,128,
+  142,57,81,110,113,64,18,123,  101,136,98,45,102,79,83,102,  85,86,54,106,94,38,117,74,  115,80,89,151,109,98,137,128,
+  57,126,70,74,47,54,112,60,  146,46,72,36,108,101,48,40,  135,151,135,128,78,111,70,101,  107,100,25,118,65,57,106,128,
+  67,74,25,123,123,65,42,95,  135,62,82,86,45,70,104,59,  92,30,63,55,143,71,26,83,  97,107,87,111,25,86,45,128,
+  65,102,90,116,124,16,45,51,  110,129,118,128,73,120,108,139,  139,102,109,110,120,134,85,66,  104,100,96,129,134,94,84,128,
+  68,135,60,114,78,126,135,56,  22,121,35,117,101,118,119,120,  109,82,99,96,29,91,118,96,  63,104,18,139,47,66,138,128,
+  78,148,152,94,129,120,107,96,  141,71,52,46,111,50,97,36,  101,133,87,23,91,150,92,38,  110,103,97,84,79,68,85,128,
+  118,16,110,124,126,74,99,94,  132,79,88,135,103,84,161,153,  35,41,125,78,122,107,74,90,  93,122,134,117,97,91,38,128,
+
+  56,121,75,93,68,139,117,78,  23,63,43,75,128,101,116,59,  57,65,121,13,48,108,74,121,  144,120,128,78,95,135,119,128,
+  141,128,143,68,91,46,137,151,  29,50,32,92,55,73,81,107,  139,145,72,67,54,129,86,35,  105,40,40,99,98,61,142,128,
+  136,98,109,69,113,126,85,56,  56,103,117,104,17,70,148,128,  55,61,117,55,58,53,121,155,  86,126,75,104,27,105,100,128,
+  117,110,66,101,135,87,114,88,  69,93,78,106,98,98,117,132,  43,125,54,39,97,119,157,98,  79,131,147,134,59,130,143,128,
+  60,89,40,119,87,99,73,123,  138,107,135,72,78,97,83,134,  85,110,119,93,43,141,77,98,  119,124,28,104,55,131,118,128,
+  130,92,71,117,97,53,50,134,  114,136,136,98,51,71,62,115,  96,155,36,1,104,124,59,137,  117,93,26,47,129,107,82,128,
+  25,44,71,115,56,152,109,40,  117,70,60,119,146,80,83,130,  162,18,126,52,102,137,91,39,  105,113,91,105,37,46,95,128,
+  68,104,78,51,112,141,87,82,  79,68,59,77,84,49,21,91,  81,68,66,30,149,62,30,70,  81,114,25,107,6,89,62,128,
+};
+
+static const uint8 ced_hires_3[1024] = {
+  145,108,107,112,108,128,119,120,  123,130,108,110,133,138,120,117,  120,120,128,121,121,116,128,119,  128,119,122,114,121,119,123,124,
+  77,160,155,161,143,37,29,56,  47,35,127,103,75,130,115,110,  149,149,88,80,89,85,145,135,  102,93,107,99,91,89,140,135,
+  123,90,70,105,58,80,64,65,  54,64,75,114,51,62,65,58,  91,121,130,110,109,105,111,102,  107,100,93,91,94,94,90,92,
+  173,130,70,57,104,116,65,65,  135,145,70,76,167,129,117,91,  127,129,135,120,115,112,120,109,  117,108,170,131,84,86,91,132,
+  62,79,115,64,137,72,131,76,  108,65,120,128,125,126,92,124,  97,123,106,137,119,127,102,145,  120,143,106,114,81,121,87,140,
+  32,95,125,118,134,80,123,115,  112,107,111,122,99,125,101,137,  136,114,118,135,117,125,121,134,  142,145,120,115,95,109,84,135,
+  149,90,62,97,105,90,90,83,  100,81,70,84,80,69,84,99,  91,95,141,93,85,127,104,122,  139,91,91,84,114,85,87,82,
+  128,108,87,106,88,166,96,91,  95,101,96,78,98,95,111,100,  124,110,121,124,107,109,128,107,  106,107,102,86,93,92,87,95,
+
+  128,128,121,112,126,121,150,106,  125,124,120,96,136,130,135,110,  131,109,148,108,124,113,113,94,  107,139,145,102,114,94,96,95,
+  92,96,25,73,155,144,172,124,  82,104,144,100,98,115,154,121,  106,60,79,85,106,68,75,81,  150,81,74,87,105,46,52,81,
+  128,115,75,109,150,190,102,84,  108,126,118,117,98,102,82,70,  101,160,145,120,104,121,112,110,  124,103,124,102,101,99,97,98,
+  128,121,92,125,93,113,104,105,  94,104,93,99,93,102,105,98,  129,127,166,129,127,135,139,136,  142,143,142,131,131,127,123,126,
+  128,115,81,140,82,102,107,94,  83,93,83,84,80,91,94,87,  131,132,152,131,139,139,150,144,  142,131,134,133,139,135,137,136,
+  128,144,99,131,100,120,124,112,  101,111,100,102,98,109,112,105,  126,133,137,137,132,146,148,131,  144,153,139,142,147,123,115,120,
+  128,132,96,135,97,117,108,109,  98,108,97,99,95,106,109,108,  133,126,134,126,133,142,142,127,  140,133,144,121,134,129,135,117,
+  128,137,98,138,99,119,110,111,  100,110,99,102,97,112,111,111,  133,122,147,135,132,142,126,127,  144,133,138,140,129,124,135,131,
+
+  66,120,131,103,77,97,82,96,  49,122,50,79,108,111,157,61,  103,104,175,93,150,114,129,97,  163,62,107,74,85,83,86,65,
+  60,104,55,155,93,106,145,111,  154,124,81,79,114,119,104,98,  78,156,105,125,122,58,126,51,  158,67,68,140,110,97,170,83,
+  57,133,137,102,68,149,108,61,  123,146,85,82,59,65,109,45,  93,71,73,77,93,78,111,93,  84,127,90,164,148,138,91,161,
+  61,151,94,146,151,131,111,141,  77,115,111,98,134,96,86,120,  72,120,108,84,108,154,91,107,  59,119,71,81,97,99,110,111,
+  61,123,117,54,44,130,154,70,  125,120,23,146,110,91,116,88,  138,83,134,93,150,98,76,53,  90,99,140,60,102,90,84,118,
+  48,50,50,114,97,154,78,31,  88,101,72,126,31,104,99,105,  108,142,96,135,96,117,112,62,  90,103,116,59,144,151,100,63,
+  60,111,72,74,81,149,104,72,  154,140,90,149,106,117,107,164,  94,68,40,103,113,75,95,119,  78,119,86,84,153,133,135,61,
+  57,55,162,113,44,109,106,106,  156,45,51,86,91,122,91,49,  59,115,122,101,140,123,148,88,  128,112,113,99,106,160,94,146,
+
+  60,103,65,134,55,108,82,107,  109,74,62,66,51,60,154,112,  107,132,118,98,153,90,145,96,  127,103,138,131,32,128,125,71,
+  83,37,62,74,156,128,140,85,  67,149,49,158,98,78,96,58,  74,105,154,86,85,104,70,106,  71,139,144,112,70,87,62,57,
+  61,52,121,154,38,91,135,69,  50,69,82,123,119,105,113,111,  94,87,99,91,73,100,97,82,  110,100,136,69,126,142,55,51,
+  74,69,108,46,132,141,110,157,  133,69,143,44,100,133,144,154,  148,71,78,118,106,110,142,141,  77,145,112,124,118,87,69,36,
+  55,94,65,106,116,57,117,82,  111,82,114,127,58,134,115,141,  151,122,124,60,133,57,158,91,  139,85,137,75,157,84,128,56,
+  59,132,91,110,40,123,69,72,  156,84,95,148,88,151,121,95,  59,132,143,80,88,128,94,95,  76,120,121,149,98,86,94,91,
+  63,124,105,95,61,107,107,129,  65,159,114,147,62,170,118,134,  140,104,135,145,113,123,129,67,  107,117,135,114,64,66,85,147,
+  65,101,84,63,70,103,113,106,  138,80,159,36,114,114,86,133,  93,102,95,90,159,137,73,65,  75,126,75,50,154,87,101,119,
+};
+
+static const uint8 ced_hires_4[1024] = {
+  125,128,119,118,134,121,118,117,  121,123,122,116,128,124,115,120,  123,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,133,
+  89,125,49,76,12,21,28,18,  21,36,39,85,85,24,34,27,  45,91,39,57,98,41,58,61,  34,65,57,69,88,91,63,51,
+  88,107,83,89,76,130,128,123,  123,120,114,104,121,106,102,107,  111,122,121,124,104,124,105,123,  75,193,173,183,166,161,151,140,
+  57,123,96,99,90,98,90,87,  92,100,90,86,102,110,91,100,  111,104,117,127,115,104,104,107,  93,107,76,126,75,129,66,99,
+  132,140,43,135,109,85,142,143,  142,123,145,142,88,114,151,140,  111,113,116,119,86,108,108,99,  125,132,66,121,102,103,162,139,
+  127,140,45,144,105,38,131,121,  150,137,124,119,85,120,100,111,  132,147,116,143,129,149,144,158,  121,139,129,132,129,144,150,135,
+  118,171,152,134,113,134,101,109,  118,121,117,117,163,115,111,113,  122,144,155,132,114,158,126,133,  170,108,78,105,109,94,91,87,
+  107,128,102,85,100,86,92,85,  99,107,95,79,99,102,94,112,  103,175,150,158,143,171,161,149,  134,158,157,149,157,163,156,163,
+
+  109,134,92,99,79,88,99,96,  113,116,109,96,117,109,94,107,  109,117,140,123,112,109,118,129,  82,99,110,93,90,120,93,105,
+  174,130,49,65,96,52,55,98,  100,54,66,43,94,53,51,88,  117,70,89,96,67,74,69,96,  111,61,42,98,52,58,55,82,
+  104,127,95,100,95,101,95,103,  111,102,126,112,114,120,101,120,  111,128,128,120,116,125,119,128,  87,104,85,98,95,101,98,94,
+  127,145,120,137,126,116,135,123,  128,134,149,145,134,137,137,135,  138,172,148,150,144,142,146,155,  116,128,113,126,123,128,126,122,
+  127,149,116,132,127,126,132,129,  142,132,146,137,136,137,141,133,  138,152,166,158,154,155,150,158,  104,121,102,120,112,119,115,111,
+  147,147,113,127,121,115,115,109,  131,129,132,124,135,125,120,129,  126,150,153,163,145,131,145,157,  122,128,120,128,128,128,128,128,
+  132,146,128,118,123,118,136,128,  140,142,131,133,128,133,123,134,  135,144,155,144,129,141,146,152,  119,128,117,129,127,128,128,126,
+  136,161,124,119,112,127,127,117,  125,135,136,116,159,139,127,131,  135,144,151,151,142,140,146,158,  121,128,119,128,128,128,128,128,
+
+  56,117,131,52,109,89,49,75,  81,137,58,134,87,143,104,64,  67,158,68,113,116,101,108,155,  107,161,107,92,127,123,72,117,
+  86,78,40,98,54,51,93,94,  146,77,132,88,105,47,86,121,  62,41,132,68,70,140,41,104,  135,103,110,99,123,62,88,138,
+  101,108,91,108,121,139,137,42,  81,139,135,123,107,124,135,85,  101,65,99,90,94,85,106,115,  139,139,64,101,103,134,95,143,
+  70,126,127,75,75,101,140,147,  87,164,107,104,165,99,81,128,  127,71,73,92,129,93,149,99,  116,117,63,116,99,109,92,116,
+  76,80,61,56,75,111,73,103,  101,107,139,145,119,80,146,97,  80,85,92,153,99,90,106,97,  63,145,46,83,77,87,100,104,
+  146,115,124,137,168,42,104,127,  143,56,58,87,91,127,81,104,  84,80,65,78,61,37,98,132,  153,100,149,107,103,115,86,77,
+  130,158,89,75,83,106,85,85,  154,103,67,119,146,96,127,133,  57,111,108,142,164,78,117,83,  96,129,56,73,114,66,45,67,
+  144,99,131,129,98,141,102,131,  94,130,123,75,85,40,91,109,  73,161,102,81,53,102,161,124,  69,88,72,127,111,144,48,90,
+
+  63,71,74,140,135,125,89,112,  80,139,117,97,84,70,104,105,  134,91,113,99,91,128,126,100,  80,119,115,145,82,39,81,166,
+  114,92,92,84,94,122,49,104,  105,137,124,110,127,85,72,101,  21,83,117,52,78,79,102,69,  159,137,104,143,160,134,95,104,
+  121,128,79,153,100,154,100,103,  107,108,78,130,91,172,136,158,  103,65,53,145,51,100,86,107,  108,103,136,103,149,108,105,77,
+  79,64,42,108,78,77,129,65,  55,77,109,88,67,93,68,52,  53,94,114,121,109,121,106,60,  146,113,83,116,96,136,58,100,
+  84,132,50,139,86,86,148,141,  107,91,140,68,130,131,107,42,  102,135,166,160,54,62,91,126,  79,136,52,161,124,123,100,71,
+  70,99,52,78,44,116,54,121,  67,112,82,59,164,83,92,133,  108,81,120,156,91,84,129,131,  107,110,139,91,122,128,82,92,
+  118,77,72,119,66,133,131,40,  125,123,54,65,72,159,54,65,  84,86,68,142,68,101,155,79,  102,155,56,134,117,163,131,155,
+  66,82,99,78,48,100,154,96,  81,163,89,132,131,155,126,110,  54,98,82,94,51,152,112,120,  138,77,149,127,74,99,109,80,
+};
+
+static const uint8 ced_hires_5[1024] = {
+  128,127,124,128,128,147,138,139,  162,124,125,126,127,127,152,184,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+  59,56,52,84,74,62,105,50,  75,46,54,59,89,56,113,130,  119,132,110,115,136,127,123,76,  85,121,121,94,112,104,106,128,
+  126,117,109,74,77,123,124,119,  122,105,96,86,86,78,87,79,  94,169,180,158,166,149,129,116,  113,117,109,100,100,85,105,128,
+  59,98,77,92,84,100,75,88,  93,94,54,78,89,98,98,106,  103,65,103,95,118,105,93,113,  76,102,66,54,135,52,60,128,
+  114,126,138,119,130,83,102,112,  141,139,142,144,138,118,52,132,  48,39,161,146,26,1,0,0,  0,6,0,43,34,8,25,128,
+  137,140,115,130,111,132,114,131,  105,143,147,144,140,142,33,132,  35,41,72,166,126,73,105,0,  0,0,0,53,3,50,23,128,
+  112,128,104,79,120,147,178,100,  105,85,90,74,80,77,120,87,  130,112,180,83,120,100,91,85,  85,95,84,102,99,90,92,128,
+  162,138,147,153,157,142,131,128,  132,128,118,111,103,136,145,120,  140,154,95,87,111,106,97,91,  91,101,86,97,105,96,98,128,
+
+  96,79,91,123,95,99,109,128,  124,80,77,74,81,79,85,86,  101,94,91,83,107,102,93,87,  87,97,91,93,101,92,94,128,
+  130,76,66,68,64,65,77,111,  81,61,51,63,45,76,57,136,  63,56,53,46,77,64,62,49,  49,59,123,97,63,54,56,128,
+  86,86,81,86,91,131,96,85,  96,81,82,79,84,117,90,91,  106,99,96,88,112,107,98,92,  95,102,87,98,113,97,99,128,
+  114,112,109,114,116,119,125,113,  127,111,110,121,112,112,118,119,  128,127,124,116,128,128,126,120,  120,128,115,126,140,125,127,128,
+  103,101,98,103,105,118,105,102,  113,98,99,96,101,101,107,108,  147,116,113,105,128,124,115,109,  109,119,104,115,136,114,116,128,
+  121,119,116,121,123,142,128,128,  128,116,117,114,119,119,125,126,  128,128,128,123,128,128,128,127,  127,128,122,128,128,128,128,128,
+  118,116,113,118,120,127,150,122,  139,118,114,111,116,116,122,123,  128,128,128,120,128,128,128,124,  124,128,119,128,128,128,128,128,
+  127,118,115,120,139,136,123,124,  134,127,116,113,118,118,124,125,  128,128,128,122,128,128,128,126,  126,128,121,128,128,128,128,128,
+
+  144,113,79,134,119,150,168,110,  109,123,87,135,164,70,123,153,  60,128,94,93,119,96,90,82,  80,91,52,120,157,65,95,128,
+  152,77,95,138,152,63,122,108,  72,111,161,80,53,163,71,71,  154,47,85,88,86,124,118,33,  115,126,72,98,91,95,82,128,
+  128,146,140,137,59,123,98,75,  133,154,41,39,133,67,133,64,  115,98,30,37,84,76,58,26,  78,73,153,84,82,120,63,128,
+  106,93,89,71,142,126,122,109,  127,113,101,74,47,71,102,125,  99,87,54,87,115,144,165,81,  99,49,47,81,86,139,114,128,
+  95,47,91,59,105,149,149,123,  89,57,69,124,88,133,119,156,  141,64,149,165,107,86,69,108,  135,57,146,122,119,128,82,128,
+  44,75,63,157,128,38,92,158,  41,77,129,36,81,96,88,54,  44,93,58,46,115,99,80,131,  96,65,76,87,70,72,83,128,
+  165,111,57,90,104,46,83,56,  72,75,68,75,96,118,139,55,  81,118,70,56,99,104,83,106,  143,142,105,139,81,83,155,128,
+  86,127,119,93,103,124,116,150,  118,69,88,93,98,72,109,119,  122,130,119,64,126,86,111,57,  87,131,74,125,103,69,156,128,
+
+  98,68,93,126,62,132,161,123,  149,72,93,84,65,65,112,144,  101,154,60,90,129,83,167,144,  140,140,113,105,164,79,61,128,
+  112,113,139,157,96,91,153,68,  106,113,83,60,100,103,74,71,  111,102,68,110,41,73,115,67,  93,107,158,143,83,149,120,128,
+  95,117,116,117,99,81,87,114,  115,118,121,91,142,90,110,101,  82,114,128,148,135,125,105,70,  80,91,110,89,129,99,140,128,
+  85,169,69,133,73,49,110,60,  96,123,99,77,114,72,157,121,  140,133,145,94,106,142,90,139,  96,154,153,76,95,136,38,128,
+  144,125,98,54,162,77,113,95,  108,88,42,67,143,69,97,30,  68,71,136,64,127,135,138,72,  107,115,59,137,148,97,158,128,
+  122,156,149,122,108,71,79,147,  128,144,38,68,131,98,99,121,  164,143,123,52,140,126,130,91,  154,67,95,124,146,43,91,128,
+  124,62,79,123,127,91,91,135,  121,69,95,76,76,128,74,47,  52,71,100,47,105,141,145,88,  123,140,106,86,152,110,91,128,
+  123,121,138,108,94,76,62,115,  57,128,70,49,149,129,153,78,  71,120,117,115,74,80,143,95,  64,35,73,80,92,79,49,128,
+};
+
+static const uint8 ced_hires_6[1024] = {
+  131,89,116,77,83,97,102,136,  133,105,111,105,87,87,87,105,  108,118,117,124,140,105,110,104,  68,96,93,87,81,101,115,102,
+  17,136,152,147,132,68,100,48,  56,48,81,53,20,72,87,102,  143,155,79,74,108,84,137,154,  76,94,89,93,83,102,92,87,
+  66,79,73,41,49,115,104,62,  46,89,89,70,100,49,89,121,  91,76,85,75,125,76,65,130,  119,156,144,96,123,158,156,118,
+  20,106,67,42,24,96,44,57,  144,132,70,77,156,109,89,83,  121,135,126,114,134,111,112,128,  91,109,152,125,78,102,101,136,
+  43,96,88,68,104,16,25,125,  72,107,35,59,22,13,0,14,  89,145,128,73,85,131,89,165,  108,108,98,155,106,124,167,120,
+  128,81,99,82,73,82,85,63,  69,94,110,62,46,48,43,56,  155,167,150,139,160,127,125,138,  97,126,85,77,73,94,89,89,
+  67,136,152,100,83,91,124,114,  106,89,92,124,146,131,103,110,  120,137,118,86,122,92,116,123,  90,103,95,79,104,103,104,97,
+  122,74,119,104,124,104,96,98,  90,118,93,81,76,100,118,130,  143,123,120,167,152,126,138,151,  125,120,83,101,102,118,116,108,
+
+  79,88,103,83,60,34,70,38,  92,77,94,49,116,95,82,88,  43,144,139,131,133,98,98,121,  86,94,95,101,96,80,109,97,
+  88,98,105,102,82,110,51,104,  115,100,128,93,119,80,46,69,  110,117,74,90,120,121,93,128,  119,85,83,79,94,103,104,110,
+  89,75,115,84,121,92,114,102,  122,90,122,128,106,108,84,118,  107,133,113,126,141,123,115,158,  103,135,99,112,107,138,121,141,
+  94,79,115,97,115,104,113,108,  122,102,110,119,95,109,90,121,  119,127,118,122,154,126,117,158,  112,137,104,112,113,135,125,129,
+  121,83,109,77,84,83,62,98,  104,84,113,116,93,81,73,85,  118,113,99,96,123,96,108,131,  95,111,79,75,101,113,113,103,
+  140,66,86,79,79,90,80,94,  100,88,90,93,77,79,74,87,  118,152,103,106,142,108,110,143,  111,131,122,106,115,147,126,135,
+  166,67,85,78,78,89,79,93,  99,87,89,92,76,78,73,86,  130,127,102,124,155,105,121,151,  123,149,123,109,120,162,148,136,
+  154,75,95,88,88,99,89,103,  109,97,99,102,86,88,83,96,  149,130,112,108,140,112,120,145,  114,125,119,102,115,145,139,135,
+
+  31,150,134,141,48,120,36,27,  151,116,122,82,89,136,91,28,  119,68,44,150,145,40,107,55,  36,92,23,66,46,41,24,59,
+  41,45,12,0,141,58,83,30,  9,9,61,106,0,45,9,16,  14,23,66,152,39,5,42,56,  144,155,123,64,151,84,75,33,
+  107,62,98,81,89,126,87,70,  167,111,120,112,104,100,108,129,  72,124,176,74,96,72,84,77,  115,63,29,72,108,97,111,154,
+  85,144,123,65,80,95,99,66,  65,55,176,121,104,135,27,146,  62,44,158,114,101,106,105,126,  60,80,72,160,55,126,113,101,
+  36,0,87,20,72,44,8,0,  67,127,87,121,10,84,77,51,  34,21,68,32,57,64,38,24,  9,14,132,0,17,87,36,41,
+  67,104,95,43,73,154,82,100,  126,72,76,68,26,71,58,32,  24,45,36,21,55,166,135,135,  134,126,7,36,81,101,84,170,
+  75,82,59,49,43,129,38,169,  67,116,63,94,69,64,52,101,  133,117,129,120,66,52,28,69,  66,82,50,119,104,83,74,71,
+  47,131,113,126,57,142,116,109,  99,107,76,126,9,6,123,152,  96,136,103,126,142,67,90,109,  102,148,97,101,76,87,83,95,
+
+  49,8,75,129,53,138,158,150,  98,83,101,27,3,66,159,125,  128,114,158,134,119,126,153,139,  137,128,35,126,84,61,108,84,
+  34,0,18,21,24,22,93,33,  38,1,18,168,88,79,156,105,  149,47,24,56,50,69,76,64,  20,102,71,68,46,62,48,58,
+  30,53,84,128,83,135,48,120,  32,36,94,29,44,92,100,149,  138,79,46,24,153,61,30,49,  158,143,56,138,115,118,110,93,
+  74,93,20,31,73,44,42,111,  52,105,68,74,46,43,29,66,  45,67,37,7,76,88,82,101,  52,70,30,45,51,78,66,100,
+  54,32,43,64,72,117,82,71,  69,29,37,29,4,167,126,75,  26,161,51,143,53,30,104,121,  74,79,144,27,157,129,153,117,
+  55,53,53,87,24,19,80,75,  68,63,40,42,111,20,59,68,  72,114,50,26,138,29,70,86,  43,51,156,34,99,130,66,103,
+  53,0,46,31,71,79,87,57,  59,51,34,39,24,0,25,66,  1,47,140,45,144,92,34,50,  90,75,50,2,24,40,147,114,
+  63,150,105,113,95,108,52,72,  62,157,153,73,141,125,32,38,  104,104,25,64,137,147,31,105,  16,139,27,46,1,90,101,23,
+};
+
+static const uint8 ced_hires_7[1024] = {
+  120,127,110,107,125,119,124,105,  127,128,122,128,102,123,102,99,  129,154,113,122,107,119,106,109,  116,98,104,108,120,117,116,111,
+  91,83,50,46,97,86,122,78,  99,143,61,80,78,80,62,52,  17,15,56,62,51,91,64,93,  138,118,150,133,136,119,114,121,
+  173,162,166,159,148,141,205,123,  204,166,143,146,120,144,133,153,  134,173,138,140,103,106,108,117,  108,108,71,103,148,151,114,164,
+  86,147,121,122,141,130,130,109,  132,142,124,133,108,143,112,113,  130,102,125,146,122,120,113,107,  113,92,86,127,123,136,75,107,
+  137,141,85,124,139,117,115,121,  96,81,80,124,153,77,61,54,  173,171,86,199,83,61,48,51,  58,40,46,50,62,59,58,53,
+  145,149,144,119,151,127,147,125,  167,161,116,130,99,120,128,128,  128,103,131,112,106,147,108,130,  151,83,89,93,105,102,101,96,
+  129,134,99,103,101,147,132,118,  111,96,87,94,100,88,67,64,  93,74,78,87,72,84,86,74,  81,63,69,73,85,82,81,76,
+  125,123,130,99,124,109,144,109,  111,115,106,113,93,107,93,83,  92,101,134,122,117,122,132,121,  100,122,88,92,104,101,112,95,
+
+  117,129,112,84,103,101,127,90,  87,91,82,100,75,179,112,185,  110,132,107,115,106,185,103,103,  136,103,135,134,129,86,137,125,
+  139,147,101,95,125,132,138,134,  114,118,109,116,89,113,94,123,  98,96,100,109,94,106,93,96,  103,85,91,95,107,104,103,98,
+  159,165,129,148,166,141,172,150,  179,163,171,181,122,149,165,145,  138,128,145,149,124,146,132,127,  143,122,121,133,144,135,162,148,
+  171,165,133,151,163,146,162,148,  172,168,156,165,126,154,137,132,  143,140,143,155,135,156,140,142,  142,127,130,137,149,145,153,146,
+  132,133,96,93,127,105,110,92,  113,117,108,115,88,109,88,85,  94,169,171,176,140,166,165,125,  134,143,158,156,158,166,150,158,
+  169,154,126,147,134,128,175,148,  138,128,128,128,118,128,118,115,  124,125,128,128,123,128,122,125,  128,114,120,124,128,128,128,127,
+  160,159,125,129,155,136,156,153,  149,128,128,128,117,128,117,114,  123,124,128,128,122,128,121,124,  128,113,119,123,128,128,128,126,
+  168,147,128,128,128,139,128,143,  141,128,128,128,127,128,127,124,  128,128,128,128,128,128,128,128,  128,123,128,128,128,128,128,128,
+
+  26,71,23,0,18,145,93,129,  88,129,0,160,78,145,20,29,  21,64,41,28,155,90,68,73,  46,65,128,9,130,142,83,131,
+  59,84,81,102,49,55,8,132,  69,48,0,80,0,123,0,36,  37,33,0,62,129,121,64,162,  123,122,0,154,69,160,143,89,
+  155,85,24,119,133,127,86,100,  126,180,98,119,16,37,81,98,  82,71,113,46,31,50,30,33,  70,148,98,135,141,102,162,99,
+  94,74,90,98,122,40,42,20,  72,125,152,93,42,97,112,5,  103,130,75,82,21,65,120,149,  107,58,80,48,61,150,41,63,
+  118,82,161,115,70,0,55,0,  13,151,100,56,59,0,25,151,  102,102,94,0,133,95,68,98,  35,166,95,80,163,108,138,85,
+  47,27,46,3,18,108,113,138,  156,140,109,142,22,49,123,83,  98,107,78,74,32,109,58,60,  83,0,109,0,1,0,105,55,
+  71,49,44,31,49,30,38,141,  120,37,96,114,14,14,14,21,  22,111,109,31,70,103,7,100,  17,109,62,65,72,49,18,128,
+  40,154,151,137,126,105,104,61,  119,144,127,42,0,0,164,141,  135,126,95,119,107,112,29,0,  0,0,99,11,15,13,9,0,
+
+  108,133,116,95,53,149,107,100,  56,66,115,79,15,100,65,0,  1,0,7,140,111,132,101,0,  95,4,92,92,46,22,161,72,
+  58,38,113,14,88,36,11,97,  67,63,27,39,143,108,133,82,  135,0,52,60,72,93,90,149,  109,147,143,103,45,159,148,153,
+  132,115,1,88,45,51,72,56,  44,62,5,83,23,28,153,121,  167,37,147,101,0,82,26,76,  97,99,17,1,1,0,40,86,
+  83,46,30,15,41,21,26,83,  78,66,41,84,84,78,0,41,  50,103,63,122,73,69,0,61,  48,0,97,105,70,96,62,90,
+  85,84,93,6,54,125,46,123,  102,62,7,71,96,44,54,0,  54,4,166,155,31,144,107,0,  87,55,65,143,79,47,0,5,
+  163,95,121,171,161,163,63,142,  100,152,141,64,95,125,105,56,  0,89,99,83,64,43,95,89,  83,50,23,0,68,0,0,76,
+  92,65,42,71,88,88,176,120,  171,105,154,154,18,44,54,106,  112,92,134,139,0,134,133,130,  116,101,84,51,64,46,90,155,
+  152,122,117,113,19,86,30,51,  85,122,99,90,18,146,45,102,  58,13,43,75,104,119,101,0,  77,0,0,0,146,0,119,0,
+};
+
+static const uint8 ced_hires_8[1024] = {
+  130,127,155,168,146,193,173,178,  138,176,189,169,199,152,146,172,  128,127,128,122,128,128,109,122,  115,128,126,128,120,128,128,128,
+  142,160,113,159,112,138,145,87,  133,125,100,110,109,101,70,134,  52,81,111,42,63,48,9,22,  34,44,34,64,45,60,106,128,
+  98,91,146,105,104,134,59,53,  74,82,71,49,70,51,52,76,  77,69,81,64,94,79,51,64,  57,75,68,77,62,74,106,128,
+  90,132,91,134,101,123,97,99,  109,125,92,97,124,114,109,140,  123,84,137,120,149,126,95,134,  90,124,96,82,140,78,116,128,
+  68,69,88,133,99,128,124,111,  101,105,137,125,134,111,84,138,  137,75,81,64,111,89,60,64,  57,75,68,77,62,215,110,128,
+  111,181,158,179,148,178,161,155,  155,149,184,172,200,153,139,187,  156,138,159,137,153,154,108,115,  129,118,111,120,105,117,128,128,
+  91,101,111,146,104,81,82,70,  84,86,94,72,93,74,75,99,  100,92,104,87,117,102,74,87,  80,98,91,100,85,97,128,128,
+  137,111,137,155,136,166,170,156,  138,140,168,155,174,149,139,169,  119,111,123,106,128,121,93,106,  99,117,110,119,104,116,128,128,
+
+  127,121,128,160,166,147,149,173,  185,181,184,160,174,149,143,166,  164,138,146,126,151,136,178,115,  117,182,167,95,80,92,124,128,
+  113,114,134,159,120,170,172,178,  160,151,180,174,185,158,134,167,  122,114,126,109,128,124,181,188,  193,130,119,145,158,155,152,128,
+  155,164,146,164,148,177,155,147,  159,166,174,167,174,147,135,174,  105,97,186,168,122,107,79,92,  85,103,96,105,90,102,128,128,
+  161,164,146,176,146,180,170,159,  156,165,177,172,179,155,145,171,  111,103,162,180,176,113,104,98,  91,109,102,111,96,108,128,128,
+  187,159,158,190,172,180,181,159,  155,156,183,160,177,165,170,171,  148,172,125,108,128,141,95,108,  101,119,112,121,126,118,128,128,
+  128,128,120,133,128,128,128,121,  128,128,128,123,128,125,126,128,  128,128,128,128,128,128,125,128,  166,128,128,128,131,128,128,128,
+  128,128,119,128,128,128,128,122,  129,131,128,122,128,124,125,128,  128,128,128,128,128,128,124,128,  128,128,128,128,128,128,128,128,
+  128,128,128,154,128,128,133,128,  142,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,  128,128,128,128,128,128,128,128,
+
+  20,141,64,92,31,78,151,53,  145,20,53,48,65,154,102,114,  93,115,27,0,49,93,58,115,  144,65,153,88,149,88,17,128,
+  84,40,161,60,108,42,125,33,  139,68,83,54,43,81,132,82,  69,84,108,0,122,85,6,2,  53,70,102,0,36,34,41,128,
+  100,104,66,145,69,110,103,89,  76,65,98,98,84,63,121,133,  106,95,152,44,168,45,122,127,  129,38,117,149,53,137,128,128,
+  134,104,149,56,114,62,74,104,  71,45,50,137,132,122,110,44,  112,56,54,136,150,128,59,0,  0,81,103,3,0,0,33,128,
+  81,17,22,151,126,101,49,126,  22,36,76,149,81,76,40,83,  64,111,53,68,65,145,96,0,  110,0,87,0,39,98,99,128,
+  104,20,78,59,26,158,114,116,  105,158,62,77,97,99,149,60,  158,104,111,51,111,77,41,70,  35,124,61,140,87,83,139,128,
+  89,103,71,128,99,85,122,108,  62,50,67,61,120,64,44,79,  43,9,107,179,176,163,107,154,  141,87,110,163,2,37,115,128,
+  0,162,20,49,20,47,97,121,  95,83,96,88,115,58,47,97,  61,11,73,19,29,17,0,0,  0,148,82,94,111,89,40,128,
+
+  120,92,69,58,56,38,49,120,  38,153,95,56,69,140,92,44,  157,155,88,91,145,0,89,50,  109,97,0,66,0,18,97,128,
+  114,13,57,120,93,77,153,77,  152,135,99,80,88,24,69,60,  107,53,43,2,13,0,131,78,  141,49,121,0,119,153,135,128,
+  0,4,6,55,124,56,65,29,  21,40,149,41,100,137,17,39,  3,163,111,111,140,0,52,59,  72,113,76,118,116,81,92,128,
+  47,32,26,62,0,51,45,166,  142,38,168,49,140,20,96,145,  124,74,90,169,73,151,143,89,  64,116,26,75,20,162,143,128,
+  0,0,91,31,9,39,39,26,  90,29,42,35,53,27,112,56,  58,67,4,76,79,51,153,115,  122,50,137,120,19,114,123,128,
+  78,120,61,100,65,59,85,52,  72,57,93,54,61,10,72,78,  56,75,77,0,33,34,43,38,  0,0,14,44,0,44,24,128,
+  151,95,67,85,59,72,70,135,  53,60,105,54,54,25,160,139,  142,61,116,126,90,1,104,159,  139,115,141,100,41,62,100,128,
+  0,0,16,46,152,103,55,43,  56,47,60,148,157,92,133,148,  0,0,158,50,138,103,111,67,  150,157,47,12,78,21,146,128,
+};
+
+static const uint8 ced_hires_9[1024] = {
+  130,71,129,141,137,60,133,127,  113,78,80,127,94,127,117,51,  103,62,119,129,148,68,133,101,  38,42,158,125,112,190,163,124,
+  139,45,127,155,132,105,145,125,  136,78,127,128,145,136,141,61,  122,142,128,129,137,97,140,143,  1,3,145,120,131,129,135,62,
+  124,125,104,88,80,128,69,72,  68,127,36,103,135,92,101,131,  77,126,132,112,51,143,88,87,  94,151,72,95,120,110,80,161,
+  81,123,101,98,60,133,79,54,  52,155,28,125,46,72,67,101,  90,70,86,76,45,128,37,119,  86,129,81,86,102,49,41,88,
+  96,132,92,99,114,136,90,108,  121,130,15,114,132,93,122,129,  135,112,129,111,82,133,131,124,  119,129,105,108,112,39,105,147,
+  115,68,124,132,149,103,109,139,  127,72,144,126,137,143,146,76,  122,99,138,142,133,69,134,126,  32,11,131,145,107,148,126,61,
+  77,126,72,78,49,124,127,85,  75,135,43,82,121,79,83,143,  74,43,129,98,111,130,44,45,  87,106,101,92,138,51,52,89,
+  95,126,67,51,135,116,72,112,  51,131,24,106,133,97,111,143,  71,33,136,82,68,135,62,78,  22,49,63,84,114,36,104,82,
+
+  46,129,63,71,69,93,73,121,  119,140,58,72,114,107,136,142,  96,57,126,112,97,126,48,117,  55,54,76,101,134,53,81,140,
+  139,93,114,144,122,123,128,130,  146,120,146,137,129,138,134,101,  109,151,120,136,133,78,129,131,  0,6,154,131,126,122,151,58,
+  103,57,111,129,142,74,122,99,  101,56,40,136,131,124,138,107,  93,87,91,150,153,48,74,96,  29,52,125,155,90,119,156,95,
+  71,137,72,129,58,106,76,87,  65,135,10,79,127,82,98,139,  95,29,127,121,126,138,126,104,  58,30,84,95,111,27,55,91,
+  148,128,84,57,100,136,84,113,  74,138,28,114,116,81,111,129,  86,138,17,107,80,134,130,64,  160,116,80,94,104,93,101,84,
+  76,131,112,98,55,135,114,120,  76,132,40,106,110,135,121,131,  146,114,77,99,59,136,66,90,  106,140,89,78,129,75,105,92,
+  119,136,75,128,125,129,141,113,  81,139,14,124,57,64,124,132,  74,124,75,119,117,122,73,70,  119,145,101,93,142,120,105,136,
+  111,62,156,130,141,107,139,152,  124,103,148,125,129,141,131,71,  138,116,130,137,129,82,139,148,  0,0,125,133,142,126,134,55,
+
+  72,116,48,77,48,122,93,65,  57,115,22,108,127,51,83,142,  87,100,151,86,87,131,39,43,  86,113,70,90,98,53,85,72,
+  149,20,138,146,143,112,62,118,  158,65,125,130,127,150,129,33,  114,116,113,124,140,58,141,141,  24,20,165,129,84,163,122,98,
+  106,140,115,104,111,132,132,124,  123,125,23,112,87,134,120,128,  118,121,93,124,114,141,143,116,  117,135,106,137,107,90,103,86,
+  104,114,115,105,103,116,131,80,  117,119,30,147,132,120,108,122,  147,136,106,132,147,127,81,123,  134,114,71,127,123,66,120,147,
+  106,131,95,77,65,135,100,63,  69,133,8,113,86,81,116,133,  74,114,131,122,67,128,53,135,  151,132,60,61,111,92,110,88,
+  161,88,152,116,143,105,115,146,  136,61,114,137,128,126,120,44,  151,90,132,138,125,54,155,126,  8,28,138,144,121,160,160,75,
+  87,124,98,82,154,145,88,92,  46,138,32,132,81,107,138,90,  81,34,84,103,27,129,104,81,  109,61,73,49,116,47,121,91,
+  72,136,72,95,97,132,49,95,  111,125,35,117,129,103,124,133,  112,95,117,134,95,115,67,97,  102,148,126,135,108,97,99,123,
+
+  165,12,134,132,98,115,126,113,  92,122,36,138,34,110,157,63,  107,139,40,148,113,22,79,71,  31,22,127,133,78,127,95,90,
+  38,12,112,64,133,138,43,108,  175,16,168,125,147,148,91,8,  100,54,87,106,121,17,106,130,  17,47,101,161,65,95,131,83,
+  119,144,110,98,143,110,64,120,  43,121,32,102,111,138,133,123,  86,119,120,80,52,126,109,143,  98,130,90,84,85,38,91,148,
+  95,124,58,99,38,144,76,50,  77,139,32,143,125,91,120,110,  106,28,108,77,132,126,36,92,  136,57,38,62,101,62,40,90,
+  47,23,65,73,87,20,162,94,  103,127,100,162,133,118,116,37,  117,60,94,98,151,51,49,98,  24,20,116,121,117,51,42,90,
+  54,123,33,45,31,151,56,33,  42,145,36,29,25,35,119,81,  32,43,48,22,12,118,44,43,  149,32,38,50,76,65,37,95,
+  46,130,41,34,22,145,59,49,  59,134,20,125,111,79,130,70,  69,37,81,47,140,127,73,75,  123,43,61,147,80,46,72,78,
+  115,32,81,74,58,149,102,67,  71,46,65,60,49,57,61,30,  68,174,54,51,45,52,72,53,  57,53,71,79,105,84,67,123,
+};
+
+static const uint8 ced_hires_10[1024] = {
+  58,122,143,128,136,100,142,138,  75,129,136,141,139,140,49,124,  126,129,133,99,142,134,152,137,  120,138,45,14,25,143,136,140,
+  125,108,87,90,85,127,88,79,  128,11,95,136,93,104,132,81,  132,106,62,140,86,72,90,83,  121,114,149,146,89,121,124,118,
+  133,75,100,96,92,129,79,127,  127,44,117,130,100,119,132,110,  114,131,93,115,52,110,92,107,  134,108,137,144,103,119,70,100,
+  129,70,93,99,140,110,67,56,  133,40,103,132,85,100,142,72,  134,84,77,132,70,53,62,94,  79,31,106,60,43,131,68,32,
+  132,100,125,106,118,135,136,106,  130,30,113,134,97,123,128,124,  125,111,84,130,80,106,89,109,  111,56,133,127,111,123,95,111,
+  67,126,128,140,149,99,132,133,  69,144,128,139,143,147,68,121,  136,137,133,67,119,131,136,128,  136,149,79,10,23,96,114,96,
+  125,102,73,98,152,143,106,80,  137,19,128,84,119,136,89,83,  74,107,89,132,74,58,81,126,  52,49,110,73,108,116,88,42,
+  142,113,137,119,142,109,104,86,  122,38,105,107,137,130,124,108,  116,84,69,126,79,79,97,100,  101,51,133,133,97,103,115,109,
+
+  96,116,125,136,124,122,129,150,  119,148,137,131,138,134,97,111,  120,136,131,74,132,148,147,147,  126,125,57,11,16,101,139,150,
+  64,113,99,105,134,74,84,122,  61,63,130,131,122,139,112,93,  90,147,154,53,127,109,126,145,  147,109,83,59,31,99,68,83,
+  137,90,110,94,59,103,126,88,  132,21,84,127,93,105,137,97,  124,122,120,138,85,68,128,65,  88,44,132,42,58,106,79,34,
+  124,85,81,119,108,133,128,91,  138,36,114,116,86,113,128,87,  26,102,87,132,98,77,76,98,  95,93,108,117,161,118,149,141,
+  131,125,92,102,68,135,80,85,  131,41,104,110,136,120,132,145,  74,100,62,135,119,73,95,106,  84,72,119,134,102,130,74,113,
+  136,71,72,117,124,128,73,106,  138,23,122,64,52,124,130,76,  75,120,120,120,127,85,126,105,  103,120,93,146,119,110,119,124,
+  59,156,145,150,141,110,137,123,  104,147,127,129,138,125,77,138,  130,135,130,87,148,123,129,136,  129,131,48,15,26,124,110,114,
+  115,51,73,57,54,120,36,57,  115,11,101,125,70,76,140,93,  150,94,84,126,85,49,83,103,  85,56,128,111,78,103,71,102,
+
+  137,112,116,124,112,131,132,116,  126,33,113,86,134,122,127,115,  94,124,117,137,127,116,110,91,  128,86,117,135,114,119,99,120,
+  116,104,122,83,102,118,81,68,  114,12,145,131,124,108,121,145,  103,127,145,127,115,112,108,119,  140,58,140,108,123,116,98,134,
+  131,98,133,63,58,136,35,61,  131,29,113,81,78,117,131,74,  128,121,76,127,101,71,79,105,  70,83,126,131,150,127,103,111,
+  95,145,124,149,145,102,158,144,  73,115,137,128,126,118,39,154,  127,136,124,85,112,136,102,157,  149,155,63,39,21,131,157,85,
+  121,75,41,91,52,123,59,122,  133,55,87,113,80,75,144,80,  121,95,110,126,123,72,70,79,  101,39,94,99,84,136,68,51,
+  127,76,124,126,75,95,60,87,  130,36,76,111,110,138,142,109,  125,113,92,127,77,120,87,88,  92,44,116,59,67,146,52,55,
+  124,94,127,65,59,135,31,83,  154,42,119,47,70,56,99,85,  70,79,69,127,80,57,107,42,  77,38,108,135,113,116,94,95,
+  134,58,90,56,42,142,76,51,  134,21,125,102,78,135,80,81,  78,60,137,122,61,53,43,84,  147,45,82,70,120,107,56,52,
+
+  125,58,101,70,50,142,31,51,  142,44,142,128,93,121,116,115,  115,88,119,127,79,67,100,81,  78,68,109,74,131,115,109,34,
+  128,20,41,24,44,151,46,58,  143,31,53,37,38,115,104,35,  49,13,50,110,40,29,34,32,  52,77,75,38,140,67,53,62,
+  30,102,98,113,102,138,130,139,  39,47,119,112,103,104,37,128,  128,103,110,33,80,102,106,125,  86,137,98,43,38,91,95,175,
+  21,105,132,102,123,141,110,109,  31,168,125,139,150,92,18,94,  86,105,122,38,63,172,77,132,  155,93,48,68,35,60,65,79,
+  28,139,76,117,98,118,79,120,  117,45,140,50,110,154,88,110,  45,147,116,36,127,99,132,98,  137,124,54,34,56,82,164,143,
+  35,74,103,109,98,33,86,125,  55,113,149,141,121,121,50,122,  113,105,152,60,154,113,84,61,  128,49,72,16,20,144,58,61,
+  82,126,110,148,135,61,143,155,  65,78,116,108,135,109,46,91,  122,125,145,67,103,116,124,162,  127,185,84,36,46,110,131,52,
+  42,137,146,121,136,110,144,167,  67,120,128,125,146,130,34,118,  111,122,138,47,72,152,133,119,  127,160,56,29,36,97,143,120,
+};
+
+static const uint8 ced_hires_11[1024] = {
+  129,46,102,93,87,61,93,55,  96,48,77,76,65,67,91,46,  64,58,113,75,57,62,82,91,  106,70,112,123,68,70,75,108,
+  113,55,138,144,134,87,155,40,  138,123,135,133,127,142,137,53,  147,126,126,136,132,133,154,124,  123,43,142,128,28,86,28,104,
+  122,134,145,124,106,123,46,119,  48,128,47,146,45,45,47,127,  41,140,89,43,38,100,46,40,  62,104,61,104,126,110,126,124,
+  91,125,26,136,96,129,23,115,  19,144,133,118,113,115,63,132,  59,137,94,51,29,117,36,127,  39,136,46,74,118,107,102,110,
+  89,128,72,40,74,135,21,135,  34,144,57,55,66,55,51,129,  53,127,93,50,32,126,63,29,  37,128,44,72,113,117,128,118,
+  77,69,127,126,130,77,123,31,  130,136,133,130,123,130,153,82,  138,132,127,134,127,137,125,128,  123,115,114,116,75,59,43,117,
+  113,116,62,47,39,142,82,137,  41,119,39,88,88,58,47,136,  28,38,72,11,27,108,35,44,  65,155,73,100,130,105,132,111,
+  93,34,108,139,100,55,67,50,  146,50,129,131,153,145,111,31,  105,138,137,151,137,51,116,127,  146,30,74,76,15,37,10,59,
+
+  98,133,33,28,17,143,30,136,  41,115,34,100,131,73,38,123,  12,134,69,24,4,134,39,29,  52,118,53,81,110,140,124,88,
+  82,128,133,117,145,106,126,85,  134,68,159,123,131,130,125,127,  111,117,130,146,125,44,111,141,  127,118,87,70,96,9,66,50,
+  92,145,86,40,68,123,37,120,  71,114,90,126,74,74,28,128,  73,120,89,54,119,118,68,72,  51,112,35,63,137,116,114,104,
+  98,123,104,91,72,135,54,139,  112,130,98,141,114,67,63,136,  86,42,87,61,112,117,101,58,  106,134,41,69,123,123,123,117,
+  85,137,150,37,54,142,42,126,  36,124,60,30,130,98,12,132,  135,24,89,53,49,98,125,57,  119,105,38,66,134,116,118,107,
+  98,141,79,89,141,133,98,129,  133,130,72,83,48,92,48,131,  38,31,95,107,139,113,59,69,  40,143,37,65,112,132,107,111,
+  112,118,69,50,46,149,49,152,  39,130,45,42,40,30,66,118,  54,26,75,14,107,112,49,54,  65,116,73,100,115,143,78,91,
+  91,69,145,144,140,88,104,98,  127,131,122,136,136,132,115,76,  134,134,127,139,121,152,149,142,  144,83,166,167,31,77,39,99,
+
+  82,130,53,59,64,129,54,110,  44,118,48,133,87,91,27,140,  84,148,91,51,122,101,61,86,  50,134,46,84,126,106,112,112,
+  84,135,121,143,124,127,87,121,  113,142,112,75,123,109,90,133,  90,98,88,84,123,123,125,129,  92,141,44,70,118,108,114,104,
+  134,108,127,101,121,121,98,125,  121,115,125,94,128,88,98,112,  124,82,126,112,130,134,126,138,  101,122,105,119,92,85,86,126,
+  77,117,116,48,78,138,27,142,  122,120,126,63,134,48,25,112,  112,39,86,120,154,143,120,143,  28,123,36,63,109,110,108,101,
+  71,131,58,45,30,128,122,146,  62,130,49,95,90,25,18,143,  48,127,97,99,74,119,33,43,  22,146,30,57,118,112,106,106,
+  95,104,131,129,128,72,131,81,  132,75,120,142,143,138,133,92,  155,132,138,149,139,63,128,144,  160,88,119,81,75,7,47,66,
+  102,128,48,47,57,119,70,107,  140,130,80,106,65,90,32,136,  116,123,82,61,117,132,66,43,  55,154,61,89,127,128,105,122,
+  110,129,58,13,38,138,30,112,  136,126,30,98,89,135,39,126,  55,147,70,32,117,122,68,87,  58,145,57,84,130,114,103,117,
+
+  118,115,58,40,45,141,61,156,  51,109,30,30,37,33,59,110,  34,25,81,42,99,147,71,54,  97,128,88,115,120,107,128,103,
+  102,62,107,133,115,63,64,42,  135,82,88,103,135,156,108,60,  147,137,139,149,144,30,125,110,  68,80,128,101,53,14,38,71,
+  127,100,145,84,145,70,134,37,  72,39,175,114,120,112,75,120,  76,99,111,119,117,48,91,106,  110,125,102,128,84,56,63,90,
+  129,77,130,151,138,88,126,88,  117,82,96,136,127,108,124,80,  155,101,115,124,94,99,175,159,  129,107,129,129,106,107,112,114,
+  98,9,84,76,73,44,76,33,  73,22,61,73,76,66,62,27,  84,63,137,71,74,28,80,84,  106,47,83,88,29,18,15,66,
+  102,23,80,68,70,48,77,40,  82,24,57,72,77,67,80,46,  70,66,135,70,62,10,75,87,  108,64,76,91,16,23,22,74,
+  105,19,61,52,70,26,63,47,  80,27,40,55,73,79,73,33,  71,69,127,73,72,12,72,69,  65,55,77,93,18,43,34,72,
+  104,44,77,65,69,48,79,52,  74,47,58,58,60,52,78,46,  61,55,123,55,55,61,74,74,  89,67,93,106,71,67,69,73,
+};
+
+static const uint8 ced_hires_12[1024] = {
+  79,129,118,114,118,130,135,136,  122,129,124,105,133,117,122,111,  134,119,65,123,122,86,117,114,  121,123,136,124,126,111,73,121,
+  134,116,126,128,129,122,119,131,  131,128,98,130,123,67,131,120,  126,125,137,42,120,129,124,126,  132,124,121,124,126,120,89,121,
+  112,133,109,133,111,132,130,71,  99,133,71,58,135,135,120,135,  124,96,123,110,95,72,54,65,  130,119,105,127,127,103,105,127,
+  118,122,128,106,129,130,78,100,  80,135,92,115,113,114,117,119,  117,117,148,147,127,77,100,118,  129,121,108,127,127,101,91,127,
+  135,132,136,125,118,124,139,134,  122,120,34,135,116,109,141,104,  122,131,129,5,132,82,135,130,  120,130,119,128,123,119,103,118,
+  126,133,130,135,119,105,128,126,  122,109,135,125,129,124,128,144,  130,124,127,128,125,138,129,123,  129,123,141,114,116,100,75,111,
+  126,110,99,100,152,132,105,74,  73,128,77,140,108,97,135,114,  115,76,102,86,74,55,93,128,  128,65,82,127,127,109,97,127,
+  83,137,114,142,118,131,144,118,  104,132,68,106,127,101,119,118,  124,123,62,94,123,122,117,132,  123,130,118,127,145,96,99,127,
+
+  115,129,121,73,117,134,107,125,  107,139,57,129,133,75,111,122,  115,108,124,112,115,65,89,120,  136,89,100,127,127,107,91,162,
+  116,124,116,129,125,122,126,120,  126,115,136,128,123,153,120,132,  126,124,119,124,121,125,124,127,  124,129,124,115,117,98,87,112,
+  110,93,120,88,101,100,92,100,  101,103,193,110,109,83,98,97,  81,85,93,129,84,120,83,96,  93,94,86,127,127,127,127,127,
+  130,129,102,125,118,128,139,116,  89,126,144,112,143,114,130,139,  137,125,99,129,122,105,77,70,  122,122,127,127,144,123,84,127,
+  140,124,123,126,138,124,114,136,  125,129,136,130,115,106,135,110,  119,120,124,115,130,112,127,128,  111,122,116,121,123,113,80,118,
+  109,106,107,107,110,111,111,99,  109,127,112,107,106,162,102,138,  105,111,114,99,121,103,86,90,  93,107,100,127,127,170,152,127,
+  129,122,126,129,130,127,131,138,  125,126,108,134,116,77,116,126,  127,138,131,43,123,110,137,129,  121,138,123,121,123,110,70,118,
+  108,114,111,100,106,124,104,100,  108,110,114,106,106,96,119,162,  90,141,96,108,93,105,112,107,  100,92,93,127,127,144,144,127,
+
+  119,102,134,125,132,132,111,116,  139,135,118,123,93,104,109,107,  109,137,126,109,126,113,120,127,  111,127,122,127,127,96,84,127,
+  108,121,133,121,111,125,73,107,  148,128,119,114,117,108,124,100,  119,106,116,170,155,38,84,144,  123,58,113,127,127,105,99,154,
+  57,128,109,135,124,126,134,42,  110,124,72,117,140,133,125,114,  124,133,85,97,99,143,140,113,  129,122,120,127,127,99,87,127,
+  93,88,77,107,97,101,91,104,  98,96,155,103,106,69,77,79,  81,137,97,206,81,184,134,112,  92,116,64,127,127,127,127,127,
+  118,85,115,94,120,133,99,119,  127,125,107,92,120,53,74,109,  133,143,137,92,89,114,118,130,  145,133,124,127,127,100,88,127,
+  93,97,96,121,91,124,100,109,  94,125,112,118,104,82,100,96,  123,97,112,138,108,200,107,142,  135,117,94,127,127,127,127,127,
+  130,129,119,115,125,130,53,116,  116,133,51,40,120,97,122,101,  108,62,141,108,137,124,103,105,  138,40,104,127,127,109,97,127,
+  111,131,74,122,129,132,85,105,  144,128,63,62,121,83,101,114,  133,139,115,123,125,105,139,99,  136,139,110,127,127,106,92,127,
+
+  132,119,131,119,128,131,116,120,  132,131,135,136,87,90,112,113,  116,132,115,124,125,154,127,132,  94,130,122,127,127,108,106,127,
+  127,129,109,98,125,122,112,115,  115,128,120,126,142,114,126,103,  132,96,127,105,124,69,102,122,  128,103,120,127,127,98,112,127,
+  121,120,145,97,119,126,115,132,  89,127,134,127,112,113,133,129,  130,102,128,127,135,64,118,140,  138,124,104,127,127,129,118,127,
+  127,127,127,127,127,122,127,127,  127,122,127,127,127,127,127,127,  127,127,127,127,127,127,127,127,  127,127,127,127,127,127,127,127,
+  127,127,127,127,127,118,127,127,  127,118,127,127,127,127,127,127,  145,127,127,127,127,127,127,127,  127,127,127,127,127,127,127,127,
+  121,124,104,109,109,81,110,97,  104,77,120,119,111,92,119,106,  100,107,107,127,117,127,112,104,  89,124,89,127,127,197,184,127,
+  115,99,101,103,110,78,92,110,  84,142,93,113,91,65,122,79,  105,120,109,104,120,113,97,90,  86,115,97,127,127,182,217,127,
+  127,127,127,127,127,122,127,127,  127,124,127,127,127,127,127,127,  127,127,127,127,127,127,127,127,  127,127,127,127,127,127,127,127,
+};
+
+static const uint8 ced_hires_13[1024] = {
+  79,36,84,88,70,26,101,101,  57,93,76,44,82,95,88,77,  79,102,93,112,76,86,62,31,  126,102,50,102,97,110,99,131,
+  52,43,113,117,112,51,119,123,  107,86,124,118,123,119,119,49,  114,113,112,125,118,105,132,18,  125,60,129,75,77,71,51,90,
+  99,65,124,123,98,65,96,121,  107,100,76,93,112,140,131,73,  92,127,90,86,132,109,81,54,  57,124,94,100,98,115,106,112,
+  81,31,42,63,61,107,26,43,  32,76,44,28,57,75,56,165,  61,40,75,79,77,82,47,21,  39,68,76,80,81,88,78,107,
+  53,55,83,115,118,88,124,129,  135,116,109,113,126,117,121,71,  103,69,120,116,124,140,113,25,  116,121,78,69,55,61,52,95,
+  75,63,96,27,126,105,92,121,  78,64,55,123,122,101,117,34,  107,33,121,105,112,47,100,18,  60,36,113,61,64,87,64,99,
+  70,75,100,78,116,77,119,135,  81,90,103,129,128,106,113,57,  91,57,132,124,117,87,117,49,  60,46,37,98,75,96,77,107,
+  104,148,95,50,41,136,45,51,  72,150,86,83,112,103,30,141,  59,39,43,55,90,129,56,44,  57,84,26,104,80,110,75,126,
+
+  72,54,96,117,102,69,79,123,  44,79,60,58,105,135,103,47,  69,152,129,129,112,72,124,24,  118,59,89,72,68,75,69,97,
+  49,104,110,125,122,126,125,119,  116,94,112,92,115,120,104,110,  129,129,113,124,117,103,117,22,  126,38,90,48,80,82,62,97,
+  76,70,64,110,64,64,46,62,  48,63,80,67,84,137,129,30,  93,115,59,121,142,110,109,61,  115,61,70,84,90,95,95,95,
+  100,63,73,65,82,99,97,66,  73,125,80,83,138,86,127,52,  78,109,102,98,105,78,103,62,  112,76,108,111,109,108,113,111,
+  111,114,86,93,91,76,81,78,  85,82,114,98,92,112,111,94,  98,94,75,90,95,93,90,72,  91,82,103,151,113,118,141,119,
+  49,138,104,130,121,72,113,92,  114,38,101,119,105,122,105,109,  107,112,87,116,119,68,115,8,  83,24,113,61,68,71,56,94,
+  114,52,76,101,85,55,85,54,  85,94,75,85,117,114,135,41,  81,88,76,85,140,64,70,51,  76,101,82,105,98,101,102,103,
+  114,111,126,117,134,117,113,90,  68,92,84,110,117,110,122,91,  91,164,90,122,124,100,108,120,  102,94,88,118,111,108,113,111,
+
+  101,137,103,32,71,95,111,96,  97,145,123,85,103,96,79,61,  52,54,86,108,68,157,102,41,  69,71,61,101,99,107,96,119,
+  73,150,20,28,25,114,26,26,  37,112,46,33,31,22,13,156,  23,49,1,30,13,101,28,20,  32,47,30,71,90,77,68,96,
+  105,79,95,84,96,64,96,113,  63,97,73,68,104,114,113,65,  121,104,111,119,80,88,85,63,  121,61,65,125,108,129,103,127,
+  42,53,109,91,120,40,90,107,  68,92,130,92,111,108,134,29,  112,80,111,94,96,54,105,150,  132,48,101,55,61,61,49,94,
+  106,68,93,88,93,69,69,61,  73,98,117,78,128,133,126,94,  114,78,82,97,141,98,102,65,  75,66,104,102,89,122,99,112,
+  107,83,114,50,97,149,84,90,  128,149,119,100,113,91,105,79,  132,52,98,96,108,142,94,37,  91,66,96,111,86,94,85,107,
+  46,66,107,104,113,66,126,117,  117,92,123,126,115,109,116,65,  119,42,129,113,104,53,118,79,  78,127,128,50,61,72,51,99,
+  121,71,63,62,64,61,69,65,  74,70,74,66,65,54,63,58,  71,84,44,54,50,87,74,66,  113,81,60,152,130,138,134,138,
+
+  84,83,129,39,123,95,106,128,  82,84,142,138,113,108,107,57,  118,38,127,106,107,90,120,60,  59,142,45,75,98,85,68,107,
+  105,90,94,88,71,76,71,67,  72,101,88,64,81,91,95,66,  85,76,67,85,79,82,82,58,  109,96,81,127,113,121,114,123,
+  73,100,141,107,128,71,80,84,  100,85,121,103,123,126,115,69,  119,72,94,128,109,59,112,26,  83,59,116,115,83,89,73,94,
+  104,73,90,91,87,88,89,78,  90,76,102,101,111,102,89,43,  82,81,118,96,146,75,100,50,  87,96,96,113,113,106,109,114,
+  66,58,131,124,106,86,118,114,  132,74,68,119,117,117,117,35,  102,36,125,106,106,24,98,35,  71,128,128,81,81,80,47,96,
+  73,41,118,138,109,28,78,48,  115,45,110,113,95,133,88,35,  121,38,106,109,103,103,117,18,  44,86,119,71,77,86,76,105,
+  94,137,71,54,52,142,72,61,  72,154,145,98,92,97,38,93,  58,58,95,69,85,132,127,42,  91,123,55,108,102,106,97,140,
+  113,65,76,58,62,92,85,62,  80,69,57,64,67,66,51,59,  80,72,50,69,62,68,58,73,  71,60,58,101,115,108,109,118,
+};
+
+static const uint8 ced_hires_14[1024] = {
+  127,73,147,96,119,88,105,109,  120,113,103,118,88,140,101,79,  110,127,68,112,151,94,90,83,  125,83,125,127,127,127,127,127,
+  88,49,130,116,118,52,139,140,  125,89,134,128,134,134,134,59,  124,140,131,139,131,105,148,19,  145,52,137,100,89,91,77,99,
+  131,71,141,122,104,66,116,138,  125,103,87,103,122,155,146,83,  102,154,109,100,144,109,97,55,  77,116,102,118,110,122,130,121,
+  134,25,93,100,106,94,68,101,  54,113,93,18,111,141,103,68,  142,68,136,117,127,146,99,18,  62,18,121,110,88,106,105,109,
+  87,60,100,113,123,88,143,145,  153,119,119,122,136,131,136,80,  113,95,138,130,136,140,128,26,  136,113,85,93,67,79,77,103,
+  127,99,154,57,97,71,91,93,  125,69,89,116,117,76,110,81,  149,113,85,77,84,90,75,68,  110,72,120,127,127,120,127,123,
+  107,127,106,67,34,132,96,32,  46,138,56,96,38,110,100,88,  74,63,26,39,49,130,29,120,  45,34,49,93,90,113,96,118,
+  140,153,112,49,46,136,65,67,  90,152,96,92,122,118,45,150,  69,64,61,68,102,129,71,44,  77,75,33,128,92,127,99,134,
+
+  74,133,89,64,55,143,71,52,  72,150,122,145,125,84,137,124,  106,53,105,97,125,131,90,27,  24,49,71,72,81,88,67,109,
+  83,109,127,124,128,126,144,135,  134,97,122,101,125,134,119,119,  139,155,131,138,129,102,132,22,  146,29,97,71,91,102,87,105,
+  88,4,133,137,142,3,28,140,  12,31,52,141,80,31,31,20,  149,46,21,114,133,20,2,90,  69,0,136,73,62,95,79,102,
+  126,68,90,64,87,99,117,82,  91,127,90,92,147,101,142,60,  88,136,120,111,116,77,118,62,  130,67,116,128,121,114,137,119,
+  79,15,89,109,132,0,34,27,  136,25,159,134,135,122,132,40,  105,35,121,133,141,23,146,2,  28,0,125,76,68,108,72,91,
+  84,143,121,128,126,72,133,108,  132,41,111,128,115,136,120,118,  117,138,105,130,131,68,130,9,  103,16,120,86,80,91,80,101,
+  144,57,93,100,91,55,105,70,  103,96,85,94,127,129,150,50,  91,114,94,98,152,63,85,51,  96,92,89,120,109,108,123,111,
+  121,154,45,60,63,64,68,39,  59,85,56,120,31,120,40,125,  43,86,21,83,136,120,56,36,  80,36,43,124,106,113,123,115,
+
+  121,150,74,14,27,151,56,36,  50,150,69,55,34,102,58,106,  39,82,28,56,34,160,40,31,  75,45,39,131,101,109,119,112,
+  104,54,77,132,55,34,31,18,  17,42,46,106,0,82,44,80,  36,49,0,155,40,40,6,33,  43,19,25,91,101,99,89,114,
+  112,141,38,57,37,72,43,25,  101,67,64,150,25,96,40,152,  26,69,15,132,116,149,27,18,  62,32,26,112,97,105,106,108,
+  78,60,127,91,126,42,111,125,  87,96,141,103,122,124,150,40,  122,106,129,108,108,54,120,151,  152,40,108,78,73,81,74,102,
+  140,74,110,87,99,70,89,78,  91,102,128,88,139,149,142,104,  123,104,100,110,153,97,117,66,  94,57,111,124,100,129,123,119,
+  112,104,136,78,126,105,142,117,  127,127,143,142,137,115,105,119,  116,69,120,135,123,76,130,29,  62,18,144,114,88,105,106,108,
+  82,72,125,103,119,67,146,134,  136,96,134,136,126,124,132,75,  129,68,147,127,116,53,133,80,  97,119,135,74,72,91,76,106,
+  131,76,81,61,69,61,88,82,  92,73,84,76,75,69,78,66,  81,110,61,67,61,85,87,66,  132,71,67,154,139,145,139,146,
+
+  84,133,98,52,42,158,37,23,  89,160,53,107,63,86,87,104,  70,38,30,103,87,98,78,0,  32,26,51,80,58,80,75,91,
+  95,14,130,98,109,14,63,29,  116,13,156,109,122,149,89,39,  70,49,106,128,105,19,149,25,  43,4,137,91,82,95,86,110,
+  109,107,159,107,134,73,101,102,  119,89,132,114,134,142,131,80,  129,98,112,142,121,59,127,27,  103,51,123,138,94,108,99,102,
+  123,108,129,80,86,116,133,102,  135,85,156,146,124,112,121,59,  80,90,122,135,124,64,156,40,  93,40,143,127,110,116,125,119,
+  102,64,148,123,112,87,139,131,  152,77,79,129,128,132,133,45,  111,62,143,119,118,22,113,35,  90,119,135,104,92,99,72,104,
+  109,48,136,138,115,29,99,66,  134,49,121,124,106,149,104,46,  131,63,124,123,115,103,132,19,  63,78,126,96,88,105,101,112,
+  117,136,38,70,20,133,49,30,  43,174,47,36,24,27,9,70,  34,75,10,15,28,140,32,25,  69,25,32,117,95,106,112,109,
+  136,54,77,34,51,77,76,61,  86,49,60,57,54,66,64,51,  69,93,44,77,50,54,57,55,  81,42,72,112,121,115,118,126,
+};
+
+static const uint8 ced_hires_15[1024] = {
+  128,67,142,105,122,81,101,108,  115,105,99,113,90,139,98,75,  108,127,69,108,148,90,86,94,  122,79,127,128,128,128,128,128,
+  69,43,125,125,121,45,135,139,  120,81,130,123,135,133,130,54,  122,137,129,135,128,100,144,30,  140,48,139,91,84,84,66,98,
+  115,65,136,131,106,59,112,137,  120,95,82,98,123,154,142,78,  100,151,107,96,141,104,93,66,  72,112,103,112,105,119,121,120,
+  117,20,88,109,109,87,64,100,  48,105,89,13,112,140,99,63,  140,65,134,113,124,141,95,29,  57,14,123,101,83,100,94,108,
+  68,54,95,122,126,81,139,144,  148,111,115,117,137,130,132,75,  111,92,136,126,133,135,124,37,  131,109,87,85,62,73,66,102,
+  128,93,149,66,99,65,86,92,  121,64,84,111,119,76,106,76,  146,112,85,74,81,85,71,79,  108,70,121,126,126,117,128,122,
+  88,121,101,75,36,126,91,31,  42,131,51,91,40,109,96,83,  71,59,25,36,46,125,25,131,  41,30,50,84,85,105,85,117,
+  121,147,107,57,49,130,60,66,  85,145,91,87,124,117,41,145,  66,61,60,65,99,124,67,55,  72,71,35,119,87,122,89,133,
+
+  57,127,84,73,58,136,67,51,  67,142,118,140,126,83,133,119,  104,49,103,93,122,126,86,38,  19,45,73,63,77,81,56,108,
+  65,103,122,132,130,120,140,134,  129,89,118,96,126,133,115,114,  137,152,130,134,126,97,128,34,  141,25,99,63,86,95,76,104,
+  69,0,128,145,144,0,23,138,  6,23,47,135,81,30,27,14,  146,42,19,110,129,14,0,100,  64,0,137,63,57,87,67,101,
+  115,62,85,72,89,93,112,81,  86,120,86,87,149,100,138,56,  85,132,119,108,113,72,114,73,  126,63,117,122,116,111,127,118,
+  60,9,84,117,134,0,29,27,  131,18,154,129,137,121,128,35,  102,31,120,130,138,19,142,14,  24,0,126,67,63,101,61,91,
+  65,137,116,137,129,65,129,107,  127,33,107,123,116,135,116,113,  115,135,103,126,128,63,126,20,  98,12,122,77,75,84,70,100,
+  130,51,88,108,93,49,100,69,  98,89,80,89,129,128,146,44,  88,110,92,95,149,58,81,63,  91,88,91,114,104,105,116,110,
+  111,148,40,69,66,57,65,38,  55,77,52,115,34,119,36,120,  41,83,19,79,133,115,53,47,  75,32,45,118,101,111,112,114,
+
+  109,144,70,22,29,145,51,35,  45,143,64,50,37,101,54,101,  36,78,28,54,31,155,37,42,  71,41,40,124,96,106,108,111,
+  84,47,72,141,57,27,25,15,  12,34,40,100,1,81,40,74,  33,45,0,151,36,34,1,42,  38,14,26,81,95,91,77,113,
+  96,135,34,65,40,66,38,23,  96,60,59,145,27,95,36,147,  23,65,15,129,113,144,24,29,  58,28,27,104,92,101,95,107,
+  59,54,122,100,129,35,107,124,  82,88,137,98,123,123,146,35,  120,103,127,104,105,49,116,162,  147,36,110,70,68,74,63,101,
+  123,67,105,95,101,63,84,76,  86,94,123,82,140,147,138,98,  120,100,98,106,149,91,112,76,  89,52,112,117,95,126,112,118,
+  96,98,131,86,128,99,137,116,  122,120,138,137,139,114,101,114,  113,65,119,132,120,71,126,41,  58,14,145,105,83,101,95,107,
+  63,66,120,112,122,60,142,133,  131,88,130,131,127,123,128,70,  127,65,145,123,113,48,129,91,  92,115,137,65,68,84,65,105,
+  132,70,76,69,72,54,84,81,  87,66,80,71,77,68,74,62,  78,106,61,63,58,81,84,77,  127,67,68,155,136,142,140,145,
+
+  65,127,93,61,45,151,33,22,  84,152,49,102,64,85,83,99,  68,35,28,99,84,93,74,2,  27,22,53,71,53,74,64,90,
+  76,8,125,107,112,8,59,28,  111,6,152,104,123,148,85,34,  68,46,104,124,102,13,145,36,  38,2,139,82,77,88,75,110,
+  90,101,154,116,137,66,97,101,  114,81,128,109,135,141,127,75,  127,95,110,138,118,54,123,38,  98,47,125,130,89,101,88,102,
+  117,102,124,88,89,109,129,101,  130,77,152,141,125,111,117,55,  79,87,120,131,121,58,152,51,  90,36,145,122,105,113,116,118,
+  83,58,143,132,115,81,134,130,  147,70,74,124,129,131,129,40,  109,59,141,116,115,18,109,46,  85,115,137,95,87,92,61,103,
+  90,42,131,147,118,23,95,65,  129,41,117,119,107,148,100,41,  129,60,122,119,112,98,128,30,  59,74,128,87,83,99,90,112,
+  102,130,34,79,23,126,45,29,  38,166,43,32,26,27,5,65,  30,72,8,11,24,135,28,36,  64,21,34,108,90,103,101,108,
+  121,48,73,42,53,70,71,61,  81,41,56,52,55,65,60,46,  67,89,42,73,48,49,53,67,  76,38,74,106,116,112,115,125,
+};
+
+static const uint8 ced_hires_16[1024] = {
+  126,77,94,99,85,77,105,111,  93,98,106,78,81,99,92,92,  100,116,99,119,82,86,89,94,  132,90,87,127,125,126,126,131,
+  97,47,126,130,125,63,130,137,  117,95,136,122,124,126,123,57,  128,120,120,134,123,99,146,65,  132,56,132,97,96,97,97,110,
+  143,93,132,131,106,93,114,132,  110,112,123,95,107,141,130,103,  112,132,92,86,131,103,106,111,  132,107,103,143,142,143,143,130,
+  128,105,112,105,109,106,116,109,  113,112,120,113,104,107,99,110,  114,124,100,102,105,111,112,115,  125,111,116,128,128,128,128,128,
+  97,58,96,127,131,95,134,142,  146,126,121,116,127,123,126,86,  117,87,128,126,128,135,126,65,  123,114,81,98,96,97,97,110,
+  157,107,120,109,115,107,128,115,  124,121,137,109,95,107,86,117,  126,146,89,100,96,117,120,125,  146,116,117,157,156,157,157,139,
+  157,107,120,109,115,107,128,115,  124,121,137,109,95,107,86,117,  126,146,89,100,96,117,120,125,  146,116,117,157,156,157,157,139,
+  120,155,109,72,78,142,91,78,  87,159,100,85,112,108,49,156,  89,109,52,63,93,123,83,88,  109,79,80,120,119,120,120,118,
+
+  119,70,107,128,115,75,90,135,  86,88,99,71,105,140,107,80,  93,159,136,138,115,79,136,87,  118,78,88,120,118,119,119,118,
+  92,112,123,137,135,133,135,132,  127,104,124,95,116,126,108,126,  143,137,121,134,121,97,130,60,  133,51,92,93,91,92,92,106,
+  130,80,93,121,88,80,101,88,  97,94,110,82,83,142,131,90,  104,119,62,128,144,100,121,98,  119,89,90,130,129,130,130,123,
+  148,98,111,100,106,98,119,106,  115,127,128,100,131,98,124,108,  117,137,102,96,97,108,111,116,  137,107,108,148,147,148,148,135,
+  149,100,112,101,108,100,120,107,  116,113,129,101,88,99,99,110,  118,139,82,93,88,109,112,117,  138,108,110,150,148,149,149,139,
+  98,145,117,142,133,78,123,105,  125,62,112,122,105,128,109,124,  120,119,94,125,123,58,128,66,  97,57,115,98,97,98,98,112,
+  145,96,108,110,104,96,116,103,  112,109,125,97,110,112,132,106,  114,135,78,89,137,105,108,113,  134,104,106,146,144,145,145,132,
+  153,111,124,118,134,108,124,111,  120,117,133,105,101,103,113,113,  122,159,85,115,115,113,116,121,  142,112,113,153,152,153,153,136,
+
+  128,112,119,112,116,113,123,116,  120,119,127,120,111,114,106,117,  120,128,106,108,111,117,118,121,  128,117,122,128,128,128,128,128,
+  120,158,83,72,78,121,91,78,  87,121,100,72,58,70,49,172,  88,108,51,62,58,95,82,87,  108,78,79,119,118,119,119,122,
+  146,96,109,98,104,96,117,120,  113,110,126,98,94,116,108,106,  130,134,109,118,84,105,108,113,  134,104,105,145,144,145,145,137,
+  98,58,122,104,132,48,101,121,  75,101,142,96,111,115,138,58,  126,86,118,103,100,57,118,161,  139,56,102,97,96,97,97,111,
+  139,89,102,96,102,89,110,97,  106,103,124,91,124,135,126,104,  122,127,80,99,141,98,106,106,  127,97,98,138,137,138,138,129,
+  155,105,128,107,113,105,126,113,  122,119,135,107,93,105,84,115,  133,143,86,97,93,114,117,122,  143,113,114,154,153,154,154,139,
+  99,72,121,117,126,72,137,131,  129,103,136,130,116,116,121,79,  134,87,137,123,109,58,132,89,  87,121,131,98,97,98,98,111,
+  154,104,117,106,112,104,125,112,  121,118,134,106,92,104,83,114,  122,142,85,96,92,113,116,121,  142,112,113,153,152,153,153,139,
+
+  156,106,119,108,114,106,127,114,  123,120,136,108,94,106,85,116,  124,144,87,98,94,115,118,123,  144,114,115,155,154,155,155,140,
+  150,101,113,102,109,101,121,108,  117,114,130,102,89,100,80,111,  118,139,82,93,88,109,112,117,  138,108,110,150,148,149,149,140,
+  118,107,154,120,140,68,89,96,  111,92,132,106,123,133,119,78,  133,106,101,137,113,77,125,85,  106,76,118,122,116,117,117,121,
+  149,99,112,101,107,99,120,107,  116,113,129,101,102,99,83,109,  117,137,115,96,139,108,111,116,  137,107,108,148,147,148,148,135,
+  102,62,144,137,119,94,129,128,  144,82,82,123,118,124,122,62,  116,90,133,116,111,61,112,69,  90,121,131,101,100,101,101,110,
+  156,106,119,108,114,106,127,114,  123,120,136,108,94,106,85,116,  124,144,87,98,94,115,118,123,  144,114,115,155,154,155,155,140,
+  156,106,119,108,114,106,127,114,  123,120,136,108,94,106,85,116,  124,144,87,98,94,115,118,123,  144,114,115,155,154,155,155,140,
+  137,88,100,89,96,96,108,95,  104,101,117,89,76,87,67,98,  105,126,69,80,75,96,99,104,  125,95,97,137,135,136,136,123,
+};
+
+static const uint8 ced_hires_17[1024] = {
+  89,91,92,133,130,37,125,74,  42,95,144,140,137,137,121,42,  123,75,132,124,133,87,128,83,  74,53,119,94,87,91,93,108,
+  102,61,121,126,115,65,141,137,  130,96,110,126,137,135,142,73,  128,143,129,129,127,116,147,95,  154,82,141,107,100,103,106,114,
+  152,89,132,131,102,84,114,134,  128,107,81,104,125,155,153,95,  103,154,108,87,139,120,95,146,  137,146,100,157,150,154,156,136,
+  130,66,62,72,62,122,86,62,  83,84,58,60,65,91,78,190,  73,115,92,83,85,91,68,123,  115,94,83,135,128,131,134,127,
+  103,75,93,124,121,103,146,143,  159,127,96,121,140,133,145,96,  118,98,137,121,133,153,128,96,  146,146,90,108,101,104,107,114,
+  111,83,105,33,129,120,114,135,  102,75,39,131,136,117,141,54,  122,96,138,110,121,62,115,104,  96,75,125,116,109,112,115,120,
+  131,94,108,84,119,91,141,148,  104,99,90,137,142,121,136,74,  104,117,149,128,125,99,131,125,  116,95,74,136,129,133,135,128,
+  109,71,69,85,106,85,106,131,  62,57,93,110,101,126,73,52,  116,94,127,130,102,60,143,102,  94,73,52,114,107,118,113,119,
+
+  100,147,79,74,50,157,72,48,  76,157,98,143,128,85,145,139,  110,85,103,87,121,143,89,93,  85,84,74,105,98,101,104,112,
+  97,124,119,134,125,141,147,132,  139,104,99,100,129,135,127,134,  143,159,130,128,125,115,131,91,  155,61,101,102,95,99,101,109,
+  101,37,124,147,139,33,57,137,  54,39,29,139,83,34,39,44,  153,86,16,104,129,52,39,151,  86,65,140,106,99,102,105,116,
+  159,95,91,81,81,113,115,91,  112,133,87,89,150,97,149,102,  102,144,118,100,111,110,113,152,  144,123,117,164,157,160,163,142,
+  86,23,75,47,132,18,43,105,  39,54,141,105,128,124,110,29,  89,72,105,147,131,38,115,80,  71,50,78,91,84,88,90,107,
+  103,158,113,138,124,86,135,105,  137,41,88,127,119,137,128,133,  120,142,104,120,127,81,129,97,  111,67,124,108,101,105,107,116,
+  156,93,94,108,84,88,113,88,  109,99,85,92,130,128,156,99,  99,142,88,85,147,108,94,150,  141,120,99,161,154,158,160,138,
+  98,50,162,115,131,30,54,149,  51,46,135,121,106,128,119,41,  135,83,109,116,138,49,147,91,  83,62,136,103,96,99,102,111,
+
+  160,171,92,82,82,172,116,92,  113,164,88,90,85,109,82,129,  97,139,69,66,71,175,92,147,  139,118,97,159,152,155,158,137,
+  129,176,77,84,94,159,85,102,  82,160,57,105,70,85,71,176,  89,108,48,73,73,173,81,116,  108,87,76,128,121,124,127,122,
+  180,139,119,109,109,119,143,119,  140,135,125,117,112,120,109,130,  124,166,112,93,98,160,119,174,  166,145,124,180,179,177,180,152,
+  156,139,88,78,116,166,138,88,  109,177,94,86,81,89,111,115,  93,135,95,88,77,162,98,171,  135,114,93,155,148,151,154,133,
+  156,92,106,103,101,88,112,88,  114,119,110,91,149,156,157,125,  129,135,101,102,152,114,117,143,  135,114,116,155,148,151,154,133,
+  152,109,131,74,109,171,109,112,  159,167,115,117,135,115,136,115,  148,132,118,103,119,157,112,140,  131,110,109,151,144,148,150,134,
+  112,94,124,121,124,90,156,139,  149,111,118,142,137,133,148,98,  137,91,149,121,116,67,136,145,  107,155,143,111,104,107,110,113,
+  180,119,115,104,105,114,139,114,  135,120,111,113,108,115,104,125,  119,162,92,88,93,128,114,170,  161,140,119,178,174,174,178,149,
+
+  128,111,146,50,134,118,137,150,  112,103,137,155,135,132,138,87,  135,108,147,113,118,105,137,126,  107,169,65,127,120,124,126,122,
+  109,46,52,31,32,41,66,41,  62,47,127,60,65,52,41,52,  46,89,19,130,30,55,41,97,  88,67,66,108,101,105,107,132,
+  131,129,158,124,139,93,111,106,  131,102,116,120,145,150,146,102,  136,111,114,135,120,77,129,119,  110,89,130,146,123,127,129,123,
+  171,107,103,101,93,108,127,103,  124,109,99,114,130,122,116,114,  108,150,135,102,155,116,111,158,  150,129,108,170,163,166,169,141,
+  115,87,147,141,117,109,149,136,  164,92,64,135,139,141,148,71,  119,94,145,113,118,60,116,102,  99,155,143,114,107,110,113,112,
+  180,125,121,120,111,120,145,120,  141,126,117,119,130,121,126,131,  125,168,98,94,99,134,120,176,  167,146,125,180,179,177,180,152,
+  108,45,99,103,139,40,96,117,  61,87,85,145,139,127,138,51,  127,88,152,133,136,64,131,96,  87,66,135,107,100,104,106,112,
+  157,94,95,80,80,109,114,90,  119,96,86,88,83,91,80,101,  95,137,67,64,69,103,90,145,  137,116,95,154,150,150,154,125,
+};
+
+static const uint8 ced_hires_18[1024] = {
+  100,47,105,106,88,37,119,122,  76,90,99,47,87,103,100,98,  95,121,106,126,89,84,80,50,  146,101,49,119,100,126,121,130,
+  74,55,134,135,131,65,138,145,  126,83,147,122,129,128,131,71,  130,133,126,138,131,103,151,36,  145,59,129,97,84,83,73,90,
+  123,78,145,142,117,80,115,143,  127,97,99,98,117,149,143,95,  108,146,104,100,145,107,101,72,  83,123,94,119,107,123,126,111,
+  102,42,63,81,78,121,44,64,  50,73,66,32,62,84,68,187,  77,62,89,92,89,80,66,40,  63,66,76,102,89,103,100,106,
+  72,67,105,133,136,102,142,150,  155,114,132,117,131,125,133,93,  119,88,133,130,137,139,132,46,  137,120,78,89,64,76,73,94,
+  94,75,117,44,144,118,110,142,  98,61,78,127,127,109,129,55,  123,48,134,119,125,45,118,38,  80,35,112,83,72,101,83,96,
+  95,87,122,95,134,91,137,156,  101,88,126,133,133,114,125,79,  107,76,145,138,130,85,136,69,  79,46,38,117,85,112,100,104,
+  124,160,116,69,60,150,64,73,  92,147,109,87,118,112,42,163,  75,62,57,68,103,128,75,63,  80,83,28,120,89,124,96,125,
+
+  94,66,118,135,120,83,97,144,  63,76,82,62,110,143,115,69,  85,171,142,143,125,71,143,43,  138,58,89,97,78,95,93,94,
+  67,116,131,143,140,140,143,140,  136,91,135,96,120,128,115,131,  145,148,127,138,130,101,135,42,  146,37,89,70,86,94,79,96,
+  101,82,86,128,82,78,65,84,  68,60,103,72,89,145,141,49,  109,135,73,134,155,108,128,81,  135,61,70,108,97,107,117,96,
+  123,75,94,83,100,113,116,87,  92,122,102,87,143,94,139,71,  93,129,116,111,117,75,121,80,  131,72,108,126,118,118,133,111,
+  125,126,107,111,110,90,98,100,  104,78,136,102,98,121,122,116,  113,113,88,103,108,91,109,90,  113,82,103,166,118,129,154,121,
+  71,150,125,148,139,85,131,113,  134,35,124,123,110,130,117,130,  123,131,100,130,132,66,133,26,  103,24,112,82,78,81,75,92,
+  134,66,98,119,103,70,104,77,  104,91,98,89,123,122,146,64,  97,108,89,98,153,65,88,72,  98,100,82,121,109,113,126,105,
+  134,123,148,135,153,131,131,112,  86,89,109,114,122,118,134,113,  106,183,104,135,137,97,127,140,  120,93,88,129,122,120,132,113,
+
+  95,138,100,94,128,146,84,59,  64,158,26,72,140,108,64,96,  48,37,129,66,66,162,79,20,  36,60,93,75,70,90,70,98,
+  95,163,41,47,45,129,41,47,  56,110,67,38,37,31,25,179,  45,69,19,50,32,105,52,43,  62,51,33,101,100,101,98,98,
+  122,92,117,103,114,78,115,136,  82,95,96,73,110,123,125,87,  143,128,130,139,99,92,109,87,  147,65,68,148,119,143,129,127,
+  68,66,131,110,138,55,109,130,  88,90,153,97,117,117,146,52,  134,105,130,114,115,58,129,176,  158,53,106,82,71,82,78,96,
+  129,80,114,106,112,83,88,82,  92,96,140,82,134,142,139,116,  135,103,101,116,160,101,127,91,  102,70,110,124,105,134,125,113,
+  123,95,135,69,116,163,102,112,  149,147,143,105,119,100,118,101,  153,80,118,115,127,146,119,62,  116,70,102,133,102,113,117,109,
+  70,78,129,122,132,80,145,139,  137,90,147,130,121,118,129,87,  141,67,148,133,123,58,143,105,  104,132,134,80,74,95,79,100,
+  136,82,86,79,83,76,90,87,  92,67,102,69,70,65,74,80,  92,115,66,71,68,91,96,94,  140,85,71,156,136,150,142,140,
+
+  104,96,151,58,141,110,125,150,  102,82,165,143,118,117,119,80,  140,61,146,125,125,94,144,85,  85,147,50,100,110,107,95,109,
+  122,103,115,106,89,90,88,87,  92,99,110,70,86,100,108,86,  107,107,86,104,98,85,106,87,  134,101,84,138,122,134,133,128,
+  93,113,163,126,146,86,99,107,  120,83,144,108,129,135,127,92,  141,96,113,148,128,63,136,50,  109,64,121,140,95,106,100,97,
+  128,86,112,109,106,102,108,101,  111,75,126,105,117,111,101,66,  104,108,137,116,165,81,125,82,  114,102,102,129,126,121,134,116,
+  89,71,152,143,125,101,137,136,  153,72,92,124,123,126,129,57,  123,62,145,126,125,29,123,61,  97,133,134,107,94,103,79,98,
+  99,71,108,117,119,66,122,99,  108,25,83,132,126,131,130,88,  120,83,124,123,119,71,105,40,  88,140,139,89,119,109,90,120,
+  109,146,119,71,82,143,99,110,  107,147,49,135,131,127,53,105,  80,80,69,92,140,135,125,42,  60,114,37,69,92,91,80,109,
+  129,78,95,77,81,106,104,83,  99,66,82,66,73,73,63,80,  101,97,69,88,80,74,80,98,  98,65,64,114,130,119,121,120,
+};
+
+static const uint8 ced_hires_19[1024] = {
+  98,93,104,135,134,69,120,71,  84,99,150,144,141,141,121,77,  123,96,135,127,139,85,130,83,  94,88,120,97,97,97,97,113,
+  110,85,134,128,119,81,138,142,  127,104,115,130,141,138,142,90,  128,119,133,131,132,117,148,95,  129,101,142,109,109,109,109,119,
+  145,120,126,112,107,116,133,123,  131,123,115,115,110,142,136,124,  123,143,99,95,128,132,120,130,  141,135,125,144,144,144,144,134,
+  134,109,115,93,96,120,122,107,  120,112,104,104,99,106,91,186,  112,132,88,84,92,121,109,119,  130,124,114,133,133,133,133,130,
+  111,86,102,124,125,107,142,148,  156,131,104,126,144,136,144,90,  119,109,140,123,138,153,129,96,  122,136,91,110,110,110,110,118,
+  118,93,119,78,133,123,107,139,  104,97,88,134,139,118,140,98,  122,117,141,111,125,105,113,103,  114,109,123,117,117,117,117,124,
+  135,110,116,95,117,106,124,148,  121,114,105,136,141,117,132,115,  114,134,148,125,126,122,125,120,  131,126,115,134,134,134,134,130,
+  117,92,98,86,109,98,105,136,  103,95,97,112,105,129,74,96,  110,115,130,131,106,104,144,102,  113,107,97,116,116,116,116,123,
+
+  108,150,89,67,70,160,96,81,  94,161,103,147,132,80,144,139,  109,106,105,88,126,143,83,93,  104,98,88,107,107,107,107,117,
+  106,127,132,135,129,144,143,138,  136,107,104,103,133,139,126,134,  143,136,133,131,131,114,133,91,  130,96,102,105,105,105,105,114,
+  109,84,137,148,143,80,97,142,  95,87,79,143,84,81,66,88,  152,107,63,106,134,96,84,104,  105,99,141,108,108,108,108,120,
+  146,121,127,106,108,117,135,119,  132,125,116,116,133,118,126,126,  125,145,101,96,104,133,121,131,  142,137,126,145,145,145,145,136,
+  95,70,86,54,136,66,83,111,  81,73,147,109,132,128,110,74,  88,93,108,151,137,82,117,80,  91,85,75,94,94,94,94,111,
+  112,161,124,140,127,83,131,110,  135,90,92,132,123,141,128,133,  120,110,106,123,133,99,130,97,  108,102,125,111,111,111,111,121,
+  145,120,126,105,107,116,134,118,  131,124,115,115,115,117,137,125,  124,144,100,95,132,132,120,130,  141,136,125,144,144,144,144,135,
+  148,123,129,107,115,119,136,121,  134,126,118,118,113,120,115,127,  126,146,102,103,111,135,123,133,  144,138,128,147,147,147,147,136,
+
+  147,153,128,106,109,118,135,120,  133,153,117,117,112,119,104,126,  126,146,102,108,106,165,123,133,  144,138,128,147,147,147,147,136,
+  132,174,113,92,94,156,121,105,  118,160,102,102,97,104,90,172,  112,132,88,83,91,175,108,118,  129,124,113,132,132,132,132,127,
+  152,127,133,112,114,123,141,125,  138,131,122,122,117,124,110,132,  132,152,108,103,111,140,128,138,  149,144,133,152,152,152,152,138,
+  117,92,136,107,132,88,105,132,  103,110,127,110,134,133,162,96,  133,116,138,108,116,105,127,176,  144,108,118,117,117,117,117,120,
+  148,123,129,107,110,119,136,121,  134,126,118,118,135,141,139,127,  127,147,103,99,147,136,124,134,  145,139,129,148,148,148,148,134,
+  148,123,129,107,110,162,136,121,  144,159,118,118,123,120,120,127,  142,147,113,99,117,151,124,134,  145,139,129,148,148,148,148,135,
+  117,92,133,120,125,98,149,141,  144,116,122,143,138,132,145,97,  142,117,157,127,125,105,141,103,  114,150,148,117,117,117,117,120,
+  153,128,134,112,115,124,141,126,  139,131,123,123,118,125,110,132,  132,152,108,104,112,141,129,139,  150,144,134,153,153,153,153,138,
+
+  132,107,153,92,134,113,121,151,  118,111,137,154,135,129,133,112,  140,132,153,117,125,120,141,118,  129,163,113,132,132,132,132,127,
+  113,88,94,73,75,84,102,86,  99,92,127,83,78,85,71,93,  93,113,69,135,72,101,89,99,  110,105,94,113,113,113,113,129,
+  135,125,166,120,137,106,124,108,  121,114,115,115,143,149,142,115,  138,135,119,140,128,123,131,121,  132,127,131,135,135,135,135,129,
+  151,126,132,111,113,122,140,124,  137,130,121,121,116,123,109,131,  131,151,114,102,138,139,127,137,  148,143,132,151,151,151,151,137,
+  120,95,157,140,118,108,142,138,  158,99,90,136,140,141,145,100,  121,120,153,120,127,108,119,106,  117,149,148,120,120,120,120,119,
+  136,111,140,151,118,107,124,109,  137,114,106,126,111,155,113,115,  138,135,129,118,120,124,137,122,  133,127,132,136,136,136,136,131,
+  151,146,132,110,113,145,139,124,  137,154,121,121,116,123,108,130,  130,150,106,102,110,139,127,137,  148,142,132,151,151,151,151,137,
+  146,121,127,105,108,117,134,119,  132,124,116,116,111,118,103,125,  125,145,101,97,105,134,122,132,  143,137,127,146,146,146,146,131,
+};
+
+static const uint8 ced_hires_20[1024] = {
+  111,51,50,66,67,37,61,60,  62,52,151,76,117,107,89,44,  64,85,123,112,43,48,109,55,  80,47,73,105,103,120,107,126,
+  99,78,66,40,113,81,103,136,  51,33,85,110,79,133,68,53,  121,80,125,133,107,49,144,49,  79,46,58,104,94,114,94,119,
+  82,89,103,144,141,45,125,85,  60,80,142,139,139,140,118,46,  133,65,134,127,134,77,126,29,  65,28,130,79,73,84,74,99,
+  148,112,111,112,105,115,135,114,  133,109,103,112,104,109,99,132,  123,145,97,101,94,117,110,138,  144,123,128,148,147,147,148,137,
+  97,70,96,138,128,103,151,149,  163,109,95,119,143,133,146,96,  122,107,138,122,134,143,125,67,  141,129,98,104,95,92,94,114,
+  112,73,108,62,136,120,118,141,  105,57,53,129,139,116,142,70,  126,92,139,111,122,64,111,67,  100,63,134,104,102,110,102,119,
+  126,53,74,82,77,64,78,76,  74,51,58,72,68,60,68,61,  85,102,50,148,65,58,59,77,  97,64,90,122,116,120,117,129,
+  94,50,95,141,134,56,69,127,  60,37,151,133,141,146,122,58,  115,74,138,121,145,52,105,44,  78,35,128,96,85,89,89,105,
+
+  93,143,83,89,63,158,83,60,  83,139,98,142,132,86,147,141,  115,84,105,89,123,134,85,67,  69,69,82,94,100,99,89,116,
+  92,119,122,149,133,142,152,139,  144,87,98,98,132,136,128,136,  148,160,131,130,127,105,129,63,  151,58,111,91,104,103,95,113,
+  148,106,116,121,106,102,130,108,  127,89,97,117,103,117,112,113,  133,145,97,94,107,111,109,132,  144,121,140,148,147,147,148,137,
+  96,26,70,54,136,23,56,118,  57,39,143,109,136,129,114,24,  102,75,104,149,132,26,115,40,  62,32,99,95,91,85,89,104,
+  122,116,63,61,63,156,88,58,  77,162,52,66,68,77,80,112,  84,106,43,53,46,123,55,75,  100,67,77,125,117,120,120,120,
+  137,128,86,79,104,158,111,95,  93,154,68,90,84,84,82,113,  113,121,63,81,93,155,94,91,  115,82,99,136,132,131,136,128,
+  91,40,166,130,139,40,74,156,  53,33,137,119,108,132,120,47,  139,70,109,117,140,49,148,47,  74,31,142,89,83,85,84,103,
+  113,165,84,84,91,133,107,85,  101,142,80,94,94,82,76,158,  92,97,86,89,67,171,112,67,  91,63,89,116,108,115,112,117,
+
+  84,151,102,50,102,126,72,103,  59,149,74,148,115,116,104,148,  125,78,106,107,133,146,137,43,  60,158,87,85,79,92,80,104,
+  115,72,78,158,69,65,71,53,  67,50,47,103,38,82,62,99,  72,98,29,152,56,69,47,84,  93,67,72,118,125,119,113,127,
+  116,167,96,89,95,156,80,88,  85,135,64,95,66,79,70,161,  97,98,65,84,85,166,84,67,  92,59,94,117,117,113,112,116,
+  95,66,122,116,132,60,119,129,  93,85,117,101,129,126,160,66,  136,118,134,105,111,69,122,180,  161,71,127,101,101,101,96,115,
+  148,107,121,119,120,108,132,120,  135,112,126,122,123,123,121,115,  140,147,122,115,120,131,127,129,  146,120,138,149,149,149,149,138,
+  145,100,125,89,108,164,120,114,  156,141,106,107,130,110,130,109,  154,126,119,105,123,147,113,96,  136,98,123,146,132,131,136,131,
+  96,81,121,129,125,83,154,139,  147,86,111,134,134,126,142,92,  143,98,152,124,120,68,135,108,  111,140,154,101,101,106,99,117,
+  148,101,97,103,91,93,119,102,  119,82,86,90,96,95,97,105,  112,145,87,80,83,109,104,119,  153,110,113,160,153,158,154,146,
+
+  96,28,42,33,46,35,53,45,  49,31,126,40,50,53,52,43,  68,80,39,131,35,47,41,50,  74,51,47,99,104,105,97,134,
+  148,137,108,111,112,125,128,118,  139,87,95,116,95,112,110,149,  117,147,86,88,103,140,105,130,  146,146,127,149,149,149,149,138,
+  148,129,121,142,101,121,132,114,  128,114,99,114,126,128,120,128,  126,147,104,99,97,132,116,156,  146,121,126,149,149,149,149,138,
+  94,39,97,117,138,45,88,117,  67,38,75,139,136,117,133,55,  139,78,153,138,143,54,128,55,  79,53,151,97,89,100,92,107,
+  108,73,144,149,117,103,147,136,  162,65,59,127,136,135,143,70,  126,97,148,117,122,55,115,79,  109,140,154,117,114,112,100,115,
+  148,128,120,108,115,137,133,110,  129,119,99,112,110,115,116,137,  127,147,98,107,93,131,122,130,  146,149,127,149,149,149,149,138,
+  90,141,92,62,123,145,98,94,  71,157,54,115,84,131,112,151,  121,73,99,126,103,146,129,51,  68,144,73,93,87,100,88,107,
+  137,80,87,78,75,97,105,85,  110,75,63,75,82,82,81,90,  99,126,66,86,72,87,81,105,  118,85,106,131,142,134,137,133,
+};
+
+static const uint8 ced_hires_21[1024] = {
+  83,35,83,86,68,31,99,99,  55,99,74,45,80,93,87,75,  79,100,91,111,76,86,61,33,  125,101,49,100,98,111,105,129,
+  59,43,112,116,111,56,117,121,  105,92,122,118,122,117,117,48,  113,112,111,124,119,105,131,14,  124,58,128,80,75,64,53,89,
+  102,65,123,122,96,71,94,119,  105,106,73,93,110,138,129,71,  91,126,89,85,132,109,80,55,  63,122,92,108,98,117,104,108,
+  87,29,40,62,58,112,27,40,  29,82,41,27,55,73,54,164,  60,39,74,78,77,82,45,14,  40,65,75,85,83,89,81,104,
+  56,54,82,113,116,92,121,126,  133,122,107,112,124,114,119,69,  102,67,118,115,124,140,111,23,  115,119,76,70,57,61,58,92,
+  78,62,95,25,124,109,90,119,  76,70,53,123,120,99,116,31,  107,29,119,104,112,46,98,15,  58,34,111,65,65,82,61,97,
+  81,74,99,75,114,81,117,133,  79,96,101,129,126,104,112,55,  91,50,130,123,117,86,115,48,  57,41,32,98,76,95,82,105,
+  107,147,94,49,39,141,43,49,  70,156,84,83,111,101,28,140,  58,35,42,54,90,129,55,41,  53,82,24,102,79,105,77,124,
+
+  75,53,95,115,100,73,77,121,  43,85,59,58,103,133,102,46,  69,150,127,128,112,72,122,25,  117,58,87,80,75,79,76,97,
+  52,103,109,123,121,131,123,117,  114,100,110,92,113,118,102,108,  129,127,111,123,117,103,115,21,  125,36,88,52,79,81,65,95,
+  87,70,63,109,62,69,46,61,  46,69,77,67,83,135,128,27,  93,114,58,120,143,110,108,59,  114,61,69,92,90,96,100,95,
+  109,61,71,65,81,104,96,64,  70,131,79,83,136,84,125,49,  77,107,101,97,105,80,101,61,  111,75,106,115,115,112,116,109,
+  112,114,84,92,90,82,79,75,  83,89,111,97,91,110,108,93,  97,93,73,90,96,92,89,68,  88,77,101,154,114,122,143,121,
+  53,137,103,128,119,76,111,90,  112,44,99,119,103,120,104,107,  107,110,85,115,119,68,113,4,  82,21,111,63,72,62,54,91,
+  115,54,75,100,84,61,84,54,  83,100,70,85,116,112,133,43,  80,88,74,84,141,60,68,52,  81,99,81,113,103,108,109,105,
+  120,111,126,115,133,122,111,89,  67,99,81,110,116,108,120,90,  90,163,89,121,125,100,107,119,  103,94,88,124,116,114,122,112,
+
+  102,136,101,31,69,99,109,94,  95,151,120,85,100,94,78,59,  51,53,85,107,69,157,101,38,  64,67,59,99,103,108,95,116,
+  79,150,19,27,24,119,22,24,  34,118,44,32,28,19,11,155,  18,43,0,29,14,101,25,9,  34,44,25,79,93,78,75,98,
+  109,79,94,83,94,67,94,111,  63,103,71,67,102,112,111,62,  121,102,109,118,80,89,83,60,  119,63,60,126,109,130,110,127,
+  52,53,108,90,118,45,88,105,  66,98,128,92,109,106,132,29,  112,78,109,93,96,54,103,149,  131,47,99,57,55,61,53,92,
+  113,67,91,86,91,74,66,59,  71,104,115,78,126,131,125,92,  113,78,81,96,142,98,101,64,  76,66,103,108,98,123,99,112,
+  107,81,112,47,95,153,81,88,  126,155,117,100,111,89,104,77,  131,53,97,95,109,142,93,38,  90,63,95,109,89,98,95,108,
+  53,65,106,102,111,70,124,115,  115,98,121,126,113,107,115,63,  119,40,128,113,105,54,117,78,  77,126,127,58,62,74,59,98,
+  130,70,67,61,62,63,68,65,  77,75,76,63,59,56,60,58,  74,89,44,52,52,89,72,70,  112,74,65,152,122,141,139,137,
+
+  87,83,128,38,121,100,104,126,  80,90,140,138,111,106,105,56,  118,38,125,105,107,90,118,59,  57,141,42,74,97,86,75,106,
+  113,89,93,86,69,80,73,66,  73,107,89,65,79,88,93,67,  86,81,65,83,79,83,81,62,  108,93,80,129,114,126,119,125,
+  72,100,140,106,126,76,78,82,  98,91,119,103,121,124,113,68,  119,69,92,127,109,58,110,23,  82,57,114,116,85,81,78,92,
+  114,71,89,89,85,92,85,77,  88,84,100,100,109,99,87,42,  81,78,116,95,146,76,98,54,  89,93,94,119,114,113,115,110,
+  73,57,129,122,104,91,116,112,  130,80,66,119,115,115,115,33,  101,33,124,105,107,25,97,34,  70,127,127,84,81,81,57,94,
+  74,41,117,137,107,32,76,46,  113,50,108,113,93,131,86,33,  121,33,104,108,103,103,115,9,  34,85,117,79,74,86,80,101,
+  105,136,69,50,52,146,69,59,  68,160,143,98,90,95,36,91,  54,64,94,68,86,132,126,39,  89,121,50,110,105,109,106,138,
+  119,64,72,56,60,96,83,58,  74,75,59,61,65,63,48,59,  78,73,49,68,62,65,54,71,  69,57,57,111,119,111,115,117,
+};
+
+
+

contrib/google-ced/compact_enc_det_hint_code.cc

@@ -0,0 +1,169 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "compact_enc_det_hint_code.h"
+
+#include <ctype.h>                      // for isalpha
+#include <string.h>                     // for NULL, memchr, strlen, etc
+
+#include "util/basictypes.h"            // for uint8, uint32
+#include "util/string_util.h"
+
+// Upper to lower, keep digits, everything else to minus '-' (2d)
+static const char kCharsetToLowerTbl[256] = {
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37, 0x38,0x39,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+  0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
+  0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x61,0x62,0x63,0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,
+  0x70,0x71,0x72,0x73,0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+  0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d, 0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,0x2d,
+};
+
+
+static const char kIsAlpha[256] = {
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
+  0,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,0,0,0,0,0,
+
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+static const char kIsDigit[256] = {
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 1,1,1,1,1,1,1,1, 1,1,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,
+};
+
+static const char* kFakeEncodingName[] = {
+  "FakeEnc100", "FakeEnc101", "FakeEnc102", "FakeEnc103", "FakeEnc104",
+  "FakeEnc105", "FakeEnc106", "FakeEnc107", "FakeEnc108", "FakeEnc109",
+  "FakeEnc110", "FakeEnc111", "FakeEnc112", "FakeEnc113", "FakeEnc114",
+  "FakeEnc115", "FakeEnc116", "FakeEnc117", "FakeEnc118", "FakeEnc119",
+};
+static const char* kFakeEncodingName2[] = {
+  "FakeEnc_0", "FakeEnc_1", "FakeEnc_2", "FakeEnc_3", "FakeEnc_4",
+};
+
+// Return name for extended encoding
+const char* MyEncodingName(Encoding enc) {
+  if (enc < 0) {
+    return "~";
+  }
+  if (enc == ISO_8859_1) {
+    return "Latin1";   // I can't stand "ASCII" for this
+  }
+  if (enc < NUM_ENCODINGS) {
+    return EncodingName(enc);
+  }
+  // allow fake names, for exploration
+  if ((NUM_ENCODINGS <= enc) && (enc < (NUM_ENCODINGS + 4))) {
+    return kFakeEncodingName2[enc - NUM_ENCODINGS];
+  }
+  if ((100 <= enc) && (enc < 120)) {
+    return kFakeEncodingName[enc - 100];
+  }
+  return "~";
+}
+
+
+// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize charset= values
+string MakeChar44(const string& str) {
+  string res("________");     // eight underscores
+  int l_ptr = 0;
+  int d_ptr = 0;
+  for (uint32 i = 0; i < str.size(); ++i) {
+    uint8 uc = static_cast<uint8>(str[i]);
+    if (kIsAlpha[uc]) {
+      if (l_ptr < 4) {                  // Else ignore
+        res[l_ptr] = kCharsetToLowerTbl[uc];
+        l_ptr++;
+      }
+    } else if (kIsDigit[uc]) {
+      if (d_ptr < 4) {
+        res[4 + d_ptr] = kCharsetToLowerTbl[uc];
+      } else {
+        // Keep last 4 digits by shifting left
+        res[4] = res[5];
+        res[5] = res[6];
+        res[6] = res[7];
+        res[7] = kCharsetToLowerTbl[uc];
+      }
+      d_ptr++;
+    }   // If neither letter nor digit, drop entirely
+  }
+  return res;
+}
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar4(const string& str) {
+  string res("____");     // four underscores
+  int l_ptr = 0;
+  for (uint32 i = 0; i < str.size(); ++i) {
+    uint8 uc = static_cast<uint8>(str[i]);
+    if (kIsAlpha[uc] | kIsDigit[uc]) {
+      if (l_ptr < 4) {                  // Else ignore
+        res[l_ptr] = kCharsetToLowerTbl[uc];
+        l_ptr++;
+      }
+    }
+  }
+  return res;
+}
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar8(const string& str) {
+  string res("________");     // eight dots
+  int l_ptr = 0;
+  for (uint32 i = 0; i < str.size(); ++i) {
+    uint8 uc = static_cast<uint8>(str[i]);
+    if (kIsAlpha[uc] | kIsDigit[uc]) {
+      if (l_ptr < 8) {                  // Else ignore
+        res[l_ptr] = kCharsetToLowerTbl[uc];
+        l_ptr++;
+      }
+    }
+  }
+  return res;
+}

contrib/google-ced/compact_enc_det_hint_code.h

@@ -0,0 +1,45 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
+#define COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_
+
+#include <string>                        // for string
+
+#include "util/basictypes.h"             // for uint32
+#include "util/encodings/encodings.h"    // for Encoding
+
+using std::string;
+
+// Return name for extended encoding
+const char* MyEncodingName(Encoding enc);
+
+// Normalize ASCII string to first 4 alphabetic chars and last 4 digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize charset= values
+string MakeChar44(const string& str);
+
+// Normalize ASCII string to first 4 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize TLD values
+string MakeChar4(const string& str);
+
+// Normalize ASCII string to first 8 alphabetic/digit chars
+// Letters are forced to lowercase ASCII
+// Used to normalize other values
+string MakeChar8(const string& str);
+
+#endif  // COMPACT_ENC_DET_COMPACT_ENC_DET_HINT_CODE_H_

contrib/google-ced/detail_head_string.inc

@@ -0,0 +1,152 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+// Produced by stringify.cc on 2007-09-28 09:13 from file i18n/encodings/compact_enc_det/tools/detail_head.ps
+  "%!PS-Adobe-2.0\n\n/inch {72 mul} def\n/cshow {dup stringwidth pop -2 div"
+  " 0 rmoveto show} def\n\n/lmargin 0.5 inch def\n/rmargin 8.5 inch def\n/t"
+  "margin 10.5 inch def\n/bmargin 0.5 inch def\n\n\n% set to N>=0 to track "
+  "ranked encoding N\n/track-me -1 def\n/track-me2 -1 def\n\n/columns 2 def"
+  "\n\n/lpi 18 def                 % lines per inch\n/lpc lpi 10 mul def   "
+  "      % lines per column\n/lpp lpc columns mul def    % lines per page\n"
+  "/probw 3.0 inch def         % probability width\n/probr 50 def          "
+  "     % probability range\n/widowlines lpi 2 idiv def  % 1/2 inch widow a"
+  "t bottom\n/widowlines lpi def         % 1 inch widow at bottom\n\n/lpg l"
+  "pi 2 idiv def         % 1/2 inch spacing between groups\n\n/delc 4 inch "
+  "def\n/dell -1 inch lpi div def\n\n/next-line 0 def      % 24 lines per i"
+  "nch, 240 per column\n\n/Cfont /Courier findfont 6 scalefont def\n/Hfont "
+  "/Helvetica findfont 6 scalefont def\nHfont setfont\n\n\n% simple string "
+  "hash -- sum the characters\n/strhash {\n  /hstr exch def\n  /h 0 def\n  "
+  "0 1 hstr length 1 sub {/i exch def  /h h hstr i get add def} for\n  h\n}"
+  " def\n\n% convert pro at 30 per 2x to 0-2.5 inches spanning -50 to 0\n/p"
+  "rob2x {\n  30 idiv probr div probw mul neg probw add\n}def\n\n\n/cliptoc"
+  "olumn {\n  % ====== MUST MATCH ME ======\n  gsave\n  lmargin tmargin mov"
+  "eto\n  next-line lpc idiv delc mul 0 rmoveto\n  -1 18 rmoveto 0 -10.5 in"
+  "ch rlineto  delc 2 add 0 rlineto  0 10.5 inch rlineto closepath\n  clip\n"
+  "  newpath\n} def\n\n/endcliptocolumn {\n  grestore\n  % ====== MUST MATC"
+  "H ME ======\n} def\n\n\n\n/column-box {\n  lmargin tmargin moveto\n  nex"
+  "t-line 1 sub lpc idiv delc mul  next-line 1 sub lpc mod 1 add dell mul r"
+  "moveto\n  % box\n  gsave -1.5 0 rmoveto 0 detail-count dell mul neg rmov"
+  "eto  probw 3 add 0 rlineto 0.25 setlinewidth stroke grestore\n  gsave -1"
+  ".5 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlinewidth strok"
+  "e grestore\n  gsave probw .8 mul 0 rmoveto 0 detail-count dell mul neg r"
+  "lineto 0.25 setlinewidth 0.8 setgray stroke grestore\n  gsave probw .6 m"
+  "ul 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlinewidth 0.8 s"
+  "etgray stroke grestore\n  gsave probw .4 mul 0 rmoveto 0 detail-count de"
+  "ll mul neg rlineto 0.25 setlinewidth 0.8 setgray stroke grestore\n  gsav"
+  "e probw .2 mul 0 rmoveto 0 detail-count dell mul neg rlineto 0.25 setlin"
+  "ewidth 0.8 setgray stroke grestore\n  gsave probw 1.5 add 0 rmoveto 0 de"
+  "tail-count dell mul neg rlineto 0.25 setlinewidth stroke grestore\n} def"
+  "\n\n\n/IncrementLine {\n  /incr exch def\n  /next-line next-line incr ad"
+  "d def\n  next-line lpc ge   next-line incr sub lpc lt   and {\n    % We "
+  "just went to the top of column 2; redo clip\n    endcliptocolumn     % M"
+  "UST match\n    column-box\n    /next-line lpc def\n    cliptocolumn     "
+  "   % MUST match\n  } if\n  next-line lpp ge {\n    % We just went to the"
+  " top of column 3; start new page column 1\n    endcliptocolumn     % MUS"
+  "T match\n    column-box\n    showpage\n    Hfont setfont\n    /next-line"
+  " 0 def\n    show-pageno\n    cliptocolumn        % MUST match\n  } if\n}"
+  " def\n\n/IncrementLineOutside {\n  /incr exch def\n  /next-line next-lin"
+  "e incr add def\n  next-line lpc ge   next-line incr sub lpc lt   and {\n"
+  "    % We just went to the top of column 2\n    /next-line lpc def\n  } i"
+  "f\n  next-line lpp ge {\n    % We just went to the top of column 3; star"
+  "t new page column 1\n    showpage\n    Hfont setfont\n    /next-line 0 d"
+  "ef\n    show-pageno\n  } if\n} def\n\n/NextColumn {\n  lpc 1 sub Increme"
+  "ntLine\n} def\n\n/NextPage {\n    lpp 1 sub IncrementLine\n} def\n\n% Up"
+  "on entry, we are OUTSIDE the clip\n/start-detail {\n  /d-title exch def\n"
+  "\n  % align >= 1 inch at bottom of column, and/or start new column\n  lp"
+  "c next-line lpc mod sub widowlines lt {\n    % Start at top of a column\n"
+  "    next-line lpc ge {\n      % Start new page\n      showpage\n      Hf"
+  "ont setfont\n      /next-line 0 def\n      show-pageno\n    } {\n      %"
+  " Start new column\n      /next-line lpc def\n    } ifelse\n  } if\n\n  l"
+  "margin tmargin moveto\n  next-line lpc idiv delc mul  next-line lpc mod "
+  "dell mul rmoveto\n  gsave d-title show grestore\n  0 dell rmoveto\n  1 1"
+  " 4 {/j exch def gsave probw j mul 5 div -2 rmoveto 50 j 10 mul sub 20 st"
+  "ring cvs cshow grestore} for\n  2 IncrementLineOutside\n  /detail-count "
+  "1 def\n  cliptocolumn            % MUST match\n  /d-array [] def\n} def\n"
+  "\n/size-detail {\n  /d-names exch def\n  /d-size exch def\n  % zero sums"
+  "\n  /sums d-size array def\n  0 1 d-size 1 sub {/i exch def  sums i 0 pu"
+  "t} for\n  /old-d-max 0 def\n  /colors d-size array def\n  0 1 d-size 1 s"
+  "ub {/i exch def  colors i i 3 mul 17 mod 17 div put} for\n  %0 1 d-size "
+  "1 sub {/i exch def  colors i d-names i get strhash 3 mul 17 mod 17 div p"
+  "ut} for\n  %0 1 d-size 1 sub {/i exch def  ( ) show colors i get 20 stri"
+  "ng cvs show} for\n} def\n\n/count-detail {\n  /detail-total-count exch d"
+  "ef\n  % if total-count >= one column, start at top of a column\n  detail"
+  "-total-count lpp ge {\n    % Start new page\n    NextPage\n  } {\n    de"
+  "tail-total-count lpc ge {\n      % Start new column\n      NextColumn\n "
+  "   } if\n  } ifelse\n} def\n\n% highlight next entry with underbar\n/do-"
+  "flag {\ngsave\n  setrgbcolor\n  lmargin tmargin moveto\n  next-line lpc "
+  "idiv delc mul  next-line lpc mod dell mul rmoveto\n  0 -2 rmoveto\n  pro"
+  "bw 0 rlineto\n  0 dell neg rlineto\n  probw neg 0 rlineto\n  closepath\n"
+  "  fill\ngrestore\n} def\n\n/do-detail-e {\n  /d-array exch def\n  /d-enc"
+  " exch def\n  /d-label exch def\n  /d-max -999999 def\n\n  lmargin tmargi"
+  "n moveto\n  next-line lpc idiv delc mul  next-line lpc mod dell mul rmov"
+  "eto\n  0.25 setlinewidth\n\n  % show label, using encoding color\n  gsav"
+  "e\n    probw 2 add -2 rmoveto\n    detail-count 1 sub 2 mod 0.25 inch mu"
+  "l 0 rmoveto\n    % ([) show detail-count 20 string cvs show (] ) show\n "
+  "   d-enc 0 lt {\n      0 setgray\n    }{\n      colors d-enc get 1 .8 se"
+  "thsbcolor\n    } ifelse\n    d-label show\n  grestore\n  % For -prune- d"
+  "raw horizontal line\n  d-label length 8 gt {d-label 4 get (p) 0 get eq d"
+  "-label 5 get (r) 0 get eq and {\n    /prune-val d-label cvi def\n    /ne"
+  "wx prune-val 30 mul prob2x def\n    gsave newx 6 rmoveto 0 -12 rlineto 1"
+  ".5 setlinewidth 0.8 setgray stroke grestore\n    gsave probw 0 add 0 rli"
+  "neto 0.25 setlinewidth 0.8 setgray stroke grestore\n  } if } if\n\n  % t"
+  "rack max per new row\n  0 1 d-array length 1 sub {\n    /i exch def\n   "
+  " /sum sums i get d-array i get add def\n    d-max sum lt {/d-max sum def"
+  "} if\n  } for\n\n  % draw line increments\n  0 1 d-array length 1 sub {\n"
+  "    /i exch def\n    detail-count 1 gt {\n      /oldx old-d-max sums i g"
+  "et sub prob2x def\n    } {\n      /oldx 600 prob2x def\n    } ifelse\n  "
+  "  /oldy dell neg def\n    /newx d-max  sums i get d-array i get add  sub"
+  " prob2x def\n    /newy 0 def\n    gsave\n    oldx oldy rmoveto\n    newx"
+  " oldx sub  newy oldy sub rlineto\n    % if encoding is being tracked, ma"
+  "ke bold line\n    i track-me eq\n      {2 setlinewidth}\n      {i track-"
+  "me2 eq {1.25 setlinewidth} {0.25 setlinewidth} ifelse}\n      ifelse\n  "
+  "  colors i get 1 .8 sethsbcolor stroke\n    grestore\n  } for\n  /detail"
+  "-count detail-count 1 add def\n\n  % increment running total in sums, tr"
+  "ack max per row\n  0 1 d-array length 1 sub {\n    /i exch def\n    sums"
+  " i  sums i get d-array i get add  put\n  } for\n  /old-d-max d-max def\n"
+  "\n  1 IncrementLine\n} def\n\n\n/do-detail {\n  /d-array exch def\n  /d-"
+  "label exch def\n  d-label -1 d-array do-detail-e\n} def\n\n% Upon exit, "
+  "we are outside the clip\n/end-detail {\n  pop\n  endcliptocolumn        "
+  " % MUST match\n  column-box\n\n  % text labels\n  0 1 d-array length 1 s"
+  "ub {\n    /i exch def\n    gsave\n    /newx old-d-max sums i get sub pro"
+  "b2x def\n    newx 0 ge {\n      newx 0 rmoveto\n      currentpoint trans"
+  "late\n      colors i get 1 .8 sethsbcolor\n      gsave 0 dell neg  rline"
+  "to 0.25 setlinewidth stroke grestore\n      -60 rotate\n      0 -2 movet"
+  "o d-names i get show\n    } if\n    grestore\n  } for\n  d-array length "
+  "0 gt {\n    lpg IncrementLineOutside\n  } {\n    lpg 4 idiv IncrementLin"
+  "eOutside\n  } ifelse\n} def\n\n/do-src {\n/src exch def\n  lmargin tmarg"
+  "in moveto\n  next-line lpc idiv delc mul  next-line lpc mod dell mul rmo"
+  "veto\n  Cfont setfont\n  src show\n  Hfont setfont\n  1 IncrementLine\n}"
+  " def\n\n% Underline trigram in source text\n/do-highlight1 {\n  /hl-colo"
+  "r exch def\n  /hl-offset exch def\n  /hl-line exch 1 sub 2 mul def\n  gs"
+  "ave\n  lmargin tmargin moveto\n  next-line hl-line sub lpc idiv delc mul"
+  "\n  next-line hl-line sub lpc mod dell mul rmoveto\n  % Assume text is 6"
+  " chars in and 3.6 pts per char, but 2 chars per offset\n  hl-offset 2 mu"
+  "l 6 add 3.6 mul  4 rmoveto\n\n  0 setgray 0.5 setlinewidth\n  hl-color 1"
+  " eq {0 0 1 setrgbcolor} if  % Latin1 blue\n  hl-color 2 eq {1 0 1 setrgb"
+  "color} if  % Latin2 magenta\n  hl-color 3 eq {1 0.67 0 setrgbcolor} if  "
+  "% Latin7 orange\n  18 -2 rlineto stroke\n  grestore\n} def\n\n% Box trig"
+  "ram in source text\n/do-highlight2 {\n  /hl-color exch def\n  /hl-offset"
+  " exch def\n  /hl-line exch 1 sub 2 mul def\n  gsave\n  lmargin tmargin m"
+  "oveto\n  next-line hl-line sub lpc idiv delc mul\n  next-line hl-line su"
+  "b lpc mod dell mul rmoveto\n  % Assume text is 6 chars in and 3.6 pts pe"
+  "r char, but 2 chars per offset\n  hl-offset 2 mul 6 add 3.6 mul  4 rmove"
+  "to\n\n  0 setgray 0.25 setlinewidth\n  hl-color 1 eq {0 0 1 setrgbcolor}"
+  " if  % Latin1 blue\n  hl-color 2 eq {1 0 1 setrgbcolor} if  % Latin2 mag"
+  "enta\n  hl-color 3 eq {1 0.67 0 setrgbcolor} if  % Latin7 orange\n  -0.5"
+  " -0.5 rmoveto\n  22 0 rlineto\n  0 4 rlineto\n  -11 2 rlineto\n  -11 -2 "
+  "rlineto\n  closepath\n  stroke\n  grestore\n} def\n\n/show-pageno {\ngsa"
+  "ve\nlmargin bmargin moveto 0 -12 rmoveto\n(Page ) show pageno 20 string "
+  "cvs show\ngrestore\n/pageno pageno 1 add def\n} def\n\n/pageno 1 def\nsh"
+  "ow-pageno\n%=============================\n\n\n"

contrib/google-ced/util/basictypes.h

@@ -0,0 +1,331 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_BASICTYPES_H_
+#define UTIL_BASICTYPES_H_
+
+#include <limits.h>         // So we can set the bounds of our types
+#include <stddef.h>         // For size_t
+#include <string.h>         // for memcpy
+
+#include "util/port.h"    // Types that only need exist on certain systems
+
+#ifndef COMPILER_MSVC
+// stdint.h is part of C99 but MSVC doesn't have it.
+#include <stdint.h>         // For intptr_t.
+#endif
+
+typedef signed char         schar;
+typedef signed char         int8;
+typedef short               int16;
+// TODO(mbelshe) Remove these type guards.  These are
+//               temporary to avoid conflicts with npapi.h.
+#ifndef _INT32
+#define _INT32
+typedef int                 int32;
+#endif
+
+// The NSPR system headers define 64-bit as |long| when possible.  In order to
+// not have typedef mismatches, we do the same on LP64.
+#if __LP64__
+typedef long                int64;
+#else
+typedef long long           int64;
+#endif
+
+// NOTE: unsigned types are DANGEROUS in loops and other arithmetical
+// places.  Use the signed types unless your variable represents a bit
+// pattern (eg a hash value) or you really need the extra bit.  Do NOT
+// use 'unsigned' to express "this value should always be positive";
+// use assertions for this.
+
+typedef unsigned char      uint8;
+typedef unsigned short     uint16;
+// TODO(mbelshe) Remove these type guards.  These are
+//               temporary to avoid conflicts with npapi.h.
+#ifndef _UINT32
+#define _UINT32
+typedef unsigned int       uint32;
+#endif
+
+// See the comment above about NSPR and 64-bit.
+#if __LP64__
+typedef unsigned long uint64;
+#else
+typedef unsigned long long uint64;
+#endif
+
+// A type to represent a Unicode code-point value. As of Unicode 4.0,
+// such values require up to 21 bits.
+// (For type-checking on pointers, make this explicitly signed,
+// and it should always be the signed version of whatever int32 is.)
+typedef signed int         char32;
+
+const uint8  kuint8max  = (( uint8) 0xFF);
+const uint16 kuint16max = ((uint16) 0xFFFF);
+const uint32 kuint32max = ((uint32) 0xFFFFFFFF);
+const uint64 kuint64max = ((uint64) GG_LONGLONG(0xFFFFFFFFFFFFFFFF));
+const  int8  kint8min   = ((  int8) 0x80);
+const  int8  kint8max   = ((  int8) 0x7F);
+const  int16 kint16min  = (( int16) 0x8000);
+const  int16 kint16max  = (( int16) 0x7FFF);
+const  int32 kint32min  = (( int32) 0x80000000);
+const  int32 kint32max  = (( int32) 0x7FFFFFFF);
+const  int64 kint64min  = (( int64) GG_LONGLONG(0x8000000000000000));
+const  int64 kint64max  = (( int64) GG_LONGLONG(0x7FFFFFFFFFFFFFFF));
+
+// A macro to disallow the copy constructor and operator= functions
+// This should be used in the private: declarations for a class
+#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
+  TypeName(const TypeName&);               \
+  void operator=(const TypeName&)
+
+// An older, deprecated, politically incorrect name for the above.
+#define DISALLOW_EVIL_CONSTRUCTORS(TypeName) DISALLOW_COPY_AND_ASSIGN(TypeName)
+
+// A macro to disallow all the implicit constructors, namely the
+// default constructor, copy constructor and operator= functions.
+//
+// This should be used in the private: declarations for a class
+// that wants to prevent anyone from instantiating it. This is
+// especially useful for classes containing only static methods.
+#define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \
+  TypeName();                                    \
+  DISALLOW_COPY_AND_ASSIGN(TypeName)
+
+// The arraysize(arr) macro returns the # of elements in an array arr.
+// The expression is a compile-time constant, and therefore can be
+// used in defining new arrays, for example.  If you use arraysize on
+// a pointer by mistake, you will get a compile-time error.
+
+// This template function declaration is used in defining arraysize.
+// Note that the function doesn't need an implementation, as we only
+// use its type.
+template <typename T, size_t N>
+char (&ArraySizeHelper(T (&array)[N]))[N];
+
+// That gcc wants both of these prototypes seems mysterious. VC, for
+// its part, can't decide which to use (another mystery). Matching of
+// template overloads: the final frontier.
+#ifndef _MSC_VER
+template <typename T, size_t N>
+char (&ArraySizeHelper(const T (&array)[N]))[N];
+#endif
+
+#define arraysize(array) (sizeof(ArraySizeHelper(array)))
+
+
+// Use implicit_cast as a safe version of static_cast or const_cast
+// for upcasting in the type hierarchy (i.e. casting a pointer to Foo
+// to a pointer to SuperclassOfFoo or casting a pointer to Foo to
+// a const pointer to Foo).
+// When you use implicit_cast, the compiler checks that the cast is safe.
+// Such explicit implicit_casts are necessary in surprisingly many
+// situations where C++ demands an exact type match instead of an
+// argument type convertable to a target type.
+//
+// The From type can be inferred, so the preferred syntax for using
+// implicit_cast is the same as for static_cast etc.:
+//
+//   implicit_cast<ToType>(expr)
+//
+// implicit_cast would have been part of the C++ standard library,
+// but the proposal was submitted too late.  It will probably make
+// its way into the language in the future.
+template<typename To, typename From>
+inline To implicit_cast(From const &f) {
+  return f;
+}
+
+// The COMPILE_ASSERT macro can be used to verify that a compile time
+// expression is true. For example, you could use it to verify the
+// size of a static array:
+//
+//   COMPILE_ASSERT(arraysize(content_type_names) == CONTENT_NUM_TYPES,
+//                  content_type_names_incorrect_size);
+//
+// or to make sure a struct is smaller than a certain size:
+//
+//   COMPILE_ASSERT(sizeof(foo) < 128, foo_too_large);
+//
+// The second argument to the macro is the name of the variable. If
+// the expression is false, most compilers will issue a warning/error
+// containing the name of the variable.
+
+template <bool>
+struct CompileAssert {
+};
+
+#undef COMPILE_ASSERT
+#define COMPILE_ASSERT(expr, msg) \
+  typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1]
+
+// Implementation details of COMPILE_ASSERT:
+//
+// - COMPILE_ASSERT works by defining an array type that has -1
+//   elements (and thus is invalid) when the expression is false.
+//
+// - The simpler definition
+//
+//     #define COMPILE_ASSERT(expr, msg) typedef char msg[(expr) ? 1 : -1]
+//
+//   does not work, as gcc supports variable-length arrays whose sizes
+//   are determined at run-time (this is gcc's extension and not part
+//   of the C++ standard).  As a result, gcc fails to reject the
+//   following code with the simple definition:
+//
+//     int foo;
+//     COMPILE_ASSERT(foo, msg); // not supposed to compile as foo is
+//                               // not a compile-time constant.
+//
+// - By using the type CompileAssert<(bool(expr))>, we ensures that
+//   expr is a compile-time constant.  (Template arguments must be
+//   determined at compile-time.)
+//
+// - The outter parentheses in CompileAssert<(bool(expr))> are necessary
+//   to work around a bug in gcc 3.4.4 and 4.0.1.  If we had written
+//
+//     CompileAssert<bool(expr)>
+//
+//   instead, these compilers will refuse to compile
+//
+//     COMPILE_ASSERT(5 > 0, some_message);
+//
+//   (They seem to think the ">" in "5 > 0" marks the end of the
+//   template argument list.)
+//
+// - The array size is (bool(expr) ? 1 : -1), instead of simply
+//
+//     ((expr) ? 1 : -1).
+//
+//   This is to avoid running into a bug in MS VC 7.1, which
+//   causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1.
+
+
+// MetatagId refers to metatag-id that we assign to
+// each metatag <name, value> pair..
+typedef uint32 MetatagId;
+
+// Argument type used in interfaces that can optionally take ownership
+// of a passed in argument.  If TAKE_OWNERSHIP is passed, the called
+// object takes ownership of the argument.  Otherwise it does not.
+enum Ownership {
+  DO_NOT_TAKE_OWNERSHIP,
+  TAKE_OWNERSHIP
+};
+
+// bit_cast<Dest,Source> is a template function that implements the
+// equivalent of "*reinterpret_cast<Dest*>(&source)".  We need this in
+// very low-level functions like the protobuf library and fast math
+// support.
+//
+//   float f = 3.14159265358979;
+//   int i = bit_cast<int32>(f);
+//   // i = 0x40490fdb
+//
+// The classical address-casting method is:
+//
+//   // WRONG
+//   float f = 3.14159265358979;            // WRONG
+//   int i = * reinterpret_cast<int*>(&f);  // WRONG
+//
+// The address-casting method actually produces undefined behavior
+// according to ISO C++ specification section 3.10 -15 -.  Roughly, this
+// section says: if an object in memory has one type, and a program
+// accesses it with a different type, then the result is undefined
+// behavior for most values of "different type".
+//
+// This is true for any cast syntax, either *(int*)&f or
+// *reinterpret_cast<int*>(&f).  And it is particularly true for
+// conversions betweeen integral lvalues and floating-point lvalues.
+//
+// The purpose of 3.10 -15- is to allow optimizing compilers to assume
+// that expressions with different types refer to different memory.  gcc
+// 4.0.1 has an optimizer that takes advantage of this.  So a
+// non-conforming program quietly produces wildly incorrect output.
+//
+// The problem is not the use of reinterpret_cast.  The problem is type
+// punning: holding an object in memory of one type and reading its bits
+// back using a different type.
+//
+// The C++ standard is more subtle and complex than this, but that
+// is the basic idea.
+//
+// Anyways ...
+//
+// bit_cast<> calls memcpy() which is blessed by the standard,
+// especially by the example in section 3.9 .  Also, of course,
+// bit_cast<> wraps up the nasty logic in one place.
+//
+// Fortunately memcpy() is very fast.  In optimized mode, with a
+// constant size, gcc 2.95.3, gcc 4.0.1, and msvc 7.1 produce inline
+// code with the minimal amount of data movement.  On a 32-bit system,
+// memcpy(d,s,4) compiles to one load and one store, and memcpy(d,s,8)
+// compiles to two loads and two stores.
+//
+// I tested this code with gcc 2.95.3, gcc 4.0.1, icc 8.1, and msvc 7.1.
+//
+// WARNING: if Dest or Source is a non-POD type, the result of the memcpy
+// is likely to surprise you.
+
+template <class Dest, class Source>
+inline Dest bit_cast(const Source& source) {
+  // Compile time assertion: sizeof(Dest) == sizeof(Source)
+  // A compile error here means your Dest and Source have different sizes.
+  // typedef char VerifySizesAreEqual [sizeof(Dest) == sizeof(Source) ? 1 : -1];
+
+  Dest dest;
+  memcpy(&dest, &source, sizeof(dest));
+  return dest;
+}
+
+// The following enum should be used only as a constructor argument to indicate
+// that the variable has static storage class, and that the constructor should
+// do nothing to its state.  It indicates to the reader that it is legal to
+// declare a static instance of the class, provided the constructor is given
+// the base::LINKER_INITIALIZED argument.  Normally, it is unsafe to declare a
+// static variable that has a constructor or a destructor because invocation
+// order is undefined.  However, IF the type can be initialized by filling with
+// zeroes (which the loader does for static variables), AND the destructor also
+// does nothing to the storage, AND there are no virtual methods, then a
+// constructor declared as
+//       explicit MyClass(base::LinkerInitialized x) {}
+// and invoked as
+//       static MyClass my_variable_name(base::LINKER_INITIALIZED);
+namespace base {
+enum LinkerInitialized { LINKER_INITIALIZED };
+}  // base
+
+// UnaligndLoad32 is put here instead of util/port.h to
+// avoid the circular dependency between port.h and basictypes.h
+// ARM does not support unaligned memory access.
+#if defined(ARCH_CPU_X86_FAMILY)
+// x86 and x86-64 can perform unaligned loads/stores directly;
+inline uint32 UnalignedLoad32(const void* p) {
+  return *reinterpret_cast<const uint32*>(p);
+}
+#else
+#define NEED_ALIGNED_LOADS
+// If target architecture does not support unaligned loads and stores,
+// use memcpy version of UNALIGNED_LOAD32.
+inline uint32 UnalignedLoad32(const void* p) {
+  uint32 t;
+  memcpy(&t, reinterpret_cast<const uint8*>(p), sizeof(t));
+  return t;
+}
+
+#endif
+#endif  // UTIL_BASICTYPES_H_

contrib/google-ced/util/case_insensitive_hash.h

@@ -0,0 +1,88 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_CASE_INSENSITIVE_HASH_H_
+#define UTIL_CASE_INSENSITIVE_HASH_H_
+
+#include <ctype.h>
+#include <stddef.h>
+#ifndef _MSC_VER
+#include <strings.h>
+#endif
+
+#include <string>
+
+#include "util/basictypes.h"
+#include "util/string_util.h"
+
+// Functors for hashing c-strings with case-insensitive semantics.
+struct CStringCaseHash {
+  size_t operator()(const char *str) const {
+    unsigned long hash_val = 0;
+    while (*str) {
+      hash_val = 5*hash_val + tolower(*str);
+      str++;
+    }
+    return (size_t)hash_val;
+  }
+};
+
+struct CStringCaseEqual {
+  bool operator()(const char *str1, const char *str2) const {
+    return !base::strcasecmp(str1, str2);
+  }
+};
+
+// These functors, in addition to being case-insensitive, ignore all
+// non-alphanumeric characters.  This is useful when we want all variants of
+// a string -- where variants can differ in puncutation and whitespace -- to
+// map to the same value.
+struct CStringAlnumCaseHash {
+  size_t operator()(const char *str) const {
+    unsigned long hash_val = 0;
+    while (*str) {
+      if (isalnum(*str)) {
+        hash_val = 5*hash_val + tolower(*str);
+      }
+      str++;
+    }
+    return (size_t)hash_val;
+  }
+};
+
+struct CStringAlnumCaseEqual {
+  bool operator()(const char *str1, const char *str2) const {
+    while (true) {
+      // Skip until each pointer is pointing to an alphanumeric char or '\0'
+      while (!isalnum(*str1) && (*str1 != '\0')) {
+        str1++;
+      }
+      while (!isalnum(*str2) && (*str2 != '\0')) {
+        str2++;
+      }
+      if (tolower(*str1) != tolower(*str2)) {
+        return false;       // mismatch on alphanumeric char or '\0'
+      }
+      if (*str1 == '\0') {  // in which case *str2 must be '\0' as well
+        return true;        // reached '\0' in both strings without mismatch
+      }
+      str1++;
+      str2++;
+    }
+  }
+};
+
+#endif  // UTIL_CASE_INSENSITIVE_HASH_H_

contrib/google-ced/util/commandlineflags.h

@@ -0,0 +1,39 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_COMMANDLINEFLAGS_H_
+#define UTIL_COMMANDLINEFLAGS_H_
+
+
+#undef DEFINE_bool
+#define DEFINE_bool(name, default_value, comment) \
+    bool FLAGS_##name = default_value
+#undef DEFINE_int32
+#define DEFINE_int32(name, default_value, comment) \
+    int32 FLAGS_##name = default_value
+#undef DEFINE_string
+#define DEFINE_string(name, default_value, comment) \
+    string FLAGS_##name = default_value
+
+#undef DECLARE_bool
+#define DECLARE_bool(name) extern bool FLAGS_##name
+#undef DECLARE_int32
+#define DECLARE_int32(name) extern int32 FLAGS_##name
+#undef DECLARE_string
+#define DECLARE_string(name) extern string FLAGS_##name
+
+
+#endif  // UTIL_COMMANDLINEFLAGS_H_

contrib/google-ced/util/encodings/encodings.cc

@@ -0,0 +1,891 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/encodings/encodings.h"
+
+#include <string.h>                     // for strcasecmp
+#include <unordered_map>
+#include <utility>                      // for pair
+
+#include "util/basictypes.h"
+#include "util/string_util.h"
+#include "util/case_insensitive_hash.h"
+
+struct EncodingInfo {
+  // The standard name for this encoding.
+  //
+  const char* encoding_name_;
+
+  // The "preferred MIME name" of an encoding as specified by the IANA at:
+  //     http://www.iana.org/assignments/character-sets
+  //
+  //   Note that the preferred MIME name may differ slightly from the
+  //   official IANA name: i.e. ISO-8859-1 vs. ISO_8859-1:1987
+  //
+  const char* mime_encoding_name_;
+
+  // It is an internal policy that if an encoding has an IANA name,
+  // then encoding_name_ and mime_encoding_name_ must be the same string.
+  //
+  // However, there can be exceptions if there are compelling reasons.
+  // For example, Japanese mobile handsets require the name
+  // "Shift_JIS" in charset=... parameter in Content-Type headers to
+  // process emoji (emoticons) in their private encodings.  In that
+  // case, mime_encoding_name_ should be "Shift_JIS", despite
+  // encoding_name_ actually is "X-KDDI-Shift_JIS".
+
+  // Some multi-byte encodings use byte values that coincide with the
+  // ASCII codes for HTML syntax characters <>"&' and browsers like MSIE
+  // can misinterpret these, as indicated in an external XSS report from
+  // 2007-02-15. Here, we map these dangerous encodings to safer ones. We
+  // also use UTF8 instead of encodings that we don't support in our
+  // output, and we generally try to be conservative in what we send out.
+  // Where the client asks for single- or double-byte encodings that are
+  // not as common, we substitute a more common single- or double-byte
+  // encoding, if there is one, thereby preserving the client's intent
+  // to use less space than UTF-8. This also means that characters
+  // outside the destination set will be converted to HTML NCRs (&#NNN;)
+  // if requested.
+
+  Encoding preferred_web_output_encoding_;
+};
+
+static const EncodingInfo kEncodingInfoTable[] = {
+  { "ASCII", "ISO-8859-1", ISO_8859_1},
+  { "Latin2", "ISO-8859-2", ISO_8859_2},
+  { "Latin3", "ISO-8859-3", UTF8},
+      // MSIE 6 does not support ISO-8859-3 (XSS issue)
+  { "Latin4", "ISO-8859-4", ISO_8859_4},
+  { "ISO-8859-5", "ISO-8859-5", ISO_8859_5},
+  { "Arabic", "ISO-8859-6", ISO_8859_6},
+  { "Greek", "ISO-8859-7", ISO_8859_7},
+  { "Hebrew", "ISO-8859-8", MSFT_CP1255},
+      // we do not endorse the visual order
+  { "Latin5", "ISO-8859-9", ISO_8859_9},
+  { "Latin6", "ISO-8859-10", UTF8},
+      // MSIE does not support ISO-8859-10 (XSS issue)
+  { "EUC-JP",  "EUC-JP", JAPANESE_EUC_JP},
+  { "SJS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+  { "JIS", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
+      // due to potential confusion with HTML syntax chars
+  { "BIG5", "Big5", CHINESE_BIG5},
+  { "GB",  "GB2312", CHINESE_GB},
+  { "EUC-CN",
+        "EUC-CN",
+        // Misnamed. Should be EUC-TW.
+        CHINESE_BIG5},
+      // MSIE treats "EUC-CN" like GB2312, which is not EUC-TW,
+      // and EUC-TW is rare, so we prefer Big5 for output.
+  { "KSC", "EUC-KR", KOREAN_EUC_KR},
+  { "Unicode",
+    "UTF-16LE",
+        // Internet Explorer doesn't recognize "ISO-10646-UCS-2"
+        UTF8
+        // due to potential confusion with HTML syntax chars
+        },
+  { "EUC",
+        "EUC",  // Misnamed. Should be EUC-TW.
+        CHINESE_BIG5
+        // MSIE does not recognize "EUC" (XSS issue),
+        // and EUC-TW is rare, so we prefer Big5 for output.
+        },
+  { "CNS",
+        "CNS",  // Misnamed. Should be EUC-TW.
+        CHINESE_BIG5},
+      // MSIE does not recognize "CNS" (XSS issue),
+      // and EUC-TW is rare, so we prefer Big5 for output.
+  { "BIG5-CP950",
+        "BIG5-CP950",  // Not an IANA name
+        CHINESE_BIG5
+        // MSIE does not recognize "BIG5-CP950" (XSS issue)
+        },
+  { "CP932", "CP932",  // Not an IANA name
+        JAPANESE_SHIFT_JIS},  // MSIE does not recognize "CP932" (XSS issue)
+  { "UTF8", "UTF-8", UTF8},
+  { "Unknown",
+        "x-unknown",  // Not an IANA name
+        UTF8},  // UTF-8 is our default output encoding
+  { "ASCII-7-bit", "US-ASCII", ASCII_7BIT},
+  { "KOI8R", "KOI8-R", RUSSIAN_KOI8_R},
+  { "CP1251", "windows-1251", RUSSIAN_CP1251},
+  { "CP1252", "windows-1252", MSFT_CP1252},
+  { "KOI8U",
+        "KOI8-U",
+        ISO_8859_5},  // because koi8-u is not as common
+  { "CP1250", "windows-1250", MSFT_CP1250},
+  { "ISO-8859-15", "ISO-8859-15", ISO_8859_15},
+  { "CP1254", "windows-1254", MSFT_CP1254},
+  { "CP1257", "windows-1257", MSFT_CP1257},
+  { "ISO-8859-11", "ISO-8859-11", ISO_8859_11},
+  { "CP874", "windows-874", MSFT_CP874},
+  { "CP1256", "windows-1256", MSFT_CP1256},
+  { "CP1255", "windows-1255", MSFT_CP1255},
+  { "ISO-8859-8-I", "ISO-8859-8-I", MSFT_CP1255},
+      // Java does not support iso-8859-8-i
+  { "VISUAL", "ISO-8859-8", MSFT_CP1255},
+      // we do not endorse the visual order
+  { "CP852", "cp852", MSFT_CP1250},
+      // because cp852 is not as common
+  { "CSN_369103", "csn_369103", MSFT_CP1250},
+      // MSIE does not recognize "csn_369103" (XSS issue)
+  { "CP1253", "windows-1253", MSFT_CP1253},
+  { "CP866", "IBM866", RUSSIAN_CP1251},
+      // because cp866 is not as common
+  { "ISO-8859-13", "ISO-8859-13", UTF8},
+      // because iso-8859-13 is not widely supported
+  { "ISO-2022-KR", "ISO-2022-KR", KOREAN_EUC_KR},
+      // due to potential confusion with HTML syntax chars
+  { "GBK", "GBK", GBK},
+  { "GB18030", "GB18030", GBK},
+      // because gb18030 is not widely supported
+  { "BIG5_HKSCS", "BIG5-HKSCS", CHINESE_BIG5},
+      // because Big5-HKSCS is not widely supported
+  { "ISO_2022_CN", "ISO-2022-CN", CHINESE_GB},
+      // due to potential confusion with HTML syntax chars
+  { "TSCII", "tscii", UTF8},
+      // we do not have an output converter for this font encoding
+  { "TAM", "tam", UTF8},
+      // we do not have an output converter for this font encoding
+  { "TAB", "tab", UTF8},
+      // we do not have an output converter for this font encoding
+  { "JAGRAN", "jagran", UTF8},
+      // we do not have an output converter for this font encoding
+  { "MACINTOSH", "MACINTOSH", ISO_8859_1},
+      // because macintosh is relatively uncommon
+  { "UTF7", "UTF-7",
+        UTF8},  // UTF-7 has been the subject of XSS attacks and is deprecated
+  { "BHASKAR", "bhaskar",
+        UTF8},  // we do not have an output converter for this font encoding
+  { "HTCHANAKYA", "htchanakya",  // not an IANA charset name.
+        UTF8},  // we do not have an output converter for this font encoding
+  { "UTF-16BE", "UTF-16BE",
+        UTF8},  // due to potential confusion with HTML syntax chars
+  { "UTF-16LE", "UTF-16LE",
+        UTF8},  // due to potential confusion with HTML syntax chars
+  { "UTF-32BE", "UTF-32BE",
+        UTF8},  // unlikely to cause XSS bugs, but very uncommon on Web
+  { "UTF-32LE", "UTF-32LE",
+        UTF8},  // unlikely to cause XSS bugs, but very uncommon on Web
+  { "X-BINARYENC", "x-binaryenc",  // Not an IANA name
+        UTF8},  // because this one is not intended for output (just input)
+  { "HZ-GB-2312", "HZ-GB-2312",
+        CHINESE_GB},  // due to potential confusion with HTML syntax chars
+  { "X-UTF8UTF8", "x-utf8utf8",  // Not an IANA name
+        UTF8},  // because this one is not intended for output (just input)
+  { "X-TAM-ELANGO", "x-tam-elango",
+        UTF8},  // we do not have an output converter for this font encoding
+  { "X-TAM-LTTMBARANI", "x-tam-lttmbarani",
+        UTF8},  // we do not have an output converter for this font encoding
+  { "X-TAM-SHREE", "x-tam-shree",
+        UTF8},  // we do not have an output converter for this font encoding
+  { "X-TAM-TBOOMIS", "x-tam-tboomis",
+        UTF8},  // we do not have an output converter for this font encoding
+  { "X-TAM-TMNEWS", "x-tam-tmnews",
+        UTF8},  // we do not have an output converter for this font encoding
+  { "X-TAM-WEBTAMIL", "x-tam-webtamil",
+        UTF8},  // we do not have an output converter for this font encoding
+
+  { "X-KDDI-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+      // KDDI version of Shift_JIS with Google Emoji PUA mappings.
+      // Note that MimeEncodingName() returns "Shift_JIS", since KDDI uses
+      // "Shift_JIS" in HTTP headers and email messages.
+
+  { "X-DoCoMo-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+      // DoCoMo version of Shift_JIS with Google Emoji PUA mappings.
+      // See the comment at KDDI_SHIFT_JIS for other issues.
+
+  { "X-SoftBank-Shift_JIS", "Shift_JIS", JAPANESE_SHIFT_JIS},
+      // SoftBank version of Shift_JIS with Google Emoji PUA mappings.
+      // See the comment at KDDI_SHIFT_JIS for other issues.
+
+  { "X-KDDI-ISO-2022-JP", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
+      // KDDI version of ISO-2022-JP with Google Emoji PUA mappings.
+      // See the comment at KDDI_SHIFT_JIS for other issues.
+      // The preferred Web encoding is due to potential confusion with
+      // HTML syntax chars.
+
+  { "X-SoftBank-ISO-2022-JP", "ISO-2022-JP", JAPANESE_SHIFT_JIS},
+      // SoftBank version of ISO-2022-JP with Google Emoji PUA mappings.
+      // See the comment at KDDI_SHIFT_JIS for other issues.
+      // The preferred Web encoding is due to potential confusion with
+      // HTML syntax chars.
+
+      // Please refer to NOTE: section in the comments in the definition
+      // of "struct I18NInfoByEncoding", before adding new encodings.
+
+};
+
+
+
+COMPILE_ASSERT(arraysize(kEncodingInfoTable) == NUM_ENCODINGS,
+               kEncodingInfoTable_has_incorrect_size);
+
+Encoding default_encoding() {return LATIN1;}
+
+// *************************************************************
+// Encoding predicates
+//   IsValidEncoding()
+//   IsEncEncCompatible
+//   IsEncodingWithSupportedLanguage
+//   IsSupersetOfAscii7Bit
+//   Is8BitEncoding
+//   IsCJKEncoding
+//   IsHebrewEncoding
+//   IsRightToLeftEncoding
+//   IsLogicalRightToLeftEncoding
+//   IsVisualRightToLeftEncoding
+//   IsIso2022Encoding
+//   IsIso2022JpOrVariant
+//   IsShiftJisOrVariant
+//   IsJapaneseCellPhoneCarrierSpecificEncoding
+// *************************************************************
+
+bool IsValidEncoding(Encoding enc) {
+  return ((enc >= 0) && (enc < kNumEncodings));
+}
+
+bool IsEncEncCompatible(const Encoding from, const Encoding to) {
+  // Tests compatibility between the "from" and "to" encodings; in
+  // the typical case -- when both are valid known encodings -- this
+  // returns true iff converting from first to second is a no-op.
+  if (!IsValidEncoding(from) || !IsValidEncoding(to)) {
+    return false;  // we only work with valid encodings...
+  } else if (to == from) {
+    return true;   // the trivial common case
+  }
+
+  if (to == UNKNOWN_ENCODING) {
+    return true;   // all valid encodings are compatible with the unknown
+  }
+
+  if (from == UNKNOWN_ENCODING) {
+    return false;  // no unknown encoding is compatible with one that is
+  }
+
+  if (from == ASCII_7BIT) {
+    return IsSupersetOfAscii7Bit(to);
+  }
+
+  return (from == ISO_8859_1 && to == MSFT_CP1252) ||
+         (from == ISO_8859_8 && to == HEBREW_VISUAL) ||
+         (from == HEBREW_VISUAL && to == ISO_8859_8) ||
+         (from == ISO_8859_9 && to == MSFT_CP1254) ||
+         (from == ISO_8859_11 && to == MSFT_CP874) ||
+         (from == JAPANESE_SHIFT_JIS && to == JAPANESE_CP932) ||
+         (from == CHINESE_BIG5 && to == CHINESE_BIG5_CP950) ||
+         (from == CHINESE_GB && to == GBK) ||
+         (from == CHINESE_GB && to == GB18030) ||
+         (from == CHINESE_EUC_CN && to == CHINESE_EUC_DEC) ||
+         (from == CHINESE_EUC_CN && to == CHINESE_CNS) ||
+         (from == CHINESE_EUC_DEC && to == CHINESE_EUC_CN) ||
+         (from == CHINESE_EUC_DEC && to == CHINESE_CNS) ||
+         (from == CHINESE_CNS && to == CHINESE_EUC_CN) ||
+         (from == CHINESE_CNS && to == CHINESE_EUC_DEC);
+}
+
+// To be a superset of 7-bit Ascii means that bytes 0...127 in the given
+// encoding represent the same characters as they do in ISO_8859_1.
+
+// TODO: This list could be expanded.  Many other encodings are supersets
+// of 7-bit Ascii.  In fact, Japanese JIS and Unicode are the only two
+// encodings that I know for a fact should *not* be in this list.
+bool IsSupersetOfAscii7Bit(Encoding e) {
+  switch (e) {
+    case ISO_8859_1:
+    case ISO_8859_2:
+    case ISO_8859_3:
+    case ISO_8859_4:
+    case ISO_8859_5:
+    case ISO_8859_6:
+    case ISO_8859_7:
+    case ISO_8859_8:
+    case ISO_8859_9:
+    case ISO_8859_10:
+    case JAPANESE_EUC_JP:
+    case JAPANESE_SHIFT_JIS:
+    case CHINESE_BIG5:
+    case CHINESE_GB:
+    case CHINESE_EUC_CN:
+    case KOREAN_EUC_KR:
+    case CHINESE_EUC_DEC:
+    case CHINESE_CNS:
+    case CHINESE_BIG5_CP950:
+    case JAPANESE_CP932:
+    case UTF8:
+    case UNKNOWN_ENCODING:
+    case ASCII_7BIT:
+    case RUSSIAN_KOI8_R:
+    case RUSSIAN_CP1251:
+    case MSFT_CP1252:
+    case RUSSIAN_KOI8_RU:
+    case MSFT_CP1250:
+    case ISO_8859_15:
+    case MSFT_CP1254:
+    case MSFT_CP1257:
+    case ISO_8859_11:
+    case MSFT_CP874:
+    case MSFT_CP1256:
+    case MSFT_CP1255:
+    case ISO_8859_8_I:
+    case HEBREW_VISUAL:
+    case CZECH_CP852:
+    case MSFT_CP1253:
+    case RUSSIAN_CP866:
+    case ISO_8859_13:
+    case GBK:
+    case GB18030:
+    case BIG5_HKSCS:
+    case MACINTOSH_ROMAN:
+      return true;
+    default:
+      return false;
+  }
+}
+
+// To be an 8-bit encoding means that there are fewer than 256 symbols.
+// Each byte determines a new character; there are no multi-byte sequences.
+
+// TODO: This list could maybe be expanded.  Other encodings may be 8-bit.
+bool Is8BitEncoding(Encoding e) {
+  switch (e) {
+    case ASCII_7BIT:
+    case ISO_8859_1:
+    case ISO_8859_2:
+    case ISO_8859_3:
+    case ISO_8859_4:
+    case ISO_8859_5:
+    case ISO_8859_6:
+    case ISO_8859_7:
+    case ISO_8859_8:
+    case ISO_8859_8_I:
+    case ISO_8859_9:
+    case ISO_8859_10:
+    case ISO_8859_11:
+    case ISO_8859_13:
+    case ISO_8859_15:
+    case MSFT_CP1252:
+    case MSFT_CP1253:
+    case MSFT_CP1254:
+    case MSFT_CP1255:
+    case MSFT_CP1256:
+    case MSFT_CP1257:
+    case RUSSIAN_KOI8_R:
+    case RUSSIAN_KOI8_RU:
+    case RUSSIAN_CP866:
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool IsCJKEncoding(Encoding e) {
+  switch (e) {
+    case JAPANESE_EUC_JP:
+    case JAPANESE_SHIFT_JIS:
+    case JAPANESE_JIS:
+    case CHINESE_BIG5:
+    case CHINESE_GB:
+    case CHINESE_EUC_CN:
+    case KOREAN_EUC_KR:
+    case CHINESE_EUC_DEC:
+    case CHINESE_CNS:
+    case CHINESE_BIG5_CP950:
+    case JAPANESE_CP932:
+    case ISO_2022_KR:
+    case GBK:
+    case GB18030:
+    case BIG5_HKSCS:
+    case ISO_2022_CN:
+    case HZ_GB_2312:
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool IsHebrewEncoding(Encoding e) {
+  return (e == ISO_8859_8 ||
+          e == ISO_8859_8_I ||
+          e == MSFT_CP1255 ||
+          e == HEBREW_VISUAL);
+}
+
+
+
+bool IsRightToLeftEncoding(Encoding enc) {
+  switch (enc) {
+    case MSFT_CP1255:
+    case MSFT_CP1256:
+    case ARABIC_ENCODING:
+    case HEBREW_ENCODING:
+    case ISO_8859_8_I:
+    case HEBREW_VISUAL:
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool IsLogicalRightToLeftEncoding(Encoding enc) {
+  return IsRightToLeftEncoding(enc) && !IsVisualRightToLeftEncoding(enc);
+}
+
+// Note that despite an RFC to the contrary, ARABIC_ENCODING (ISO-8859-6)
+// is NOT visual.
+bool IsVisualRightToLeftEncoding(Encoding enc) {
+  switch (enc) {
+    case HEBREW_ENCODING:
+    case HEBREW_VISUAL:
+      return true;
+    default:
+      return false;
+  }
+}
+
+
+
+
+
+bool IsIso2022Encoding(Encoding enc) {
+  return (IsIso2022JpOrVariant(enc) ||
+          enc == ISO_2022_KR ||
+          enc == ISO_2022_CN);
+}
+
+bool IsIso2022JpOrVariant(Encoding enc) {
+  return (enc == JAPANESE_JIS ||
+          enc == KDDI_ISO_2022_JP ||
+          enc == SOFTBANK_ISO_2022_JP);
+}
+
+bool IsShiftJisOrVariant(Encoding enc) {
+  return (enc == JAPANESE_SHIFT_JIS ||
+          enc == JAPANESE_CP932 ||
+          enc == KDDI_SHIFT_JIS ||
+          enc == DOCOMO_SHIFT_JIS ||
+          enc == SOFTBANK_SHIFT_JIS);
+}
+
+bool IsJapaneseCellPhoneCarrierSpecificEncoding(Encoding enc) {
+  return (enc == KDDI_ISO_2022_JP ||
+          enc == KDDI_SHIFT_JIS ||
+          enc == DOCOMO_SHIFT_JIS ||
+          enc == SOFTBANK_SHIFT_JIS ||
+          enc == SOFTBANK_ISO_2022_JP);
+}
+
+
+// *************************************************************
+// ENCODING NAMES
+//   EncodingName() [Encoding to name]
+//   MimeEncodingName() [Encoding to name]
+//   EncodingFromName() [name to Encoding]
+//   EncodingNameAliasToEncoding() [name to Encoding]
+//   default_encoding_name()
+//   invalid_encoding_name()
+// *************************************************************
+
+const char * EncodingName(const Encoding enc) {
+  if ( (enc < 0) || (enc >= kNumEncodings) )
+    return invalid_encoding_name();
+  return kEncodingInfoTable[enc].encoding_name_;
+}
+
+// TODO: Unify MimeEncodingName and EncodingName, or determine why
+// such a unification is not possible.
+
+const char * MimeEncodingName(Encoding enc) {
+  if ( (enc < 0) || (enc >= kNumEncodings) )
+    return "";  // TODO: Should this be invalid_encoding_name()?
+  return kEncodingInfoTable[enc].mime_encoding_name_;
+}
+
+bool EncodingFromName(const char* enc_name, Encoding *encoding) {
+  *encoding = UNKNOWN_ENCODING;
+  if ( enc_name == NULL ) return false;
+
+  for ( int i = 0; i < kNumEncodings; i++ ) {
+    if (!base::strcasecmp(enc_name, kEncodingInfoTable[i].encoding_name_) ) {
+      *encoding = static_cast<Encoding>(i);
+      return true;
+    }
+  }
+  return false;
+}
+
+// The encoding_map maps standard and non-standard encoding-names
+// (strings) to Encoding enums. It is used only by
+// EncodingNameAliasToEncoding. Note that the map uses
+// case-insensitive hash and comparison functions.
+
+typedef std::unordered_map<const char *, Encoding,
+           CStringAlnumCaseHash,
+           CStringAlnumCaseEqual> EncodingMap;
+
+static const EncodingMap& GetEncodingMap() {
+  static EncodingMap encoding_map;
+  if (!encoding_map.empty()) {
+    // Already initialized
+    return encoding_map;
+  }
+
+  // Initialize the map with all the "standard" encoding names,
+  // i.e., the ones returned by EncodingName and MimeEncodingName.
+  //
+  // First, add internal encoding names returned by EncodingName().
+  for (int i = 0; i < NUM_ENCODINGS; ++i) {
+    Encoding e = static_cast<Encoding>(i);
+    // Internal encoding names must be unique.
+    // The internal names are guaranteed to be unique by the CHECK_EQ.
+    const char *encoding_name = EncodingName(e);
+    // CHECK_EQ(0, encoding_map.count(encoding_name))
+    //  << "Duplicate found for " << encoding_name;
+    encoding_map[encoding_name] = e;
+  }
+  // Then, add mime encoding names returned by MimeEncodingName().
+  // We don't override existing entries, to give precedence to entries
+  // added earlier.
+  for (int i = 0; i < NUM_ENCODINGS; ++i) {
+    Encoding e = static_cast<Encoding>(i);
+    // Note that MimeEncodingName() can return the same mime encoding
+    // name for different encoding enums like JAPANESE_SHIFT_JIS and
+    // KDDI_SHIFT_JIS.  In that case, the encoding enum first seen
+    // will be the value for the encoding name in the map.
+    const char *mime_encoding_name = MimeEncodingName(e);
+    if (encoding_map.count(mime_encoding_name) == 0) {
+      encoding_map[mime_encoding_name] = e;
+    }
+  }
+
+  // Add some non-standard names: alternate spellings, common typos,
+  // etc. (It does no harm to add names already in the map.) Note
+  // that although the map is case-insensitive, by convention the
+  // keys are written here in lower case. For ease of maintenance,
+  // they are listed in alphabetical order.
+  encoding_map["5601"] = KOREAN_EUC_KR;
+  encoding_map["646"] = ASCII_7BIT;
+  encoding_map["852"] = CZECH_CP852;
+  encoding_map["866"] = RUSSIAN_CP866;
+  encoding_map["8859-1"] = ISO_8859_1;
+  encoding_map["ansi-1251"] = RUSSIAN_CP1251;
+  encoding_map["ansi_x3.4-1968"] = ASCII_7BIT;
+  encoding_map["arabic"] = ISO_8859_6;
+  encoding_map["ascii"] = ISO_8859_1;
+  encoding_map["ascii-7-bit"] = ASCII_7BIT;  // not iana standard
+  encoding_map["asmo-708"] = ISO_8859_6;
+  encoding_map["bhaskar"] = BHASKAR;
+  encoding_map["big5"] = CHINESE_BIG5;
+  encoding_map["big5-cp950"] = CHINESE_BIG5_CP950;  // not iana standard
+  encoding_map["big5-hkscs"] = BIG5_HKSCS;
+  encoding_map["chinese"] = CHINESE_GB;
+  encoding_map["cns"] = CHINESE_CNS;  // not iana standard
+  encoding_map["cns11643"] = CHINESE_CNS;
+  encoding_map["cp1250"] = MSFT_CP1250;  // not iana standard
+  encoding_map["cp1251"] = RUSSIAN_CP1251;  // not iana standard
+  encoding_map["cp1252"] = MSFT_CP1252;  // not iana standard
+  encoding_map["cp1253"] = MSFT_CP1253;  // not iana standard
+  encoding_map["cp1254"] = MSFT_CP1254;  // not iana standard
+  encoding_map["cp1255"] = MSFT_CP1255;
+  encoding_map["cp1256"] = MSFT_CP1256;
+  encoding_map["cp1257"] = MSFT_CP1257;  // not iana standard
+  encoding_map["cp819"] = ISO_8859_1;
+  encoding_map["cp852"] = CZECH_CP852;
+  encoding_map["cp866"] = RUSSIAN_CP866;
+  encoding_map["cp-866"] = RUSSIAN_CP866;
+  encoding_map["cp874"] = MSFT_CP874;
+  encoding_map["cp932"] = JAPANESE_CP932;  // not iana standard
+  encoding_map["cp950"] = CHINESE_BIG5_CP950;   // not iana standard
+  encoding_map["csbig5"] = CHINESE_BIG5;
+  encoding_map["cseucjpkdfmtjapanese"] = JAPANESE_EUC_JP;
+  encoding_map["cseuckr"] = KOREAN_EUC_KR;
+  encoding_map["csgb2312"] = CHINESE_GB;
+  encoding_map["csibm852"] = CZECH_CP852;
+  encoding_map["csibm866"] = RUSSIAN_CP866;
+  encoding_map["csiso2022jp"] = JAPANESE_JIS;
+  encoding_map["csiso2022kr"] = ISO_2022_KR;
+  encoding_map["csiso58gb231280"] = CHINESE_GB;
+  encoding_map["csiso88598i"] = ISO_8859_8_I;
+  encoding_map["csisolatin1"] = ISO_8859_1;
+  encoding_map["csisolatin2"] = ISO_8859_2;
+  encoding_map["csisolatin3"] = ISO_8859_3;
+  encoding_map["csisolatin4"] = ISO_8859_4;
+  encoding_map["csisolatin5"] = ISO_8859_9;
+  encoding_map["csisolatin6"] = ISO_8859_10;
+  encoding_map["csisolatinarabic"] = ISO_8859_6;
+  encoding_map["csisolatincyrillic"] = ISO_8859_5;
+  encoding_map["csisolatingreek"] = ISO_8859_7;
+  encoding_map["csisolatinhebrew"] = ISO_8859_8;
+  encoding_map["csksc56011987"] = KOREAN_EUC_KR;
+  encoding_map["csmacintosh"] = MACINTOSH_ROMAN;
+  encoding_map["csn-369103"] = CZECH_CSN_369103;
+  encoding_map["csshiftjis"] = JAPANESE_SHIFT_JIS;
+  encoding_map["csunicode"] = UTF16BE;
+  encoding_map["csunicode11"] = UTF16BE;
+  encoding_map["csunicode11utf7"] = UTF7;
+  encoding_map["csunicodeascii"] = UTF16BE;
+  encoding_map["csunicodelatin1"] = UTF16BE;
+  encoding_map["cyrillic"] = ISO_8859_5;
+  encoding_map["ecma-114"] = ISO_8859_6;
+  encoding_map["ecma-118"] = ISO_8859_7;
+  encoding_map["elot_928"] = ISO_8859_7;
+  encoding_map["euc"] = CHINESE_EUC_DEC;  // not iana standard
+  encoding_map["euc-cn"] = CHINESE_EUC_CN;  // not iana standard
+  encoding_map["euc-dec"] = CHINESE_EUC_DEC;  // not iana standard
+  encoding_map["euc-jp"] = JAPANESE_EUC_JP;
+  encoding_map["euc-kr"] = KOREAN_EUC_KR;
+  encoding_map["eucgb2312_cn"] = CHINESE_GB;
+  encoding_map["gb"] = CHINESE_GB;  // not iana standard
+  encoding_map["gb18030"] = GB18030;
+  encoding_map["gb2132"] = CHINESE_GB;  // common typo
+  encoding_map["gb2312"] = CHINESE_GB;
+  encoding_map["gb_2312-80"] = CHINESE_GB;
+  encoding_map["gbk"] = GBK;
+  encoding_map["greek"] = ISO_8859_7;
+  encoding_map["greek8"] = ISO_8859_7;
+  encoding_map["hebrew"] = ISO_8859_8;
+  encoding_map["htchanakya"] = HTCHANAKYA;
+  encoding_map["hz-gb-2312"] = HZ_GB_2312;
+  encoding_map["ibm819"] = ISO_8859_1;
+  encoding_map["ibm852"] = CZECH_CP852;
+  encoding_map["ibm874"] = MSFT_CP874;
+  encoding_map["iso-10646"] = UTF16BE;
+  encoding_map["iso-10646-j-1"] = UTF16BE;
+  encoding_map["iso-10646-ucs-2"] = UNICODE;
+  encoding_map["iso-10646-ucs-4"] = UTF32BE;
+  encoding_map["iso-10646-ucs-basic"] = UTF16BE;
+  encoding_map["iso-10646-unicode-latin1"] = UTF16BE;
+  encoding_map["iso-2022-cn"] = ISO_2022_CN;
+  encoding_map["iso-2022-jp"] = JAPANESE_JIS;
+  encoding_map["iso-2022-kr"] = ISO_2022_KR;
+  encoding_map["iso-8559-1"] = ISO_8859_1;   // common typo
+  encoding_map["iso-874"] = MSFT_CP874;
+  encoding_map["iso-8858-1"] = ISO_8859_1;   // common typo
+  // iso-8859-0 was a temporary name, eventually renamed iso-8859-15
+  encoding_map["iso-8859-0"] = ISO_8859_15;
+  encoding_map["iso-8859-1"] = ISO_8859_1;
+  encoding_map["iso-8859-10"] = ISO_8859_10;
+  encoding_map["iso-8859-11"] = ISO_8859_11;
+  encoding_map["iso-8859-13"] = ISO_8859_13;
+  encoding_map["iso-8859-15"] = ISO_8859_15;
+  encoding_map["iso-8859-2"] = ISO_8859_2;
+  encoding_map["iso-8859-3"] = ISO_8859_3;
+  encoding_map["iso-8859-4"] = ISO_8859_4;
+  encoding_map["iso-8859-5"] = ISO_8859_5;
+  encoding_map["iso-8859-6"] = ISO_8859_6;
+  encoding_map["iso-8859-7"] = ISO_8859_7;
+  encoding_map["iso-8859-8"] = ISO_8859_8;
+  encoding_map["iso-8859-8-i"] = ISO_8859_8_I;
+  encoding_map["iso-8859-9"] = ISO_8859_9;
+  encoding_map["iso-9959-1"] = ISO_8859_1;   // common typo
+  encoding_map["iso-ir-100"] = ISO_8859_1;
+  encoding_map["iso-ir-101"] = ISO_8859_2;
+  encoding_map["iso-ir-109"] = ISO_8859_3;
+  encoding_map["iso-ir-110"] = ISO_8859_4;
+  encoding_map["iso-ir-126"] = ISO_8859_7;
+  encoding_map["iso-ir-127"] = ISO_8859_6;
+  encoding_map["iso-ir-138"] = ISO_8859_8;
+  encoding_map["iso-ir-144"] = ISO_8859_5;
+  encoding_map["iso-ir-148"] = ISO_8859_9;
+  encoding_map["iso-ir-149"] = KOREAN_EUC_KR;
+  encoding_map["iso-ir-157"] = ISO_8859_10;
+  encoding_map["iso-ir-58"] = CHINESE_GB;
+  encoding_map["iso-latin-1"] = ISO_8859_1;
+  encoding_map["iso_2022-cn"] = ISO_2022_CN;
+  encoding_map["iso_2022-kr"] = ISO_2022_KR;
+  encoding_map["iso_8859-1"] = ISO_8859_1;
+  encoding_map["iso_8859-10:1992"] = ISO_8859_10;
+  encoding_map["iso_8859-11"] = ISO_8859_11;
+  encoding_map["iso_8859-13"] = ISO_8859_13;
+  encoding_map["iso_8859-15"] = ISO_8859_15;
+  encoding_map["iso_8859-1:1987"] = ISO_8859_1;
+  encoding_map["iso_8859-2"] = ISO_8859_2;
+  encoding_map["iso_8859-2:1987"] = ISO_8859_2;
+  encoding_map["iso_8859-3"] = ISO_8859_3;
+  encoding_map["iso_8859-3:1988"] = ISO_8859_3;
+  encoding_map["iso_8859-4"] = ISO_8859_4;
+  encoding_map["iso_8859-4:1988"] = ISO_8859_4;
+  encoding_map["iso_8859-5"] = ISO_8859_5;
+  encoding_map["iso_8859-5:1988"] = ISO_8859_5;
+  encoding_map["iso_8859-6"] = ISO_8859_6;
+  encoding_map["iso_8859-6:1987"] = ISO_8859_6;
+  encoding_map["iso_8859-7"] = ISO_8859_7;
+  encoding_map["iso_8859-7:1987"] = ISO_8859_7;
+  encoding_map["iso_8859-8"] = ISO_8859_8;
+  encoding_map["iso_8859-8:1988:"] = ISO_8859_8;
+  encoding_map["iso_8859-9"] = ISO_8859_9;
+  encoding_map["iso_8859-9:1989"] = ISO_8859_9;
+  encoding_map["jagran"] = JAGRAN;
+  encoding_map["jis"] = JAPANESE_JIS;   // not iana standard
+  encoding_map["koi8-cs"] = CZECH_CSN_369103;
+  encoding_map["koi8-r"] = RUSSIAN_KOI8_R;
+  encoding_map["koi8-ru"] = RUSSIAN_KOI8_RU;  // not iana standard
+  encoding_map["koi8-u"] = RUSSIAN_KOI8_RU;
+  encoding_map["koi8r"] = RUSSIAN_KOI8_R;  // not iana standard
+  encoding_map["koi8u"] = RUSSIAN_KOI8_RU;  // not iana standard
+  encoding_map["korean"] = KOREAN_EUC_KR;  // i assume this is what is meant
+  encoding_map["ks-c-5601"] = KOREAN_EUC_KR;  // not iana standard
+  encoding_map["ks-c-5601-1987"] = KOREAN_EUC_KR;  // not iana standard
+  encoding_map["ks_c_5601-1989"] = KOREAN_EUC_KR;
+  encoding_map["ksc"] = KOREAN_EUC_KR;  // not iana standard
+  encoding_map["l1"] = ISO_8859_1;
+  encoding_map["l2"] = ISO_8859_2;
+  encoding_map["l3"] = ISO_8859_3;
+  encoding_map["l4"] = ISO_8859_4;
+  encoding_map["l5"] = ISO_8859_9;
+  encoding_map["l6"] = ISO_8859_10;
+  encoding_map["latin-1"] = ISO_8859_1;  // not iana standard
+  encoding_map["latin1"] = ISO_8859_1;
+  encoding_map["latin2"] = ISO_8859_2;
+  encoding_map["latin3"] = ISO_8859_3;
+  encoding_map["latin4"] = ISO_8859_4;
+  encoding_map["latin5"] = ISO_8859_9;
+  encoding_map["latin6"] = ISO_8859_10;
+  encoding_map["mac"] = MACINTOSH_ROMAN;
+  encoding_map["macintosh"] = MACINTOSH_ROMAN;
+  encoding_map["macintosh-roman"] = MACINTOSH_ROMAN;
+  encoding_map["ms932"] = JAPANESE_CP932;  // not iana standard
+  encoding_map["ms_kanji"] = JAPANESE_CP932;
+  encoding_map["shift-jis"] = JAPANESE_SHIFT_JIS;
+  encoding_map["shift_jis"] = JAPANESE_SHIFT_JIS;
+  encoding_map["sjis"] = JAPANESE_SHIFT_JIS;  // not iana standard
+  encoding_map["sjs"] = JAPANESE_SHIFT_JIS;  // not iana standard
+  encoding_map["sun_eu_greek"] = ISO_8859_7;
+  encoding_map["tab"] = TAMIL_BI;
+  encoding_map["tam"] = TAMIL_MONO;
+  encoding_map["tis-620"] = ISO_8859_11;
+  encoding_map["tscii"] = TSCII;
+  encoding_map["un"] = UNKNOWN_ENCODING;  // not iana standard
+  encoding_map["unicode"] = UNICODE;  // not iana standard
+  encoding_map["unicode-1-1-utf-7"] = UTF7;
+  encoding_map["unicode-1-1-utf-8"] = UTF8;
+  encoding_map["unicode-2-0-utf-7"] = UTF7;
+  encoding_map["unknown"] = UNKNOWN_ENCODING;   // not iana standard
+  encoding_map["us"] = ISO_8859_1;
+  encoding_map["us-ascii"] = ISO_8859_1;
+  encoding_map["utf-16be"] = UTF16BE;
+  encoding_map["utf-16le"] = UTF16LE;
+  encoding_map["utf-32be"] = UTF32BE;
+  encoding_map["utf-32le"] = UTF32LE;
+  encoding_map["utf-7"] = UTF7;
+  encoding_map["utf-8"] = UTF8;
+  encoding_map["utf7"] = UTF7;
+  encoding_map["utf8"] = UTF8;  // not iana standard
+  encoding_map["visual"] = HEBREW_VISUAL;
+  encoding_map["win-1250"] = MSFT_CP1250;  // not iana standard
+  encoding_map["win-1251"] = RUSSIAN_CP1251;  // not iana standard
+  encoding_map["window-874"] = MSFT_CP874;
+  encoding_map["windows-1250"] = MSFT_CP1250;
+  encoding_map["windows-1251"] = RUSSIAN_CP1251;
+  encoding_map["windows-1252"] = MSFT_CP1252;
+  encoding_map["windows-1253"] = MSFT_CP1253;
+  encoding_map["windows-1254"] = MSFT_CP1254;
+  encoding_map["windows-1255"] = MSFT_CP1255;
+  encoding_map["windows-1256"] = MSFT_CP1256;
+  encoding_map["windows-1257"] = MSFT_CP1257;
+  encoding_map["windows-31j"] = JAPANESE_CP932;
+  encoding_map["windows-874"] = MSFT_CP874;
+  encoding_map["windows-936"] = GBK;
+  encoding_map["x-big5"] = CHINESE_BIG5;
+  encoding_map["x-binaryenc"] = BINARYENC;  // not iana standard
+  encoding_map["x-cp1250"] = MSFT_CP1250;
+  encoding_map["x-cp1251"] = RUSSIAN_CP1251;
+  encoding_map["x-cp1252"] = MSFT_CP1252;
+  encoding_map["x-cp1253"] = MSFT_CP1253;
+  encoding_map["x-cp1254"] = MSFT_CP1254;
+  encoding_map["x-cp1255"] = MSFT_CP1255;
+  encoding_map["x-cp1256"] = MSFT_CP1256;
+  encoding_map["x-cp1257"] = MSFT_CP1257;
+  encoding_map["x-euc-jp"] = JAPANESE_EUC_JP;
+  encoding_map["x-euc-tw"] = CHINESE_CNS;
+  encoding_map["x-gbk"] = GBK;
+  encoding_map["x-iso-10646-ucs-2-be"] = UTF16BE;
+  encoding_map["x-iso-10646-ucs-2-le"] = UTF16LE;
+  encoding_map["x-iso-10646-ucs-4-be"] = UTF32BE;
+  encoding_map["x-iso-10646-ucs-4-le"] = UTF32LE;
+  encoding_map["x-jis"] = JAPANESE_JIS;  // not iana standard
+  encoding_map["x-mac-roman"] = MACINTOSH_ROMAN;
+  encoding_map["x-shift_jis"] = JAPANESE_SHIFT_JIS;  // not iana standard
+  encoding_map["x-sjis"] = JAPANESE_SHIFT_JIS;
+  encoding_map["x-unicode-2-0-utf-7"] = UTF7;
+  encoding_map["x-utf8utf8"] = UTF8UTF8;  // not iana standard
+  encoding_map["x-x-big5"] = CHINESE_BIG5;
+  encoding_map["zh_cn.euc"] = CHINESE_GB;
+  encoding_map["zh_tw-big5"] = CHINESE_BIG5;
+  encoding_map["zh_tw-euc"] = CHINESE_CNS;
+
+  // Remove they entry for the empty string, if any.
+  encoding_map.erase("");
+
+  return encoding_map;
+}
+
+// ----------------------------------------------------------------------
+// EncodingNameAliasToEncoding()
+//
+// This function takes an encoding name/alias and returns the Encoding
+// enum. The input is case insensitive. It is the union of the common
+// IANA standard names, the charset names used in Netscape Navigator,
+// and some common names we have been using.
+// See: http://www.iana.org/assignments/character-sets
+// http://physics.hallym.ac.kr/resource/relnotes/windows-2.0.html
+//
+// UNKNOWN_ENCODING is returned if none matches.
+//
+// TODO: Check if it is possible to remove the non-standard,
+// non-netscape-use names. It is because this routine is used for
+// encoding detections from html meta info. Non-standard names may
+// introduce noise on encoding detection.
+//
+// TODO: Unify EncodingNameAliasToEncoding and EncodingFromName,
+// or determine why such a unification is not possible.
+// ----------------------------------------------------------------------
+Encoding EncodingNameAliasToEncoding(const char *encoding_name) {
+  if (!encoding_name) {
+    return UNKNOWN_ENCODING;
+  }
+
+  const EncodingMap& encoding_map = GetEncodingMap();
+
+  EncodingMap::const_iterator emi = encoding_map.find(encoding_name);
+  if (emi != encoding_map.end()) {
+    return emi->second;
+  } else {
+    return UNKNOWN_ENCODING;
+  }
+}
+
+const char* default_encoding_name() {
+  return kEncodingInfoTable[LATIN1].encoding_name_;
+}
+
+static const char* const kInvalidEncodingName = "invalid_encoding";
+
+const char *invalid_encoding_name() {
+  return kInvalidEncodingName;
+}
+
+
+
+// *************************************************************
+// Miscellany
+// *************************************************************
+
+
+Encoding PreferredWebOutputEncoding(Encoding enc) {
+  return IsValidEncoding(enc)
+      ? kEncodingInfoTable[enc].preferred_web_output_encoding_
+      : UTF8;
+}

contrib/google-ced/util/encodings/encodings.h

@@ -0,0 +1,299 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_ENCODINGS_ENCODINGS_H_
+#define UTIL_ENCODINGS_ENCODINGS_H_
+
+// This interface defines the Encoding enum and various functions that
+// depend only on Encoding values.
+
+// A hash-function for Encoding, hash<Encoding>, is defined in
+// i18n/encodings/public/encodings-hash.h
+
+// On some Windows projects, UNICODE may be defined, which would prevent the
+// Encoding enum below from compiling. Note that this is a quick fix that does
+// not break any existing projects. The UNICODE enum may someday be changed
+// to something more specific and non-colliding, but this involves careful
+// testing of changes in many other projects.
+#undef UNICODE
+
+// NOTE: The Encoding enum must always start at 0. This assumption has
+// been made and used.
+
+#ifndef SWIG
+
+#include "util/encodings/encodings.pb.h"
+
+#else
+
+// TODO: Include a SWIG workaround header file.
+
+#endif
+
+const int kNumEncodings = NUM_ENCODINGS;
+
+// some of the popular encoding aliases
+// TODO: Make these static const Encoding values instead of macros.
+#define LATIN1           ISO_8859_1
+#define LATIN2           ISO_8859_2
+#define LATIN3           ISO_8859_3
+#define LATIN4           ISO_8859_4
+#define CYRILLIC         ISO_8859_5
+#define ARABIC_ENCODING  ISO_8859_6     // avoiding the same name as language
+#define GREEK_ENCODING   ISO_8859_7     // avoiding the same name as language
+#define HEBREW_ENCODING  ISO_8859_8     // avoiding the same name as language
+#define LATIN5           ISO_8859_9
+#define LATIN6           ISO_8859_10
+#define KOREAN_HANGUL    KOREAN_EUC_KR
+
+// The default Encoding (LATIN1).
+Encoding default_encoding();
+
+
+
+// *************************************************************
+// Encoding predicates
+//   IsValidEncoding()
+//   IsEncEncCompatible
+//   IsSupersetOfAscii7Bit
+//   Is8BitEncoding
+//   IsCJKEncoding
+//   IsHebrewEncoding
+//   IsRightToLeftEncoding
+//   IsLogicalRightToLeftEncoding
+//   IsVisualRightToLeftEncoding
+//   IsIso2022Encoding
+//   IsIso2022JpOrVariant
+//   IsShiftJisOrVariant
+//   IsJapaneseCellPhoneCarrierSpecificEncoding
+// *************************************************************
+
+// IsValidEncoding
+// ===================================
+//
+// Function to check if the input language enum is within range.
+//
+
+bool IsValidEncoding(Encoding enc);
+
+//
+// IsEncEncCompatible
+// ------------------
+//
+// This function is to determine whether or not converting from the
+// first encoding to the second requires any changes to the underlying
+// text (e.g.  ASCII_7BIT is a subset of UTF8).
+//
+// TODO: the current implementation is likely incomplete.  It would be
+// good to consider the full matrix of all pairs of encodings and to fish out
+// all compatible pairs.
+//
+bool IsEncEncCompatible(const Encoding from, const Encoding to);
+
+// To be a superset of 7-bit Ascii means that bytes 0...127 in the given
+// encoding represent the same characters as they do in ISO_8859_1.
+
+// WARNING: This function does not currently return true for all encodings that
+// are supersets of Ascii 7-bit.
+bool IsSupersetOfAscii7Bit(Encoding e);
+
+// To be an 8-bit encoding means that there are fewer than 256 symbols.
+// Each byte determines a new character; there are no multi-byte sequences.
+
+// WARNING: This function does not currently return true for all encodings that
+// are 8-bit encodings.
+bool Is8BitEncoding(Encoding e);
+
+// IsCJKEncoding
+// -------------
+//
+// This function returns true if the encoding is either Chinese
+// (simplified or traditional), Japanese, or Korean. Note: UTF8 is not
+// considered a CJK encoding.
+bool IsCJKEncoding(Encoding e);
+
+// IsHebrewEncoding
+// -------------
+//
+// This function returns true if the encoding is a Hebrew specific
+// encoding (not UTF8, etc).
+bool IsHebrewEncoding(Encoding e);
+
+// IsRightToLeftEncoding
+// ---------------------
+//
+// Returns true if the encoding is a right-to-left encoding.
+//
+// Note that the name of this function is somewhat misleading. There is nothing
+// "right to left" about these encodings. They merely contain code points for
+// characters in RTL languages such as Hebrew and Arabic. But this is also
+// true for UTF-8.
+//
+// TODO: Get rid of this function. The only special-case we
+// should need to worry about are visual encodings. Anything we
+// need to do for all 'RTL' encodings we need to do for UTF-8 as well.
+bool IsRightToLeftEncoding(Encoding enc);
+
+// IsLogicalRightToLeftEncoding
+// ----------------------------
+//
+// Returns true if the encoding is a logical right-to-left encoding.
+// Logical right-to-left encodings are those that the browser renders
+// right-to-left and applies the BiDi algorithm to. Therefore the characters
+// appear in reading order in the file, and indexing, snippet generation etc.
+// should all just work with no special processing.
+//
+// TODO: Get rid of this function. The only special-case we
+// should need to worry about are visual encodings.
+bool IsLogicalRightToLeftEncoding(Encoding enc);
+
+// IsVisualRightToLeftEncoding
+// ---------------------------
+//
+// Returns true if the encoding is a visual right-to-left encoding.
+// Visual right-to-left encodings are those that the browser renders
+// left-to-right and does not apply the BiDi algorithm to. Therefore each
+// line appears in reverse order in the file, lines are manually wrapped
+// by abusing <br> or <p> tags, etc. Visual RTL encoding is a relic of
+// the prehistoric days when browsers couldn't render right-to-left, but
+// unfortunately some visual pages persist to this day. These documents require
+// special processing so that we don't index or snippet them with each line
+// reversed.
+bool IsVisualRightToLeftEncoding(Encoding enc);
+
+// IsIso2022Encoding
+// -----------------
+//
+// Returns true if the encoding is a kind of ISO 2022 such as
+// ISO-2022-JP.
+bool IsIso2022Encoding(Encoding enc);
+
+// IsIso2022JpOrVariant
+// --------------------
+//
+// Returns true if the encoding is ISO-2022-JP or a variant such as
+// KDDI's ISO-2022-JP.
+bool IsIso2022JpOrVariant(Encoding enc);
+
+// IsShiftJisOrVariant
+// --------------------
+//
+// Returns true if the encoding is Shift_JIS or a variant such as
+// KDDI's Shift_JIS.
+bool IsShiftJisOrVariant(Encoding enc);
+
+// IsJapanesCellPhoneCarrierSpecificEncoding
+// -----------------------------------------
+//
+// Returns true if it's Japanese cell phone carrier specific encoding
+// such as KDDI_SHIFT_JIS.
+bool IsJapaneseCellPhoneCarrierSpecificEncoding(Encoding enc);
+
+
+
+// *************************************************************
+// ENCODING NAMES
+//
+// This interface defines a standard name for each valid encoding, and
+// a standard name for invalid encodings. (Some names use all upper
+// case, but others use mixed case.)
+//
+//   EncodingName() [Encoding to name]
+//   MimeEncodingName() [Encoding to name]
+//   EncodingFromName() [name to Encoding]
+//   EncodingNameAliasToEncoding() [name to Encoding]
+//   default_encoding_name()
+//   invalid_encoding_name()
+// *************************************************************
+
+// EncodingName
+// ------------
+//
+// Given the encoding, returns its standard name.
+// Return invalid_encoding_name() if the encoding is invalid.
+//
+const char* EncodingName(Encoding enc);
+
+//
+// MimeEncodingName
+// ----------------
+//
+// Return the "preferred MIME name" of an encoding.
+//
+// This name is suitable for using in HTTP headers, HTML tags,
+// and as the "charset" parameter of a MIME Content-Type.
+const char* MimeEncodingName(Encoding enc);
+
+
+// The maximum length of an encoding name
+const int kMaxEncodingNameSize = 50;
+
+// The standard name of the default encoding.
+const char* default_encoding_name();
+
+// The name used for an invalid encoding.
+const char* invalid_encoding_name();
+
+// EncodingFromName
+// ----------------
+//
+// If enc_name matches the standard name of an Encoding, using a
+// case-insensitive comparison, set *encoding to that Encoding and
+// return true.  Otherwise set *encoding to UNKNOWN_ENCODING and
+// return false.
+//
+// REQUIRES: encoding must not be NULL.
+//
+bool EncodingFromName(const char* enc_name, Encoding *encoding);
+
+//
+// EncodingNameAliasToEncoding
+// ---------------------------
+//
+// If enc_name matches the standard name or an alias of an Encoding,
+// using a case-insensitive comparison, return that
+// Encoding. Otherwise, return UNKNOWN_ENCODING.
+//
+// Aliases include most mime-encoding names (e.g., "ISO-8859-7" for
+// GREEK), alternate names (e.g., "cyrillic" for ISO_8859_5) and
+// common variations with hyphens and underscores (e.g., "koi8-u" and
+// "koi8u" for RUSSIAN_KOI8_R).
+
+Encoding EncodingNameAliasToEncoding(const char *enc_name);
+
+// *************************************************************
+// Miscellany
+// *************************************************************
+
+// PreferredWebOutputEncoding
+// --------------------------
+//
+// Some multi-byte encodings use byte values that coincide with the
+// ASCII codes for HTML syntax characters <>"&' and browsers like MSIE
+// can misinterpret these, as indicated in an external XSS report from
+// 2007-02-15. Here, we map these dangerous encodings to safer ones. We
+// also use UTF8 instead of encodings that we don't support in our
+// output, and we generally try to be conservative in what we send out.
+// Where the client asks for single- or double-byte encodings that are
+// not as common, we substitute a more common single- or double-byte
+// encoding, if there is one, thereby preserving the client's intent
+// to use less space than UTF-8. This also means that characters
+// outside the destination set will be converted to HTML NCRs (&#NNN;)
+// if requested.
+Encoding PreferredWebOutputEncoding(Encoding enc);
+
+
+#endif  // UTIL_ENCODINGS_ENCODINGS_H_

contrib/google-ced/util/encodings/encodings.pb.h

@@ -0,0 +1,181 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_ENCODINGS_ENCODINGS_PB_H_
+#define UTIL_ENCODINGS_ENCODINGS_PB_H_
+
+enum Encoding {
+  ISO_8859_1           =  0,  // Teragram ASCII
+  ISO_8859_2           =  1,  // Teragram Latin2
+  ISO_8859_3           =  2,  // in BasisTech but not in Teragram
+  ISO_8859_4           =  3,  // Teragram Latin4
+  ISO_8859_5           =  4,  // Teragram ISO-8859-5
+  ISO_8859_6           =  5,  // Teragram Arabic
+  ISO_8859_7           =  6,  // Teragram Greek
+  ISO_8859_8           =  7,  // Teragram Hebrew
+  ISO_8859_9           =  8,  // in BasisTech but not in Teragram
+  ISO_8859_10          =  9,  // in BasisTech but not in Teragram
+  JAPANESE_EUC_JP      = 10,  // Teragram EUC_JP
+  JAPANESE_SHIFT_JIS   = 11,  // Teragram SJS
+  JAPANESE_JIS         = 12,  // Teragram JIS
+  CHINESE_BIG5         = 13,  // Teragram BIG5
+  CHINESE_GB           = 14,  // Teragram GB
+  CHINESE_EUC_CN       = 15,  // Misnamed. Should be EUC_TW. Was Basis Tech
+                              // CNS11643EUC, before that Teragram EUC-CN(!)
+                              // See //i18n/basistech/basistech_encodings.h
+  KOREAN_EUC_KR        = 16,  // Teragram KSC
+  UNICODE              = 17,  // Teragram Unicode
+  CHINESE_EUC_DEC      = 18,  // Misnamed. Should be EUC_TW. Was Basis Tech
+                              // CNS11643EUC, before that Teragram EUC.
+  CHINESE_CNS          = 19,  // Misnamed. Should be EUC_TW. Was Basis Tech
+                              // CNS11643EUC, before that Teragram CNS.
+  CHINESE_BIG5_CP950   = 20,  // Teragram BIG5_CP950
+  JAPANESE_CP932       = 21,  // Teragram CP932
+  UTF8                 = 22,
+  UNKNOWN_ENCODING     = 23,
+  ASCII_7BIT           = 24,  // ISO_8859_1 with all characters <= 127.
+                              // Should be present only in the crawler
+                              // and in the repository,
+                              // *never* as a result of Document::encoding().
+  RUSSIAN_KOI8_R       = 25,  // Teragram KOI8R
+  RUSSIAN_CP1251       = 26,  // Teragram CP1251
+
+  //----------------------------------------------------------
+  // These are _not_ output from teragram. Instead, they are as
+  // detected in the headers of usenet articles.
+  MSFT_CP1252          = 27,  // 27: CP1252 aka MSFT euro ascii
+  RUSSIAN_KOI8_RU      = 28,  // CP21866 aka KOI8-U, used for Ukrainian.
+                              // Misnamed, this is _not_ KOI8-RU but KOI8-U.
+                              // KOI8-U is used much more often than KOI8-RU.
+  MSFT_CP1250          = 29,  // CP1250 aka MSFT eastern european
+  ISO_8859_15          = 30,  // aka ISO_8859_0 aka ISO_8859_1 euroized
+  //----------------------------------------------------------
+
+  //----------------------------------------------------------
+  // These are in BasisTech but not in Teragram. They are
+  // needed for new interface languages. Now detected by
+  // research langid
+  MSFT_CP1254          = 31,  // used for Turkish
+  MSFT_CP1257          = 32,  // used in Baltic countries
+  //----------------------------------------------------------
+
+  //----------------------------------------------------------
+  //----------------------------------------------------------
+  // New encodings detected by Teragram
+  ISO_8859_11          = 33,  // aka TIS-620, used for Thai
+  MSFT_CP874           = 34,  // used for Thai
+  MSFT_CP1256          = 35,  // used for Arabic
+
+  //----------------------------------------------------------
+  // Detected as ISO_8859_8 by Teragram, but can be found in META tags
+  MSFT_CP1255          = 36,  // Logical Hebrew Microsoft
+  ISO_8859_8_I         = 37,  // Iso Hebrew Logical
+  HEBREW_VISUAL        = 38,  // Iso Hebrew Visual
+  //----------------------------------------------------------
+
+  //----------------------------------------------------------
+  // Detected by research langid
+  CZECH_CP852          = 39,
+  CZECH_CSN_369103     = 40,  // aka ISO_IR_139 aka KOI8_CS
+  MSFT_CP1253          = 41,  // used for Greek
+  RUSSIAN_CP866        = 42,
+  //----------------------------------------------------------
+
+  //----------------------------------------------------------
+  // Handled by iconv in glibc
+  ISO_8859_13          = 43,
+  ISO_2022_KR          = 44,
+  GBK                  = 45,
+  GB18030              = 46,
+  BIG5_HKSCS           = 47,
+  ISO_2022_CN          = 48,
+
+  //-----------------------------------------------------------
+  // Detected by xin liu's detector
+  // Handled by transcoder
+  // (Indic encodings)
+
+  TSCII                = 49,
+  TAMIL_MONO           = 50,
+  TAMIL_BI             = 51,
+  JAGRAN               = 52,
+
+
+  MACINTOSH_ROMAN      = 53,
+  UTF7                 = 54,
+  BHASKAR              = 55,  // Indic encoding - Devanagari
+  HTCHANAKYA           = 56,  // 56 Indic encoding - Devanagari
+
+  //-----------------------------------------------------------
+  // These allow a single place (inputconverter and outputconverter)
+  // to do UTF-16 <==> UTF-8 bulk conversions and UTF-32 <==> UTF-8
+  // bulk conversions, with interchange-valid checking on input and
+  // fallback if needed on ouput.
+  UTF16BE              = 57,  // big-endian UTF-16
+  UTF16LE              = 58,  // little-endian UTF-16
+  UTF32BE              = 59,  // big-endian UTF-32
+  UTF32LE              = 60,  // little-endian UTF-32
+  //-----------------------------------------------------------
+
+  //-----------------------------------------------------------
+  // An encoding that means "This is not text, but it may have some
+  // simple ASCII text embedded". Intended input conversion (not yet
+  // implemented) is to keep strings of >=4 seven-bit ASCII characters
+  // (follow each kept string with an ASCII space), delete the rest of
+  // the bytes. This will pick up and allow indexing of e.g. captions
+  // in JPEGs. No output conversion needed.
+  BINARYENC            = 61,
+  //-----------------------------------------------------------
+
+  //-----------------------------------------------------------
+  // Some Web pages allow a mixture of HZ-GB and GB-2312 by using
+  // ~{ ... ~} for 2-byte pairs, and the browsers support this.
+  HZ_GB_2312           = 62,
+  //-----------------------------------------------------------
+
+  //-----------------------------------------------------------
+  // Some external vendors make the common input error of
+  // converting MSFT_CP1252 to UTF8 *twice*. No output conversion needed.
+  UTF8UTF8             = 63,
+  //-----------------------------------------------------------
+
+  //-----------------------------------------------------------
+  // Handled by transcoder for tamil language specific font
+  // encodings without the support for detection at present.
+  TAM_ELANGO           = 64,  // Elango - Tamil
+  TAM_LTTMBARANI       = 65,  // Barani - Tamil
+  TAM_SHREE            = 66,  // Shree - Tamil
+  TAM_TBOOMIS          = 67,  // TBoomis - Tamil
+  TAM_TMNEWS           = 68,  // TMNews - Tamil
+  TAM_WEBTAMIL         = 69,  // Webtamil - Tamil
+  //-----------------------------------------------------------
+
+  //-----------------------------------------------------------
+  // Shift_JIS variants used by Japanese cell phone carriers.
+  KDDI_SHIFT_JIS       = 70,
+  DOCOMO_SHIFT_JIS     = 71,
+  SOFTBANK_SHIFT_JIS   = 72,
+  // ISO-2022-JP variants used by KDDI and SoftBank.
+  KDDI_ISO_2022_JP     = 73,
+  SOFTBANK_ISO_2022_JP = 74,
+  //-----------------------------------------------------------
+
+  NUM_ENCODINGS        = 75,  // Always keep this at the end. It is not a
+                              // valid Encoding enum, it is only used to
+                              // indicate the total number of Encodings.
+};
+
+#endif  // UTIL_ENCODINGS_ENCODINGS_PB_H_

contrib/google-ced/util/encodings/encodings_unittest.cc

@@ -0,0 +1,34 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/encodings/encodings.h"
+
+#include "gtest/gtest.h"
+
+TEST(EncodingsTest, EncodingNameAliasToEncoding) {
+  // Test that cases, non-alpha-numeric chars are ignored.
+  EXPECT_EQ(ISO_8859_1, EncodingNameAliasToEncoding("iso_8859_1"));
+  EXPECT_EQ(ISO_8859_1, EncodingNameAliasToEncoding("iso-8859-1"));
+
+  // Test that spaces are ignored.
+  EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF8"));
+  EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF 8"));
+  EXPECT_EQ(UTF8, EncodingNameAliasToEncoding("UTF-8"));
+
+  // Test alphanumeric differences are counted.
+  EXPECT_NE(UTF8, EncodingNameAliasToEncoding("UTF-7"));
+  EXPECT_NE(KOREAN_EUC_KR, EncodingNameAliasToEncoding("euc-jp"));
+}

contrib/google-ced/util/languages/languages.cc

@@ -0,0 +1,349 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "util/languages/languages.h"
+
+#include "util/basictypes.h"
+#include "util/string_util.h"
+
+
+Language default_language() {return ENGLISH;}
+
+
+// Language names and codes
+
+struct LanguageInfo {
+  const char * language_name_;
+  const char * language_code_639_1_;   // the ISO-639-1 code for the language
+  const char * language_code_639_2_;   // the ISO-639-2 code for the language
+  const char * language_code_other_;   // some nonstandard code for the language
+};
+
+static const LanguageInfo kLanguageInfoTable[] = {
+  { "ENGLISH",             "en", "eng", NULL},
+  { "DANISH",              "da", "dan", NULL},
+  { "DUTCH",               "nl", "dut", NULL},
+  { "FINNISH",             "fi", "fin", NULL},
+  { "FRENCH",              "fr", "fre", NULL},
+  { "GERMAN",              "de", "ger", NULL},
+  { "HEBREW",              "he", "heb", NULL},
+  { "ITALIAN",             "it", "ita", NULL},
+  { "Japanese",            "ja", "jpn", NULL},
+  { "Korean",              "ko", "kor", NULL},
+  { "NORWEGIAN",           "nb", "nor", NULL},
+  { "POLISH",              "pl", "pol", NULL},
+  { "PORTUGUESE",          "pt", "por", NULL},
+  { "RUSSIAN",             "ru", "rus", NULL},
+  { "SPANISH",             "es", "spa", NULL},
+  { "SWEDISH",             "sv", "swe", NULL},
+  { "Chinese",             "zh", "chi", "zh-CN"},
+  { "CZECH",               "cs", "cze", NULL},
+  { "GREEK",               "el", "gre", NULL},
+  { "ICELANDIC",           "is", "ice", NULL},
+  { "LATVIAN",             "lv", "lav", NULL},
+  { "LITHUANIAN",          "lt", "lit", NULL},
+  { "ROMANIAN",            "ro", "rum", NULL},
+  { "HUNGARIAN",           "hu", "hun", NULL},
+  { "ESTONIAN",            "et", "est", NULL},
+  // TODO: Although Teragram has two output names "TG_UNKNOWN_LANGUAGE"
+  // and "Unknown", they are essentially the same. Need to unify them.
+  // "un" and "ut" are invented by us, not from ISO-639.
+  //
+  { "TG_UNKNOWN_LANGUAGE", NULL, NULL, "ut"},
+  { "Unknown",             NULL, NULL, "un"},
+  { "BULGARIAN",           "bg", "bul", NULL},
+  { "CROATIAN",            "hr", "scr", NULL},
+  { "SERBIAN",             "sr", "scc", NULL},
+  { "IRISH",               "ga", "gle", NULL},
+  { "GALICIAN",            "gl", "glg", NULL},
+  // Impossible to tell Tagalog from Filipino at the moment.
+  // Use ISO 639-2 code for Filipino here.
+  { "TAGALOG",             NULL, "fil", NULL},
+  { "TURKISH",             "tr", "tur", NULL},
+  { "UKRAINIAN",           "uk", "ukr", NULL},
+  { "HINDI",               "hi", "hin", NULL},
+  { "MACEDONIAN",          "mk", "mac", NULL},
+  { "BENGALI",             "bn", "ben", NULL},
+  { "INDONESIAN",          "id", "ind", NULL},
+  { "LATIN",               "la", "lat", NULL},
+  { "MALAY",               "ms", "may", NULL},
+  { "MALAYALAM",           "ml", "mal", NULL},
+  { "WELSH",               "cy", "wel", NULL},
+  { "NEPALI",              "ne", "nep", NULL},
+  { "TELUGU",              "te", "tel", NULL},
+  { "ALBANIAN",            "sq", "alb", NULL},
+  { "TAMIL",               "ta", "tam", NULL},
+  { "BELARUSIAN",          "be", "bel", NULL},
+  { "JAVANESE",            "jw", "jav", NULL},
+  { "OCCITAN",             "oc", "oci", NULL},
+  { "URDU",                "ur", "urd", NULL},
+  { "BIHARI",              "bh", "bih", NULL},
+  { "GUJARATI",            "gu", "guj", NULL},
+  { "THAI",                "th", "tha", NULL},
+  { "ARABIC",              "ar", "ara", NULL},
+  { "CATALAN",             "ca", "cat", NULL},
+  { "ESPERANTO",           "eo", "epo", NULL},
+  { "BASQUE",              "eu", "baq", NULL},
+  { "INTERLINGUA",         "ia", "ina", NULL},
+  { "KANNADA",             "kn", "kan", NULL},
+  { "PUNJABI",             "pa", "pan", NULL},
+  { "SCOTS_GAELIC",        "gd", "gla", NULL},
+  { "SWAHILI",             "sw", "swa", NULL},
+  { "SLOVENIAN",           "sl", "slv", NULL},
+  { "MARATHI",             "mr", "mar", NULL},
+  { "MALTESE",             "mt", "mlt", NULL},
+  { "VIETNAMESE",          "vi", "vie", NULL},
+  { "FRISIAN",             "fy", "fry", NULL},
+  { "SLOVAK",              "sk", "slo", NULL},
+  { "ChineseT",
+    NULL,  NULL,  // We intentionally set these 2 fields to NULL to avoid
+                  // confusion between CHINESE_T and CHINESE.
+    "zh-TW"},
+  { "FAROESE",             "fo", "fao", NULL},
+  { "SUNDANESE",           "su", "sun", NULL},
+  { "UZBEK",               "uz", "uzb", NULL},
+  { "AMHARIC",             "am", "amh", NULL},
+  { "AZERBAIJANI",         "az", "aze", NULL},
+  { "GEORGIAN",            "ka", "geo", NULL},
+  { "TIGRINYA",            "ti", "tir", NULL},
+  { "PERSIAN",             "fa", "per", NULL},
+  { "BOSNIAN",             "bs", "bos", NULL},
+  { "SINHALESE",           "si", "sin", NULL},
+  { "NORWEGIAN_N",         "nn", "nno", NULL},
+  { "PORTUGUESE_P",        NULL, NULL, "pt-PT"},
+  { "PORTUGUESE_B",        NULL, NULL, "pt-BR"},
+  { "XHOSA",               "xh", "xho", NULL},
+  { "ZULU",                "zu", "zul", NULL},
+  { "GUARANI",             "gn", "grn", NULL},
+  { "SESOTHO",             "st", "sot", NULL},
+  { "TURKMEN",             "tk", "tuk", NULL},
+  { "KYRGYZ",              "ky", "kir", NULL},
+  { "BRETON",              "br", "bre", NULL},
+  { "TWI",                 "tw", "twi", NULL},
+  { "YIDDISH",             "yi", "yid", NULL},
+  { "SERBO_CROATIAN",      "sh", NULL, NULL},
+  { "SOMALI",              "so", "som", NULL},
+  { "UIGHUR",              "ug", "uig", NULL},
+  { "KURDISH",             "ku", "kur", NULL},
+  { "MONGOLIAN",           "mn", "mon", NULL},
+  { "ARMENIAN",            "hy", "arm", NULL},
+  { "LAOTHIAN",            "lo", "lao", NULL},
+  { "SINDHI",              "sd", "snd", NULL},
+  { "RHAETO_ROMANCE",      "rm", "roh", NULL},
+  { "AFRIKAANS",           "af", "afr", NULL},
+  { "LUXEMBOURGISH",       "lb", "ltz", NULL},
+  { "BURMESE",             "my", "bur", NULL},
+  // KHMER is known as Cambodian for Google user interfaces.
+  { "KHMER",               "km", "khm", NULL},
+  { "TIBETAN",             "bo", "tib", NULL},
+  { "DHIVEHI",             "dv", "div", NULL},
+  { "CHEROKEE",            NULL, "chr", NULL},
+  { "SYRIAC",              NULL, "syr", NULL},
+  { "LIMBU",               NULL, NULL, "sit-NP"},
+  { "ORIYA",               "or", "ori", NULL},
+  { "ASSAMESE",            "as", "asm", NULL},
+  { "CORSICAN",            "co", "cos", NULL},
+  { "INTERLINGUE",         "ie", "ine", NULL},
+  { "KAZAKH",              "kk", "kaz", NULL},
+  { "LINGALA",             "ln", "lin", NULL},
+  { "MOLDAVIAN",           "mo", "mol", NULL},
+  { "PASHTO",              "ps", "pus", NULL},
+  { "QUECHUA",             "qu", "que", NULL},
+  { "SHONA",               "sn", "sna", NULL},
+  { "TAJIK",               "tg", "tgk", NULL},
+  { "TATAR",               "tt", "tat", NULL},
+  { "TONGA",               "to", "tog", NULL},
+  { "YORUBA",              "yo", "yor", NULL},
+  { "CREOLES_AND_PIDGINS_ENGLISH_BASED", NULL, "cpe", NULL},
+  { "CREOLES_AND_PIDGINS_FRENCH_BASED",  NULL, "cpf", NULL},
+  { "CREOLES_AND_PIDGINS_PORTUGUESE_BASED", NULL, "cpp", NULL},
+  { "CREOLES_AND_PIDGINS_OTHER", NULL, "crp", NULL},
+  { "MAORI",               "mi", "mao", NULL},
+  { "WOLOF",               "wo", "wol", NULL},
+  { "ABKHAZIAN",           "ab", "abk", NULL},
+  { "AFAR",                "aa", "aar", NULL},
+  { "AYMARA",              "ay", "aym", NULL},
+  { "BASHKIR",             "ba", "bak", NULL},
+  { "BISLAMA",             "bi", "bis", NULL},
+  { "DZONGKHA",            "dz", "dzo", NULL},
+  { "FIJIAN",              "fj", "fij", NULL},
+  { "GREENLANDIC",         "kl", "kal", NULL},
+  { "HAUSA",               "ha", "hau", NULL},
+  { "HAITIAN_CREOLE",       "ht", NULL, NULL},
+  { "INUPIAK",             "ik", "ipk", NULL},
+  { "INUKTITUT",           "iu", "iku", NULL},
+  { "KASHMIRI",            "ks", "kas", NULL},
+  { "KINYARWANDA",         "rw", "kin", NULL},
+  { "MALAGASY",            "mg", "mlg", NULL},
+  { "NAURU",               "na", "nau", NULL},
+  { "OROMO",               "om", "orm", NULL},
+  { "RUNDI",               "rn", "run", NULL},
+  { "SAMOAN",              "sm", "smo", NULL},
+  { "SANGO",               "sg", "sag", NULL},
+  { "SANSKRIT",            "sa", "san", NULL},
+  { "SISWANT",             "ss", "ssw", NULL},
+  { "TSONGA",              "ts", "tso", NULL},
+  { "TSWANA",              "tn", "tsn", NULL},
+  { "VOLAPUK",             "vo", "vol", NULL},
+  { "ZHUANG",              "za", "zha", NULL},
+  { "KHASI",               NULL, "kha", NULL},
+  { "SCOTS",               NULL, "sco", NULL},
+  { "GANDA",               "lg", "lug", NULL},
+  { "MANX",                "gv", "glv", NULL},
+  { "MONTENEGRIN",         NULL, NULL, "sr-ME"},
+  { "XX",                  NULL, NULL, "XX"},
+};
+
+COMPILE_ASSERT(arraysize(kLanguageInfoTable) == NUM_LANGUAGES + 1,
+               kLanguageInfoTable_has_incorrect_length);
+
+
+// LANGUAGE NAMES
+
+const char* default_language_name() {
+  return kLanguageInfoTable[ENGLISH].language_name_;
+}
+
+static const char* const kInvalidLanguageName = "invalid_language";
+
+const char *invalid_language_name() {
+  return kInvalidLanguageName;
+}
+
+const char* LanguageName(Language lang) {
+  return IsValidLanguage(lang)
+      ? kLanguageInfoTable[lang].language_name_
+      : kInvalidLanguageName;
+}
+
+
+
+// LANGUAGE CODES
+
+
+// The space before invalid_language_code is intentional. It is used
+// to prevent it matching any two letter language code.
+//
+static const char* const kInvalidLanguageCode = " invalid_language_code";
+
+const char *invalid_language_code() {
+  return kInvalidLanguageCode;
+}
+
+const char * LanguageCode(Language lang) {
+  if (! IsValidLanguage(lang))
+    return kInvalidLanguageCode;
+  const LanguageInfo& info = kLanguageInfoTable[lang];
+  if (info.language_code_639_1_) {
+    return info.language_code_639_1_;
+  } else if (info.language_code_639_2_) {
+    return info.language_code_639_2_;
+  } else if (info.language_code_other_) {
+    return info.language_code_other_;
+  } else {
+    return kInvalidLanguageCode;
+  }
+}
+
+const char* default_language_code() {
+  return kLanguageInfoTable[ENGLISH].language_code_639_1_;
+}
+
+const char* LanguageCodeISO639_1(Language lang) {
+  if (! IsValidLanguage(lang))
+    return kInvalidLanguageCode;
+  if (const char* code = kLanguageInfoTable[lang].language_code_639_1_)
+    return code;
+  return kInvalidLanguageCode;
+}
+
+const char* LanguageCodeISO639_2(Language lang) {
+  if (! IsValidLanguage(lang))
+    return kInvalidLanguageCode;
+  if (const char* code = kLanguageInfoTable[lang].language_code_639_2_)
+    return code;
+  return kInvalidLanguageCode;
+}
+
+const char* LanguageCodeWithDialects(Language lang) {
+  if (lang == CHINESE)
+    return "zh-CN";
+  return LanguageCode(lang);
+}
+
+
+
+bool LanguageFromCode(const char* lang_code, Language *language) {
+  *language = UNKNOWN_LANGUAGE;
+  if ( lang_code == NULL ) return false;
+
+  for ( int i = 0 ; i < kNumLanguages ; i++ ) {
+    const LanguageInfo& info = kLanguageInfoTable[i];
+    if ((info.language_code_639_1_ &&
+         !base::strcasecmp(lang_code, info.language_code_639_1_)) ||
+        (info.language_code_639_2_ &&
+         !base::strcasecmp(lang_code, info.language_code_639_2_)) ||
+        (info.language_code_other_ &&
+         !base::strcasecmp(lang_code, info.language_code_other_))) {
+      *language = static_cast<Language>(i);
+      return true;
+    }
+  }
+
+  // For convenience, this function can also parse the non-standard
+  // five-letter language codes "zh-cn" and "zh-tw" which are used by
+  // front-ends such as GWS to distinguish Simplified from Traditional
+  // Chinese.
+  if (!base::strcasecmp(lang_code, "zh-cn") ||
+      !base::strcasecmp(lang_code, "zh_cn")) {
+    *language = CHINESE;
+    return true;
+  }
+  if (!base::strcasecmp(lang_code, "zh-tw") ||
+      !base::strcasecmp(lang_code, "zh_tw")) {
+    *language = CHINESE_T;
+    return true;
+  }
+  if (!base::strcasecmp(lang_code, "sr-me") ||
+      !base::strcasecmp(lang_code, "sr_me")) {
+    *language = MONTENEGRIN;
+    return true;
+  }
+
+  // Process language-code synonyms.
+  if (!base::strcasecmp(lang_code, "he")) {
+    *language = HEBREW;  // Use "iw".
+    return true;
+  }
+  if (!base::strcasecmp(lang_code, "in")) {
+    *language = INDONESIAN;  // Use "id".
+    return true;
+  }
+  if (!base::strcasecmp(lang_code, "ji")) {
+    *language = YIDDISH;  // Use "yi".
+    return true;
+  }
+
+  // Process language-detection synonyms.
+  // These distinct languages cannot be differentiated by our current
+  // language-detection algorithms.
+  if (!base::strcasecmp(lang_code, "fil")) {
+    *language = TAGALOG;
+    return true;
+  }
+
+  return false;
+}

contrib/google-ced/util/languages/languages.h

@@ -0,0 +1,381 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_LANGUAGES_LANGUAGES_H_
+#define UTIL_LANGUAGES_LANGUAGES_H_
+
+// This interface defines the Language enum and functions that depend
+// only on Language values.
+
+// A hash-function for Language, hash<Language>, is defined in
+// i18n/languages/public/languages-hash.h
+
+#ifndef SWIG
+// Language enum defined in languages.proto
+// Also description on how to add languages.
+#include "util/languages/languages.pb.h"
+
+#else
+
+// TODO: Include a header containing swig-compatible enum.
+
+#endif
+
+const int kNumLanguages = NUM_LANGUAGES;
+
+// Return the default language (ENGLISH).
+Language default_language();
+
+
+// *******************************************
+// Language predicates
+//   IsValidLanguage()
+//   IS_LANGUAGE_UNKNOWN()
+//   IsCJKLanguage()
+//   IsChineseLanguage()
+//   IsNorwegianLanguage()
+//   IsPortugueseLanguage()
+//   IsRightToLeftLanguage()
+//   IsMaybeRightToLeftLanguage()
+//   IsSameLanguage()
+//   IsScriptRequiringLongerSnippets()
+// *******************************************
+
+// IsValidLanguage
+// ===============
+//
+// Function to check if the input is within range of the Language enum. If
+// IsValidLanguage(lang) returns true, it is safe to call
+// static_cast<Language>(lang).
+//
+inline bool IsValidLanguage(int lang) {
+  return ((lang >= 0) && (lang < kNumLanguages));
+}
+
+// Return true if the language is "unknown". (This function was
+// previously a macro, hence the spelling in all caps.)
+//
+inline bool IS_LANGUAGE_UNKNOWN(Language lang) {
+  return lang == TG_UNKNOWN_LANGUAGE || lang == UNKNOWN_LANGUAGE;
+}
+
+// IsCJKLanguage
+// -------------
+//
+// This function returns true if the language is either Chinese
+// (simplified or traditional), Japanese, or Korean.
+bool IsCJKLanguage(Language lang);
+
+// IsChineseLanguage
+// -----------------
+//
+// This function returns true if the language is either Chinese
+// (simplified or traditional)
+bool IsChineseLanguage(Language lang);
+
+// IsNorwegianLanguage
+// --------------------
+//
+// This function returns true if the language is any of the Norwegian
+// (regular or Nynorsk).
+bool IsNorwegianLanguage(Language lang);
+
+// IsPortugueseLanguage
+// --------------------
+//
+// This function returns true if the language is any of the Portuguese
+// languages (regular, Portugal or Brazil)
+bool IsPortugueseLanguage(Language lang);
+
+// IsSameLanguage
+// --------------
+//
+// WARNING: This function provides only a simple test on the values of
+// the two Language arguments. It returns false if either language is
+// invalid. It returns true if the language arguments are equal, or
+// if they are both Chinese languages, both Norwegian languages, or
+// both Portuguese languages, as defined by IsChineseLanguage,
+// IsNorwegianLanguage, and IsPortugueseLanguage. Otherwise it returns
+// false.
+bool IsSameLanguage(Language lang1, Language lang2);
+
+
+// IsRightToLeftLanguage
+// ---------------------
+//
+// This function returns true if the language is only written right-to-left
+// (E.g., Hebrew, Arabic, Persian etc.)
+//
+// IMPORTANT NOTE: Technically we're talking about scripts, not languages.
+// There are languages that can be written in more than one script.
+// Examples:
+//   - Kurdish and Azeri ('AZERBAIJANI') can be written left-to-right in
+//     Latin or Cyrillic script, and right-to-left in Arabic script.
+//   - Sindhi and Punjabi are written in different scripts, depending on
+//     region and dialect.
+//   - Turkmen used an Arabic script historically, but not any more.
+//   - Pashto and Uyghur can use Arabic script, but use a Roman script
+//     on the Internet.
+//   - Kashmiri and Urdu are written either with Arabic or Devanagari script.
+//
+// This function only returns true for languages that are always, unequivocally
+// written in right-to-left script.
+//
+// TODO: If we want to do anything special with multi-script languages
+// we should create new 'languages' for each language+script, as we do for
+// traditional vs. simplified Chinese. However most such languages are rare in
+// use and even rarer on the web, so this is unlikely to be something we'll
+// be concerned with for a while.
+bool IsRightToLeftLanguage(Language lang);
+
+// IsMaybeRightToLeftLanguage
+// --------------------------
+//
+// This function returns true if the language may appear on the web in a
+// right-to-left script (E.g., Hebrew, Arabic, Persian, Urdu, Kurdish, etc.)
+//
+// NOTE: See important notes under IsRightToLeftLanguage(...).
+//
+// This function returns true for languages that *may* appear on the web in a
+// right-to-left script, even if they may also appear in a left-to-right
+// script.
+//
+// This function should typically be used in cases where doing some work on
+// left-to-right text would be OK (usually a no-op), and this function is used
+// just to cut down on unnecessary work on regular, LTR text.
+bool IsMaybeRightToLeftLanguage(Language lang);
+
+// IsScriptRequiringLongerSnippets
+// --------------------
+//
+// This function returns true if the script chracteristics require longer
+// snippet length (Devanagari, Bengali, Gurmukhi,
+// Gujarati, Oriya, Tamil, Telugu, Kannada, Malayalam).
+// COMMENTED OUT TO REDUCE DEPENDENCIES ON GOOGLE3 CODE
+// bool IsScriptRequiringLongerSnippets(UnicodeScript script);
+
+
+// *******************************************
+// LANGUAGE NAMES
+//
+// This interface defines a standard name for each valid Language,
+// and a standard name for invalid languages. Some language names use all
+// uppercase letters, but others use mixed case.
+//   LanguageName() [Language to name]
+//   LanguageEnumName() [language to enum name]
+//   LanguageFromName() [name to Language]
+//   default_language_name()
+//   invalid_language_name()
+// *******************************************
+
+// Given a Language, returns its standard name.
+// Return invalid_language_name() if the language is invalid.
+const char* LanguageName(Language lang);
+
+// Given a Language, return the name of the enum constant for that
+// language. In all but a few cases, this is the same as its standard
+// name. For example, LanguageName(CHINESE) returns "Chinese", but
+// LanguageEnumName(CHINESE) returns "CHINESE". This is intended for
+// code that is generating C++ code, where the enum constant is more
+// useful than its integer value.  Return "NUM_LANGUAGES" if
+// the language is invalid.
+const char* LanguageEnumName(Language lang);
+
+// The maximum length of a standard language name.
+const int kMaxLanguageNameSize = 50;
+
+// The standard name for the default language.
+const char* default_language_name();
+
+// The standard name for all invalid languages.
+const char* invalid_language_name();
+
+// If lang_name matches the standard name of a Language, using a
+// case-insensitive comparison, set *language to that Language and
+// return true.
+// Otherwise, set *language to UNKNOWN_LANGUAGE and return false.
+//
+// For backwards compatibility, "HATIAN_CREOLE" is allowed as a name
+// for HAITIAN_CREOLE, and "QUECHAU" is allowed as a name for QUECHUA.
+// For compatibility with LanguageEnumName, "UNKNOWN_LANGUAGE" is allowed
+// as a name for UNKNOWN_LANGUAGE (the return value is true in this case,
+// as it is for "Unknown"), and "CHINESE_T" is allowed as a name for
+// CHINESE_T (i.e., a synonym for "ChineseT").
+//
+// REQUIRES: language must not be NULL.
+//
+bool LanguageFromName(const char* lang_name, Language *language);
+
+
+
+// *******************************************
+// LANGUAGE CODES
+//
+// This interface defines a standard code for each valid language, and
+// a standard code for invalid languages. These are derived from ISO codes,
+// with some Google additions.
+//   LanguageCode()
+//   default_language_code()
+//   invalid_language_code()
+//   LanguageCodeWithDialects()
+//   LanguageCodeISO639_1()
+//   LanguageCodeISO639_2()
+// *******************************************
+
+// Given a Language, return its standard code. There are Google-specific codes:
+//     For CHINESE_T, return "zh-TW".
+//     For TG_UNKNOWN_LANGUAGE, return "ut".
+//     For UNKNOWN_LANGUAGE, return "un".
+//     For PORTUGUESE_P, return "pt-PT".
+//     For PORTUGUESE_B, return "pt-BR".
+//     For LIMBU, return "sit-NP".
+//     For CHEROKEE, return "chr".
+//     For SYRIAC, return "syr".
+// Otherwise return the ISO 639-1 two-letter language code for lang.
+// If lang is invalid, return invalid_language_code().
+//
+// NOTE: See the note below about the codes for Chinese languages.
+//
+const char* LanguageCode(Language lang);
+
+// The maximum length of a language code.
+const int kMaxLanguageCodeSize = 50;
+
+// The standard code for the default language.
+const char* default_language_code();
+
+// The standard code for all invalid languages.
+const char* invalid_language_code();
+
+
+// --------------------------------------------
+// NOTE: CHINESE LANGUAGE CODES
+//
+// There are three functions that return codes for Chinese languages.
+// LanguageCode(lang) and LanguageCodeWithDialects(lang) are defined here.
+// LanguageCode(lang, encoding) is defined in i18n/encodings.lang_enc.h.
+// The following list shows the different results.
+//
+// LanguageCode(CHINESE) returns "zh"
+// LanguageCode(CHINESE_T) returns "zh-TW".
+//
+// LanguageCodeWithDialects(CHINESE) returns "zh-CN".
+// LanguageCodeWithDialects(CHINESE_T) returns "zh-TW".
+//
+// LanguageCode(CHINESE_T, <any encoding>) returns "zh-TW".
+// LanguageCode(CHINESE, CHINESE_BIG5) returns "zh-TW".
+// LanguageCode(CHINESE, <any other encoding>) returns "zh-CN".
+//
+// --------------------------------------------
+
+// LanguageCodeWithDialects
+// ------------------------
+//
+// If lang is CHINESE, return "zh-CN". Otherwise return LanguageCode(lang).
+const char* LanguageCodeWithDialects(Language lang);
+
+// LanguageCodeISO639_1
+// --------------------
+//
+// Return the ISO 639-1 two-letter language code for lang.
+// Return invalid_language_code() if lang is invalid or does not have
+// an ISO 639-1 two-letter language code.
+const char* LanguageCodeISO639_1(Language lang);
+
+// LanguageCodeISO639_2
+// --------------------
+//
+// Return the ISO 639-2 three-letter language for lang.
+// Return invalid_language_code() if lang is invalid or does not have
+// an ISO 639-2 three-letter language code.
+const char* LanguageCodeISO639_2(Language lang);
+
+// LanguageFromCode
+// ----------------
+//
+// If lang_code matches the code for a Language, using a case-insensitive
+// comparison, set *lang to that Language and return true.
+// Otherwise, set *lang to UNKNOWN_LANGUAGE and return false.
+//
+// lang_code can be an ISO 639-1 (two-letter) code, an ISO 639-2
+// (three-letter) code, or a Google-specific code (see LanguageCode).
+//
+// Certain language-code aliases are also allowed:
+//   For "zh-cn" and "zh_cn", set *lang to CHINESE.
+//   For "zh-tw" and "zh_tw", set *lang to CHINESE_T.
+//   For "he", set *lang to HEBREW.
+//   For "in", set *lang to INDONESIAN.
+//   For "ji", set *lang to YIDDISH.
+//   For "fil", set *lang to TAGALOG.
+//
+// REQUIRES: 'lang' must not be NULL.
+bool LanguageFromCode(const char* lang_code, Language *language);
+
+
+// LanguageFromCodeOrName
+// ----------------------
+//
+// If lang_code_or_name is a language code or a language name.
+// set *language to the corresponding Language and return true.
+// Otherwise set *language to UNKNOWN_LANGUAGE and return false.
+//
+bool LanguageFromCodeOrName(const char* lang_code_or_name,
+                            Language* language);
+
+// LanguageNameFromCode
+// --------------------
+//
+// If language_code is the code for a Language (see LanguageFromCode),
+// return the standard name of that language (see LanguageName).
+// Otherwise return invalid_language_name().
+//
+const char* LanguageNameFromCode(const char* language_code);
+
+
+// Miscellany
+
+// LanguageCodeToUnderscoreForm
+// ----------------------------
+//
+// Given a language code, convert the dash "-" to underscore "_".
+//
+// Specifically, if result_length <= strlen(lang_code), set result[0]
+// to '\0' and return false. Otherwise, copy lang_code to result,
+// converting every dash to an underscore, converting every character
+// before the first dash or underscore to lower case, and converting
+// every character after the first dash or underscore to upper
+// case. If there is no dash or underscore, convert the entire string
+// to lower case.
+//
+// REQUIRES: 'lang_code' must not be NULL. 'result' must not be NULL.
+
+bool LanguageCodeToUnderscoreForm(const char* lang_code,
+                                  char* result,
+                                  int result_length);
+
+//
+// AlwaysPutInExpectedRestrict
+// ---------------------------
+//
+// For Web pages in certain top-level domains, Web Search always
+// applies a "country restrict". If 'tld' matches one of those, using
+// a case-SENSITIVE comparison, set *expected_language to the Language
+// most commonly found in that top-level domain and return true.
+// Otherwise, set *expected_language to UNKNOWN_LANGUAGE and return false.
+bool AlwaysPutInExpectedRestrict(const char *tld, Language *expected_language);
+
+
+#endif  // UTIL_LANGUAGES_LANGUAGES_H_

contrib/google-ced/util/languages/languages.pb.h

@@ -0,0 +1,191 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_LANGUAGES_LANGUAGES_PB_H_
+#define UTIL_LANGUAGES_LANGUAGES_PB_H_
+
+enum Language {
+  ENGLISH       = 0,
+  DANISH        = 1,
+  DUTCH         = 2,
+  FINNISH       = 3,
+  FRENCH        = 4,
+  GERMAN        = 5,
+  HEBREW        = 6,
+  ITALIAN       = 7,
+  JAPANESE      = 8,
+  KOREAN        = 9,
+  NORWEGIAN     = 10,
+  POLISH        = 11,
+  PORTUGUESE    = 12,
+  RUSSIAN       = 13,
+  SPANISH       = 14,
+  SWEDISH       = 15,
+  CHINESE       = 16,
+  CZECH         = 17,
+  GREEK         = 18,
+  ICELANDIC     = 19,
+  LATVIAN       = 20,
+  LITHUANIAN    = 21,
+  ROMANIAN      = 22,
+  HUNGARIAN     = 23,
+  ESTONIAN      = 24,
+  TG_UNKNOWN_LANGUAGE   = 25,
+  UNKNOWN_LANGUAGE      = 26,
+  BULGARIAN     = 27,
+  CROATIAN      = 28,
+  SERBIAN       = 29,
+  IRISH         = 30,      // UI only.
+  GALICIAN      = 31,
+  TAGALOG       = 32,      // Tagalog (tl) + Filipino (fil),
+  TURKISH       = 33,
+  UKRAINIAN     = 34,
+  HINDI         = 35,
+  MACEDONIAN    = 36,
+  BENGALI       = 37,
+  INDONESIAN    = 38,
+  LATIN         = 39,      // UI only.
+  MALAY         = 40,
+  MALAYALAM     = 41,
+  WELSH         = 42,      // UI only.
+  NEPALI        = 43,
+  TELUGU        = 44,
+  ALBANIAN      = 45,
+  TAMIL         = 46,
+  BELARUSIAN    = 47,
+  JAVANESE      = 48,      // UI only.
+  OCCITAN       = 49,      // UI only.
+  URDU          = 50,
+  BIHARI        = 51,
+  GUJARATI      = 52,
+  THAI          = 53,
+  ARABIC        = 54,
+  CATALAN       = 55,
+  ESPERANTO     = 56,
+  BASQUE        = 57,
+  INTERLINGUA   = 58,      // UI only.
+  KANNADA       = 59,
+  PUNJABI       = 60,
+  SCOTS_GAELIC  = 61,      // UI only.
+  SWAHILI       = 62,
+  SLOVENIAN     = 63,
+  MARATHI       = 64,
+  MALTESE       = 65,
+  VIETNAMESE    = 66,
+  FRISIAN       = 67,      // UI only.
+  SLOVAK        = 68,
+  CHINESE_T     = 69,      // This is added to solve the problem of
+                           // distinguishing Traditional and Simplified
+                           // Chinese when the encoding is UTF8.
+  FAROESE       = 70,      // UI only.
+  SUNDANESE     = 71,      // UI only.
+  UZBEK         = 72,
+  AMHARIC       = 73,
+  AZERBAIJANI   = 74,
+  GEORGIAN      = 75,
+  TIGRINYA      = 76,      // UI only.
+  PERSIAN       = 77,
+  BOSNIAN       = 78,      // UI only. LangId language: CROATIAN (28)
+  SINHALESE     = 79,
+  NORWEGIAN_N   = 80,      // UI only. LangId language: NORWEGIAN (10)
+  PORTUGUESE_P  = 81,      // UI only. LangId language: PORTUGUESE (12)
+  PORTUGUESE_B  = 82,      // UI only. LangId language: PORTUGUESE (12)
+  XHOSA         = 83,      // UI only.
+  ZULU          = 84,      // UI only.
+  GUARANI       = 85,
+  SESOTHO       = 86,      // UI only.
+  TURKMEN       = 87,      // UI only.
+  KYRGYZ        = 88,
+  BRETON        = 89,      // UI only.
+  TWI           = 90,      // UI only.
+  YIDDISH       = 91,      // UI only.
+  SERBO_CROATIAN= 92,      // UI only. LangId language: SERBIAN (29)
+  SOMALI        = 93,      // UI only.
+  UIGHUR        = 94,
+  KURDISH       = 95,
+  MONGOLIAN     = 96,
+  ARMENIAN      = 97,
+  LAOTHIAN      = 98,
+  SINDHI        = 99,
+  RHAETO_ROMANCE= 100,     // UI only.
+  AFRIKAANS     = 101,
+  LUXEMBOURGISH = 102,     // UI only.
+  BURMESE       = 103,
+  KHMER         = 104,
+  TIBETAN       = 105,
+  DHIVEHI       = 106,     // sometimes spelled Divehi, lang of Maldives
+  CHEROKEE      = 107,
+  SYRIAC        = 108,     // UI only.
+  LIMBU         = 109,     // UI only.
+  ORIYA         = 110,
+  ASSAMESE      = 111,     // UI only.
+  CORSICAN      = 112,     // UI only.
+  INTERLINGUE   = 113,     // UI only.
+  KAZAKH        = 114,
+  LINGALA       = 115,     // UI only.
+  MOLDAVIAN     = 116,     // UI only. LangId language: ROMANIAN (22)
+  PASHTO        = 117,
+  QUECHUA       = 118,     // UI only.
+  SHONA         = 119,     // UI only.
+  TAJIK         = 120,
+  TATAR         = 121,     // UI only.
+  TONGA         = 122,     // UI only.
+  YORUBA        = 123,     // UI only.
+  CREOLES_AND_PIDGINS_ENGLISH_BASED       = 124,   // UI only.
+  CREOLES_AND_PIDGINS_FRENCH_BASED        = 125,   // UI only.
+  CREOLES_AND_PIDGINS_PORTUGUESE_BASED    = 126,   // UI only.
+  CREOLES_AND_PIDGINS_OTHER               = 127,   // UI only.
+  MAORI         = 128,     // UI only.
+  WOLOF         = 129,     // UI only.
+  ABKHAZIAN     = 130,     // UI only.
+  AFAR          = 131,     // UI only.
+  AYMARA        = 132,     // UI only.
+  BASHKIR       = 133,     // UI only.
+  BISLAMA       = 134,     // UI only.
+  DZONGKHA      = 135,     // UI only.
+  FIJIAN        = 136,     // UI only.
+  GREENLANDIC   = 137,     // UI only.
+  HAUSA         = 138,     // UI only.
+  HAITIAN_CREOLE= 139,     // UI only.
+  INUPIAK       = 140,     // UI only.
+  INUKTITUT     = 141,
+  KASHMIRI      = 142,     // UI only.
+  KINYARWANDA   = 143,     // UI only.
+  MALAGASY      = 144,     // UI only.
+  NAURU         = 145,     // UI only.
+  OROMO         = 146,     // UI only.
+  RUNDI         = 147,     // UI only.
+  SAMOAN        = 148,     // UI only.
+  SANGO         = 149,     // UI only.
+  SANSKRIT      = 150,
+  SISWANT       = 151,     // UI only.
+  TSONGA        = 152,     // UI only.
+  TSWANA        = 153,     // UI only.
+  VOLAPUK       = 154,     // UI only.
+  ZHUANG        = 155,     // UI only.
+  KHASI         = 156,     // UI only.
+  SCOTS         = 157,     // UI only.
+  GANDA         = 158,     // UI only.
+  MANX          = 159,     // UI only.
+  MONTENEGRIN   = 160,     // UI only. LangId language: SERBIAN (29)
+  NUM_LANGUAGES = 161,        // Always keep this at the end. It is not a
+                              // valid Language enum. It is only used to
+                              // indicate the total number of Languages.
+  // NOTE: If you add a language, you will break a unittest. See the note
+  // at the top of this enum.
+};
+
+#endif  // UTIL_LANGUAGES_LANGUAGES_PB_H_

contrib/google-ced/util/logging.h

@@ -0,0 +1,25 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_LOGGING_H_
+#define UTIL_LOGGING_H_
+
+#undef CHECK
+#define CHECK(expr)
+#undef DCHECK
+#define DCHECK(expr)
+
+#endif  // UTIL_LOGGING_H_

contrib/google-ced/util/port.h

@@ -0,0 +1,53 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_PORT_H_
+#define UTIL_PORT_H_
+
+#include <stdarg.h>
+
+#if defined(_MSC_VER)
+#define GG_LONGLONG(x) x##I64
+#define GG_ULONGLONG(x) x##UI64
+#else
+#define GG_LONGLONG(x) x##LL
+#define GG_ULONGLONG(x) x##ULL
+#endif
+
+// Per C99 7.8.14, define __STDC_CONSTANT_MACROS before including <stdint.h>
+// to get the INTn_C and UINTn_C macros for integer constants.  It's difficult
+// to guarantee any specific ordering of header includes, so it's difficult to
+// guarantee that the INTn_C macros can be defined by including <stdint.h> at
+// any specific point.  Provide GG_INTn_C macros instead.
+
+#define GG_INT8_C(x)    (x)
+#define GG_INT16_C(x)   (x)
+#define GG_INT32_C(x)   (x)
+#define GG_INT64_C(x)   GG_LONGLONG(x)
+
+#define GG_UINT8_C(x)   (x ## U)
+#define GG_UINT16_C(x)  (x ## U)
+#define GG_UINT32_C(x)  (x ## U)
+#define GG_UINT64_C(x)  GG_ULONGLONG(x)
+
+// Define an OS-neutral wrapper for shared library entry points
+#if defined(_WIN32)
+#define API_CALL __stdcall
+#else
+#define API_CALL
+#endif
+
+#endif  // UTIL_PORT_H_

contrib/google-ced/util/string_util.h

@@ -0,0 +1,61 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_STRING_UTIL_H_
+#define UTIL_STRING_UTIL_H_
+
+#include <string.h>
+
+namespace base {
+
+#if defined(_WIN32)
+// Compare the two strings s1 and s2 without regard to case using
+// the current locale; returns 0 if they are equal, 1 if s1 > s2, and -1 if
+// s2 > s1 according to a lexicographic comparison.
+inline int strcasecmp(const char* s1, const char* s2) {
+  return _stricmp(s1, s2);
+}
+inline int strncasecmp(const char* s1, const char* s2, size_t n) {
+  return _strnicmp(s1, s2, n);
+}
+#else
+inline int strcasecmp(const char* s1, const char* s2) {
+  return ::strcasecmp(s1, s2);
+}
+inline int strncasecmp(const char* s1, const char* s2, size_t n) {
+  return ::strncasecmp(s1, s2, n);
+}
+#endif
+}
+
+#ifndef HAVE_MEMRCHR
+#if defined(__GLIBC__) && ((__GLIBC__ > 2) || ((__GLIBC__ == 2) && (__GLIBC_MINOR__ >= 2)))
+#define HAVE_MEMRCHR
+#endif
+#endif
+
+#ifndef HAVE_MEMRCHR
+inline void* memrchr(const void* s, int c, size_t n) {
+  const unsigned char* p = (const unsigned char*) s;
+  for (p += n; n > 0; n--) {
+    if (*--p == c)
+      return (void*) p;
+  }
+  return NULL;
+}
+#endif
+
+#endif  // UTIL_STRING_UTIL_H_

contrib/google-ced/util/varsetter.h

@@ -0,0 +1,66 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef UTIL_VARSETTER_H_
+#define UTIL_VARSETTER_H_
+
+//
+// Use a VarSetter object to temporarily set an object of some sort to
+// a particular value.  When the VarSetter object is destructed, the
+// underlying object will revert to its former value.
+//
+// Sample code:
+//
+#if 0
+{
+  bool b = true;
+  {
+    VarSetter<bool> bool_setter(&b, false);
+    // Now b == false.
+  }
+  // Now b == true again.
+}
+#endif
+
+template <class C>
+class VarSetter {
+public:
+
+  // Constructor that just sets the object to a fixed value
+  VarSetter(C* object, const C& value) : object_(object), old_value_(*object) {
+    *object = value;
+  }
+
+  ~VarSetter() { *object_ = old_value_; }
+
+private:
+
+  C*const object_;
+  C old_value_;
+
+  // Disallow
+  VarSetter(const VarSetter&);
+  VarSetter& operator=(const VarSetter&);
+
+  // VarSetters always live on the stack
+  static void* operator new (size_t);
+  static void* operator new[](size_t);  // Redundant, no default ctor
+
+  static void operator delete (void*);
+  static void operator delete[](void*);
+};
+
+#endif  // UTIL_VARSETTER_H_

src/libmime/mime_encoding.c

@@ -23,6 +23,7 @@
 #include "mime_encoding.h"
 #include "message.h"
 #include "contrib/fastutf8/fastutf8.h"
+#include "contrib/google-ced/ced_c.h"
 #include <unicode/ucnv.h>
 #include <unicode/ucsdet.h>
 #if U_ICU_VERSION_MAJOR_NUM >= 44
@@ -561,38 +562,22 @@ rspamd_mime_charset_utf_enforce (gchar *in, gsize len)
 const char *
 rspamd_mime_charset_find_by_content (const gchar *in, gsize inlen)
 {
-	static UCharsetDetector *csd;
-	const UCharsetMatch **csm, *sel = NULL;
-	UErrorCode uc_err = U_ZERO_ERROR;
-	gint32 matches, i, max_conf = G_MININT32, conf;
-	gdouble mean = 0.0, stddev = 0.0;
-
-	if (csd == NULL) {
-		csd = ucsdet_open (&uc_err);
-
-		g_assert (csd != NULL);
-	}
+	int nconsumed;
+	bool is_reliable;
+	const gchar *ced_name;
 
 	if (rspamd_fast_utf8_validate (in, inlen) == 0) {
 		return UTF8_CHARSET;
 	}
 
-	ucsdet_setText (csd, in, inlen, &uc_err);
-	csm = ucsdet_detectAll (csd, &matches, &uc_err);
 
-	for (i = 0; i < matches; i ++) {
-		if ((conf = ucsdet_getConfidence (csm[i], &uc_err)) > max_conf) {
-			max_conf = conf;
-			sel = csm[i];
-		}
+	ced_name = ced_encoding_detect (in, inlen, NULL, NULL,
+			NULL, 0, CED_EMAIL_CORPUS,
+			false, &nconsumed, &is_reliable);
 
-		mean += (conf - mean) / (i + 1);
-		gdouble err = fabs (conf - mean);
-		stddev += (err - stddev) / (i + 1);
-	}
+	if (ced_name) {
 
-	if (sel && ((max_conf > 50) || (max_conf - mean > stddev * 1.25))) {
-		return ucsdet_getName (sel, &uc_err);
+		return ced_name;
 	}
 
 	return NULL;