From 0cf15476a35bfada850efc07eb78bc2bb041a8e6 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Lang Date: Tue, 26 Aug 2008 12:13:15 +0000 Subject: [PATCH] Adds support for commit logs reencoding to UTF-8 before insertion in the database (#834, #917, #1663). Source encoding of commit logs can be selected in Application settings -> Repositories. git-svn-id: http://redmine.rubyforge.org/svn/trunk@1767 e93f8b46-1217-0410-a6f0-8f06a7374b81 --- app/models/changeset.rb | 19 ++++++++++++- app/models/setting.rb | 39 ++++++++++++++++++++++++++ app/views/settings/_repositories.rhtml | 3 ++ config/settings.yml | 3 ++ lang/bg.yml | 1 + lang/cs.yml | 1 + lang/da.yml | 1 + lang/de.yml | 1 + lang/en.yml | 1 + lang/es.yml | 1 + lang/fi.yml | 1 + lang/fr.yml | 1 + lang/he.yml | 1 + lang/hu.yml | 1 + lang/it.yml | 1 + lang/ja.yml | 1 + lang/ko.yml | 1 + lang/lt.yml | 1 + lang/nl.yml | 1 + lang/no.yml | 1 + lang/pl.yml | 1 + lang/pt-br.yml | 1 + lang/pt.yml | 1 + lang/ro.yml | 1 + lang/ru.yml | 1 + lang/sr.yml | 1 + lang/sv.yml | 1 + lang/th.yml | 1 + lang/uk.yml | 1 + lang/zh-tw.yml | 1 + lang/zh.yml | 1 + 31 files changed, 90 insertions(+), 1 deletion(-) diff --git a/app/models/changeset.rb b/app/models/changeset.rb index 0663af34e..656c67544 100644 --- a/app/models/changeset.rb +++ b/app/models/changeset.rb @@ -15,6 +15,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +require 'iconv' + class Changeset < ActiveRecord::Base belongs_to :repository has_many :changes, :dependent => :delete_all @@ -43,7 +45,7 @@ class Changeset < ActiveRecord::Base end def comments=(comment) - write_attribute(:comments, comment.strip) + write_attribute(:comments, to_utf8(comment.to_s.strip)) end def committed_on=(date) @@ -131,4 +133,19 @@ class Changeset < ActiveRecord::Base def next @next ||= Changeset.find(:first, :conditions => ['id > ? AND repository_id = ?', self.id, self.repository_id], :order => 'id ASC') end + + private + + def to_utf8(str) + return str if /\A[\r\n\t\x20-\x7e]*\Z/n.match(str) # for us-ascii + encoding = Setting.commit_logs_encoding.to_s.strip + unless encoding.blank? || encoding == 'UTF-8' + begin + return Iconv.conv('UTF-8', encoding, str) + rescue Iconv::Failure + # do nothing here + end + end + str + end end diff --git a/app/models/setting.rb b/app/models/setting.rb index 185991d9b..072afa0db 100644 --- a/app/models/setting.rb +++ b/app/models/setting.rb @@ -33,6 +33,45 @@ class Setting < ActiveRecord::Base '%H:%M', '%I:%M %p' ] + + ENCODINGS = %w(US-ASCII + windows-1250 + windows-1251 + windows-1252 + windows-1253 + windows-1254 + windows-1255 + windows-1256 + windows-1257 + windows-1258 + windows-31j + ISO-2022-JP + ISO-2022-KR + ISO-8859-1 + ISO-8859-2 + ISO-8859-3 + ISO-8859-4 + ISO-8859-5 + ISO-8859-6 + ISO-8859-7 + ISO-8859-8 + ISO-8859-9 + ISO-8859-13 + ISO-8859-15 + KOI8-R + UTF-8 + UTF-16 + UTF-16BE + UTF-16LE + EUC-JP + Shift_JIS + GB18030 + GBK + ISCII91 + EUC-KR + Big5 + Big5-HKSCS + TIS-620) cattr_accessor :available_settings @@available_settings = YAML::load(File.open("#{RAILS_ROOT}/config/settings.yml")) diff --git a/app/views/settings/_repositories.rhtml b/app/views/settings/_repositories.rhtml index 127801be2..a8c924430 100644 --- a/app/views/settings/_repositories.rhtml +++ b/app/views/settings/_repositories.rhtml @@ -16,6 +16,9 @@

<%= text_field_tag 'settings[repositories_encodings]', Setting.repositories_encodings, :size => 60 %>
<%= l(:text_comma_separated) %>

+ +

+<%= select_tag 'settings[commit_logs_encoding]', options_for_select(Setting::ENCODINGS, Setting.commit_logs_encoding) %>

<%= l(:text_issues_ref_in_commit_messages) %> diff --git a/config/settings.yml b/config/settings.yml index 9a59d5686..ac79edb8d 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -122,6 +122,9 @@ default_projects_public: # multiple values accepted, comma separated repositories_encodings: default: '' +# encoding used to convert commit logs to UTF-8 +commit_logs_encoding: + default: 'UTF-8' ui_theme: default: '' emails_footer: diff --git a/lang/bg.yml b/lang/bg.yml index 6a9676119..f15ba4da7 100644 --- a/lang/bg.yml +++ b/lang/bg.yml @@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/cs.yml b/lang/cs.yml index cc5847ee4..b7b1b9eaf 100644 --- a/lang/cs.yml +++ b/lang/cs.yml @@ -639,3 +639,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/da.yml b/lang/da.yml index 1cb55cc48..3227dc7bd 100644 --- a/lang/da.yml +++ b/lang/da.yml @@ -636,3 +636,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/de.yml b/lang/de.yml index f0be84490..5093c31fd 100644 --- a/lang/de.yml +++ b/lang/de.yml @@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/en.yml b/lang/en.yml index 6c4a7bd8a..6cd6d4270 100644 --- a/lang/en.yml +++ b/lang/en.yml @@ -209,6 +209,7 @@ setting_time_format: Time format setting_cross_project_issue_relations: Allow cross-project issue relations setting_issue_list_default_columns: Default columns displayed on the issue list setting_repositories_encodings: Repositories encodings +setting_commit_logs_encoding: Commit messages encoding setting_emails_footer: Emails footer setting_protocol: Protocol setting_per_page_options: Objects per page options diff --git a/lang/es.yml b/lang/es.yml index 7f27d53e5..58d27f28d 100644 --- a/lang/es.yml +++ b/lang/es.yml @@ -637,3 +637,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/fi.yml b/lang/fi.yml index 031d59d63..394ff23f3 100644 --- a/lang/fi.yml +++ b/lang/fi.yml @@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/fr.yml b/lang/fr.yml index 5b621ebaa..e6c4b3340 100644 --- a/lang/fr.yml +++ b/lang/fr.yml @@ -210,6 +210,7 @@ setting_time_format: Format d'heure setting_cross_project_issue_relations: Autoriser les relations entre demandes de différents projets setting_issue_list_default_columns: Colonnes affichées par défaut sur la liste des demandes setting_repositories_encodings: Encodages des dépôts +setting_commit_logs_encoding: Encodage des messages de commit setting_emails_footer: Pied-de-page des emails setting_protocol: Protocole setting_per_page_options: Options d'objets affichés par page diff --git a/lang/he.yml b/lang/he.yml index 9c8fbae7e..652b0ed8e 100644 --- a/lang/he.yml +++ b/lang/he.yml @@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/hu.yml b/lang/hu.yml index a5d208cb6..15c420539 100644 --- a/lang/hu.yml +++ b/lang/hu.yml @@ -635,3 +635,4 @@ setting_mail_handler_api_key: API kulcs text_email_delivery_not_configured: "Az E-mail küldés nincs konfigurálva, és az értesítések ki vannak kapcsolva.\nÁllítsd be az SMTP szervert a config/email.yml fájlban és indítsd újra az alkalmazást, hogy érvénybe lépjen." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/it.yml b/lang/it.yml index 5d62e3255..3e1c1d53b 100644 --- a/lang/it.yml +++ b/lang/it.yml @@ -634,3 +634,4 @@ setting_mail_handler_api_key: chiave API text_email_delivery_not_configured: "La consegna via e-mail non è configurata e le notifiche sono disabilitate.\nConfigura il tuo server SMTP in config/email.yml e riavvia l'applicazione per abilitarle." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/ja.yml b/lang/ja.yml index f3ade750d..e7805393a 100644 --- a/lang/ja.yml +++ b/lang/ja.yml @@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/ko.yml b/lang/ko.yml index 961255e01..c85ce72e2 100644 --- a/lang/ko.yml +++ b/lang/ko.yml @@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/lt.yml b/lang/lt.yml index 58f798506..188faf215 100644 --- a/lang/lt.yml +++ b/lang/lt.yml @@ -637,3 +637,4 @@ setting_mail_handler_api_key: API raktas text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/nl.yml b/lang/nl.yml index 08e300143..fd5bff1f6 100644 --- a/lang/nl.yml +++ b/lang/nl.yml @@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/no.yml b/lang/no.yml index 7af029db9..2cdc0bcd6 100644 --- a/lang/no.yml +++ b/lang/no.yml @@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/pl.yml b/lang/pl.yml index 0a29b7c47..69e04e4dd 100644 --- a/lang/pl.yml +++ b/lang/pl.yml @@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/pt-br.yml b/lang/pt-br.yml index 3ab24aeb0..2b25e023f 100644 --- a/lang/pt-br.yml +++ b/lang/pt-br.yml @@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/pt.yml b/lang/pt.yml index bd32cd140..6505b762a 100644 --- a/lang/pt.yml +++ b/lang/pt.yml @@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/ro.yml b/lang/ro.yml index 2e562e237..4887bcf92 100644 --- a/lang/ro.yml +++ b/lang/ro.yml @@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/ru.yml b/lang/ru.yml index 61e713694..095e4db34 100644 --- a/lang/ru.yml +++ b/lang/ru.yml @@ -638,3 +638,4 @@ setting_mail_handler_api_key: API ключ text_email_delivery_not_configured: "Параметры работы с почтовым сервером не настроены и функция уведомления по email не активна.\nНастроить параметры для вашего SMTP сервера вы можете в файле config/email.yml. Для применения изменений перезапустите приложение." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/sr.yml b/lang/sr.yml index ba79d4ef2..f81d0e054 100644 --- a/lang/sr.yml +++ b/lang/sr.yml @@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/sv.yml b/lang/sv.yml index 9505759c8..5e120b3f2 100644 --- a/lang/sv.yml +++ b/lang/sv.yml @@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/th.yml b/lang/th.yml index 6c4ce30d6..3ae37e551 100644 --- a/lang/th.yml +++ b/lang/th.yml @@ -637,3 +637,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/uk.yml b/lang/uk.yml index 4fa27f7d3..cf5f384b9 100644 --- a/lang/uk.yml +++ b/lang/uk.yml @@ -636,3 +636,4 @@ setting_mail_handler_api_key: API key text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/zh-tw.yml b/lang/zh-tw.yml index 1e6483631..f13331df4 100644 --- a/lang/zh-tw.yml +++ b/lang/zh-tw.yml @@ -635,3 +635,4 @@ enumeration_doc_categories: 文件分類 enumeration_activities: 活動 (時間追蹤) field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding diff --git a/lang/zh.yml b/lang/zh.yml index 7158a7f83..682b59eb3 100644 --- a/lang/zh.yml +++ b/lang/zh.yml @@ -635,3 +635,4 @@ enumeration_activities: 活动（时间跟踪） text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them." field_parent_title: Parent page label_issue_watchers: Watchers +setting_commit_logs_encoding: Commit messages encoding -- 2.39.5