Adds support for commit logs reencoding to UTF-8 before insertion in the database...

Source encoding of commit logs can be selected in Application settings -> Repositories.

git-svn-id: http://redmine.rubyforge.org/svn/trunk@1767 e93f8b46-1217-0410-a6f0-8f06a7374b81

diff --git a/app/models/changeset.rb b/app/models/changeset.rb
index 0663af34e7a32e93ba8d2f57da63a2254b61374c..656c67544f3eebf0eb3d9abf3ec29ed9b7332caf 100644 (file)
--- a/app/models/changeset.rb
+++ b/app/models/changeset.rb
@@ -15,6 +15,8 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
+require 'iconv'
+
 class Changeset < ActiveRecord::Base
   belongs_to :repository
   has_many :changes, :dependent => :delete_all
@@ -43,7 +45,7 @@ class Changeset < ActiveRecord::Base
   end
   
   def comments=(comment)
-    write_attribute(:comments, comment.strip)
+    write_attribute(:comments, to_utf8(comment.to_s.strip))
   end
 
   def committed_on=(date)
@@ -131,4 +133,19 @@ class Changeset < ActiveRecord::Base
   def next
     @next ||= Changeset.find(:first, :conditions => ['id > ? AND repository_id = ?', self.id, self.repository_id], :order => 'id ASC')
   end
+  
+  private
+  
+  def to_utf8(str)
+    return str if /\A[\r\n\t\x20-\x7e]*\Z/n.match(str) # for us-ascii
+    encoding = Setting.commit_logs_encoding.to_s.strip
+    unless encoding.blank? || encoding == 'UTF-8'
+      begin
+        return Iconv.conv('UTF-8', encoding, str)
+      rescue Iconv::Failure
+        # do nothing here
+      end
+    end
+    str
+  end
 end

diff --git a/app/models/setting.rb b/app/models/setting.rb
index 185991d9bbf9b812185ee1205f318a9ef6bf5ac2..072afa0db12f099f7733cdc7046b67fef2f861ea 100644 (file)
--- a/app/models/setting.rb
+++ b/app/models/setting.rb
@@ -33,6 +33,45 @@ class Setting < ActiveRecord::Base
     '%H:%M',
     '%I:%M %p'
     ]
+    
+  ENCODINGS = %w(US-ASCII
+                  windows-1250
+                  windows-1251
+                  windows-1252
+                  windows-1253
+                  windows-1254
+                  windows-1255
+                  windows-1256
+                  windows-1257
+                  windows-1258
+                  windows-31j
+                  ISO-2022-JP
+                  ISO-2022-KR
+                  ISO-8859-1
+                  ISO-8859-2
+                  ISO-8859-3
+                  ISO-8859-4
+                  ISO-8859-5
+                  ISO-8859-6
+                  ISO-8859-7
+                  ISO-8859-8
+                  ISO-8859-9
+                  ISO-8859-13
+                  ISO-8859-15
+                  KOI8-R
+                  UTF-8
+                  UTF-16
+                  UTF-16BE
+                  UTF-16LE
+                  EUC-JP
+                  Shift_JIS
+                  GB18030
+                  GBK
+                  ISCII91
+                  EUC-KR
+                  Big5
+                  Big5-HKSCS
+                  TIS-620)
   
   cattr_accessor :available_settings
   @@available_settings = YAML::load(File.open("#{RAILS_ROOT}/config/settings.yml"))

diff --git a/app/views/settings/_repositories.rhtml b/app/views/settings/_repositories.rhtml
index 127801be2b8212f1060cb042ff64fcad2427e11c..a8c924430ac7fb43659d39e42698c17af6cb1e50 100644 (file)
--- a/app/views/settings/_repositories.rhtml
+++ b/app/views/settings/_repositories.rhtml
@@ -16,6 +16,9 @@
 
 <p><label><%= l(:setting_repositories_encodings) %></label>
 <%= text_field_tag 'settings[repositories_encodings]', Setting.repositories_encodings, :size => 60 %><br /><em><%= l(:text_comma_separated) %></em></p>
+
+<p><label><%= l(:setting_commit_logs_encoding) %></label>
+<%= select_tag 'settings[commit_logs_encoding]', options_for_select(Setting::ENCODINGS, Setting.commit_logs_encoding) %></p>
 </div>
 
 <fieldset class="box tabular settings"><legend><%= l(:text_issues_ref_in_commit_messages) %></legend>

diff --git a/config/settings.yml b/config/settings.yml
index 9a59d56869083deca2b55313b4bf655bbe8ac4fc..ac79edb8da9a3d75b34ed4ddc29e35fc9e8cd99a 100644 (file)
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -122,6 +122,9 @@ default_projects_public:
 # multiple values accepted, comma separated
 repositories_encodings:
   default: ''
+# encoding used to convert commit logs to UTF-8
+commit_logs_encoding:
+  default: 'UTF-8'
 ui_theme:
   default: ''
 emails_footer:

diff --git a/lang/bg.yml b/lang/bg.yml
index 6a9676119ef3c2fe3338b78f77a15052aaeeeec8..f15ba4da7afa9fcc0ad6e6d94b9c9fccec095fe8 100644 (file)
--- a/lang/bg.yml
+++ b/lang/bg.yml
@@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/cs.yml b/lang/cs.yml
index cc5847ee4e9638b414588984353dad81f6db93a2..b7b1b9eafe7751f5a9c01d92981bb6b5af75b9cb 100644 (file)
--- a/lang/cs.yml
+++ b/lang/cs.yml
@@ -639,3 +639,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/da.yml b/lang/da.yml
index 1cb55cc486cff655f26a1168818ead5cc13cbe18..3227dc7bd39df1ff77c4aff2097db29821d0e10f 100644 (file)
--- a/lang/da.yml
+++ b/lang/da.yml
@@ -636,3 +636,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/de.yml b/lang/de.yml
index f0be844906e4cc9bf361244be5bd79f9fb8fe5e1..5093c31fd915c9ce436e92e561efbe69f2842ba9 100644 (file)
--- a/lang/de.yml
+++ b/lang/de.yml
@@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/en.yml b/lang/en.yml
index 6c4a7bd8ad6d90b51916c27aee58b1053362fb46..6cd6d42706ca5a28f9b911dc13f3c143d07d738e 100644 (file)
--- a/lang/en.yml
+++ b/lang/en.yml
@@ -209,6 +209,7 @@ setting_time_format: Time format
 setting_cross_project_issue_relations: Allow cross-project issue relations
 setting_issue_list_default_columns: Default columns displayed on the issue list
 setting_repositories_encodings: Repositories encodings
+setting_commit_logs_encoding: Commit messages encoding
 setting_emails_footer: Emails footer
 setting_protocol: Protocol
 setting_per_page_options: Objects per page options

diff --git a/lang/es.yml b/lang/es.yml
index 7f27d53e578a7dd3caed569480cd46215e7578aa..58d27f28d4cee35d871e0bccf064abb8ade85acc 100644 (file)
--- a/lang/es.yml
+++ b/lang/es.yml
@@ -637,3 +637,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/fi.yml b/lang/fi.yml
index 031d59d63ce8a1936d370ac6aece1f496b3b0a31..394ff23f3c8995c0804e40392e36f25dee138b09 100644 (file)
--- a/lang/fi.yml
+++ b/lang/fi.yml
@@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/fr.yml b/lang/fr.yml
index 5b621ebaa053e488aae7eb5feecfb9f7e24ad09a..e6c4b33401537dbceb05f18a881d7afb90e5f521 100644 (file)
--- a/lang/fr.yml
+++ b/lang/fr.yml
@@ -210,6 +210,7 @@ setting_time_format: Format d'heure
 setting_cross_project_issue_relations: Autoriser les relations entre demandes de différents projets
 setting_issue_list_default_columns: Colonnes affichées par défaut sur la liste des demandes
 setting_repositories_encodings: Encodages des dépôts
+setting_commit_logs_encoding: Encodage des messages de commit
 setting_emails_footer: Pied-de-page des emails
 setting_protocol: Protocole
 setting_per_page_options: Options d'objets affichés par page

diff --git a/lang/he.yml b/lang/he.yml
index 9c8fbae7e8922814a6a1b00119bd3e6a254e5ba8..652b0ed8e50db14636de885b6ce75c2c1977a6bd 100644 (file)
--- a/lang/he.yml
+++ b/lang/he.yml
@@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/hu.yml b/lang/hu.yml
index a5d208cb66ceaca0e1894720c72be1760105f24e..15c4205399e3c92c1c9d087e1c8f9894abba3cd9 100644 (file)
--- a/lang/hu.yml
+++ b/lang/hu.yml
@@ -635,3 +635,4 @@ setting_mail_handler_api_key: API kulcs
 text_email_delivery_not_configured: "Az E-mail küldés nincs konfigurálva, és az értesítések ki vannak kapcsolva.\nÁllítsd be az SMTP szervert a config/email.yml fájlban és indítsd újra az alkalmazást, hogy érvénybe lépjen."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/it.yml b/lang/it.yml
index 5d62e3255558e2540600a346da6ddb32116b2378..3e1c1d53bb04ca5012461bdb0d1204060eaa308f 100644 (file)
--- a/lang/it.yml
+++ b/lang/it.yml
@@ -634,3 +634,4 @@ setting_mail_handler_api_key: chiave API
 text_email_delivery_not_configured: "La consegna via e-mail non è configurata e le notifiche sono disabilitate.\nConfigura il tuo server SMTP in config/email.yml e riavvia l'applicazione per abilitarle."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/ja.yml b/lang/ja.yml
index f3ade750dd91d0909c7dac78877627f6f92aa73b..e7805393af638797acf9530aab357e89d3cc5f35 100644 (file)
--- a/lang/ja.yml
+++ b/lang/ja.yml
@@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/ko.yml b/lang/ko.yml
index 961255e016685beb0d4491647429c886f09894fd..c85ce72e2be5673f9ca2a7870609bd2ba6edf634 100644 (file)
--- a/lang/ko.yml
+++ b/lang/ko.yml
@@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/lt.yml b/lang/lt.yml
index 58f79850606e3c88d7045d46d3e5c388b13d9996..188faf215651b2a1917f0a1c4995671cb03ad291 100644 (file)
--- a/lang/lt.yml
+++ b/lang/lt.yml
@@ -637,3 +637,4 @@ setting_mail_handler_api_key: API raktas
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/nl.yml b/lang/nl.yml
index 08e300143de0dc09e1c84b1329305561484c89a8..fd5bff1f633d2df0207e3ec397711d3143222d99 100644 (file)
--- a/lang/nl.yml
+++ b/lang/nl.yml
@@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/no.yml b/lang/no.yml
index 7af029db9fe6ba9e12a9e049cb817d626ec1b467..2cdc0bcd6b4395f61becb8812c771f47a4ba835a 100644 (file)
--- a/lang/no.yml
+++ b/lang/no.yml
@@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/pl.yml b/lang/pl.yml
index 0a29b7c473e7b5bf868b8bff61b3890673bb1117..69e04e4dda08ce11a18f39cbd345bc12588c8571 100644 (file)
--- a/lang/pl.yml
+++ b/lang/pl.yml
@@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/pt-br.yml b/lang/pt-br.yml
index 3ab24aeb04fa18290036260d269478640ee1c0c0..2b25e023f97b06f04e4401fc9e7bd1a56d9e9c5a 100644 (file)
--- a/lang/pt-br.yml
+++ b/lang/pt-br.yml
@@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."\r
 field_parent_title: Parent page\r
 label_issue_watchers: Watchers\r
+setting_commit_logs_encoding: Commit messages encoding\r

diff --git a/lang/pt.yml b/lang/pt.yml
index bd32cd1404741314914c6228509c23a0711a10bb..6505b762a2b2135af3b3e75dfa42ae7ad8ba8bde 100644 (file)
--- a/lang/pt.yml
+++ b/lang/pt.yml
@@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/ro.yml b/lang/ro.yml
index 2e562e2378ecb84610009a11df7ecf472b530bd3..4887bcf928e44cf8e9277307ee308ca09b09c2b3 100644 (file)
--- a/lang/ro.yml
+++ b/lang/ro.yml
@@ -634,3 +634,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/ru.yml b/lang/ru.yml
index 61e71369473d3de0ceac538525a20d434c15dfff..095e4db34014ca0e827917e4abb5e1915736170e 100644 (file)
--- a/lang/ru.yml
+++ b/lang/ru.yml
@@ -638,3 +638,4 @@ setting_mail_handler_api_key: API ключ
 text_email_delivery_not_configured: "Параметры работы с почтовым сервером не настроены и функция уведомления по email не активна.\nНастроить параметры для вашего SMTP сервера вы можете в файле config/email.yml. Для применения изменений перезапустите приложение."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/sr.yml b/lang/sr.yml
index ba79d4ef2bfc4de5354bb8626deb16e355da4140..f81d0e05497e1f5bd8a696bb9ef18fe81453e246 100644 (file)
--- a/lang/sr.yml
+++ b/lang/sr.yml
@@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/sv.yml b/lang/sv.yml
index 9505759c82e34d4003447cc8f212a09f4aae73f2..5e120b3f2e32aaeda2c8425c753c48bdf6ce8cb7 100644 (file)
--- a/lang/sv.yml
+++ b/lang/sv.yml
@@ -635,3 +635,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/th.yml b/lang/th.yml
index 6c4ce30d63082d0ab0f67305391528c3f85e10a6..3ae37e5515008ad50a63538c9849af65ee6843d2 100644 (file)
--- a/lang/th.yml
+++ b/lang/th.yml
@@ -637,3 +637,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/uk.yml b/lang/uk.yml
index 4fa27f7d307045a6d462c978ea2df26c90845d15..cf5f384b99cbbd1eaa241abbd424c9b9bdfe835f 100644 (file)
--- a/lang/uk.yml
+++ b/lang/uk.yml
@@ -636,3 +636,4 @@ setting_mail_handler_api_key: API key
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/zh-tw.yml b/lang/zh-tw.yml
index 1e6483631a99bd8435d22f876a7c68d672b9e213..f13331df4afd794682430f749e0569d58d13349f 100644 (file)
--- a/lang/zh-tw.yml
+++ b/lang/zh-tw.yml
@@ -635,3 +635,4 @@ enumeration_doc_categories: 文件分類
 enumeration_activities: 活動 (時間追蹤)
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding

diff --git a/lang/zh.yml b/lang/zh.yml
index 7158a7f83b364fab583500de76dddcc24eb19fff..682b59eb38f329616a871c6e134b55b1a30e66f8 100644 (file)
--- a/lang/zh.yml
+++ b/lang/zh.yml
@@ -635,3 +635,4 @@ enumeration_activities: 活动（时间跟踪）
 text_email_delivery_not_configured: "Email delivery is not configured, and notifications are disabled.\nConfigure your SMTP server in config/email.yml and restart the application to enable them."
 field_parent_title: Parent page
 label_issue_watchers: Watchers
+setting_commit_logs_encoding: Commit messages encoding