Merged r11544, r11545, r11546, r11547, r11549 from trunk to 2.3-stable (#12641)

fix that diff outputs become ??? in some non ASCII words.

Contributed by Filou Centrinov.

git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/branches/2.3-stable@11551 e93f8b46-1217-0410-a6f0-8f06a7374b81

5 files changed, 87 insertions, 16 deletions

diff --git a/app/views/common/_diff.html.erb b/app/views/common/_diff.html.erb
index 8600164c0..0d30bedd2 100644
--- a/app/views/common/_diff.html.erb
+++ b/app/views/common/_diff.html.erb
@@ -10,7 +10,7 @@
 <thead>
 <tr>
   <th colspan="4" class="filename">
-    <%= h(Redmine::CodesetUtil.to_utf8_by_setting(table_file.file_name)) %>
+    <%= table_file.file_name %>
   </th>
 </tr>
 </thead>
@@ -24,11 +24,11 @@
 <tr>
   <th class="line-num"><%= line.nb_line_left %></th>
   <td class="line-code <%= line.type_diff_left %>">
-    <pre><%= Redmine::CodesetUtil.to_utf8_by_setting(line.html_line_left).html_safe %></pre>
+    <pre><%= line.html_line_left.html_safe %></pre>
   </td>
   <th class="line-num"><%= line.nb_line_right %></th>
   <td class="line-code <%= line.type_diff_right %>">
-    <pre><%= Redmine::CodesetUtil.to_utf8_by_setting(line.html_line_right).html_safe %></pre>
+    <pre><%= line.html_line_right.html_safe %></pre>
   </td>
 </tr>
 <% end -%>
@@ -40,7 +40,7 @@
 <thead>
   <tr>
     <th colspan="3" class="filename">
-      <%= h(Redmine::CodesetUtil.to_utf8_by_setting(table_file.file_name)) %>
+      <%= table_file.file_name %>
     </th>
   </tr>
 </thead>
@@ -55,7 +55,7 @@
   <th class="line-num"><%= line.nb_line_left %></th>
   <th class="line-num"><%= line.nb_line_right %></th>
   <td class="line-code <%= line.type_diff %>">
-    <pre><%= Redmine::CodesetUtil.to_utf8_by_setting(line.html_line).html_safe %></pre>
+    <pre><%= line.html_line.html_safe %></pre>
   </td>
 </tr>
 <% end -%>
diff --git a/lib/redmine/unified_diff.rb b/lib/redmine/unified_diff.rb
index 9917a554f..6fbfb80fe 100644
--- a/lib/redmine/unified_diff.rb
+++ b/lib/redmine/unified_diff.rb
@@ -28,17 +28,9 @@ module Redmine
       lines = 0
       @truncated = false
       diff_table = DiffTable.new(diff_type, diff_style)
-      diff.each do |line|
-        line_encoding = nil
-        if line.respond_to?(:force_encoding)
-          line_encoding = line.encoding
-          # TODO: UTF-16 and Japanese CP932 which is imcompatible with ASCII
-          #       In Japan, diffrence between file path encoding
-          #       and file contents encoding is popular.
-          line.force_encoding('ASCII-8BIT')
-        end
-        unless diff_table.add_line line
-          line.force_encoding(line_encoding) if line_encoding
+      diff.each do |line_raw|
+        line = Redmine::CodesetUtil.to_utf8_by_setting(line_raw)
+        unless diff_table.add_line(line)
           self << diff_table if diff_table.length > 0
           diff_table = DiffTable.new(diff_type, diff_style)
         end
@@ -207,10 +199,16 @@ module Redmine
         while starting < max && line_left[starting] == line_right[starting]
           starting += 1
         end
+        while line_left[starting].ord.between?(128, 191) && starting > 0
+          starting -= 1
+        end
         ending = -1
         while ending >= -(max - starting) && line_left[ending] == line_right[ending]
           ending -= 1
         end
+        while line_left[ending].ord.between?(128, 191) && ending > -1
+          ending -= 1
+        end
         unless starting == 0 && ending == -1
           [starting, ending]
         end
@@ -268,6 +266,12 @@ module Redmine
     private
 
     def line_to_html(line, offsets)
+      html = line_to_html_raw(line, offsets)
+      html.force_encoding('UTF-8') if html.respond_to?(:force_encoding)
+      html
+    end
+
+    def line_to_html_raw(line, offsets)
       if offsets
         s = ''
         unless offsets.first == 0
diff --git a/test/fixtures/diffs/issue-12641-ja.diff b/test/fixtures/diffs/issue-12641-ja.diff
new file mode 100644
index 000000000..9f37cd724
--- /dev/null
+++ b/test/fixtures/diffs/issue-12641-ja.diff
@@ -0,0 +1,25 @@
+# HG changeset patch
+# User tmaruyama
+# Date 1362559296 0
+# Node ID ee54942e0289c30bea1b1973750b698b1ee7c466
+# Parent  738777832f379f6f099c25251593fc57bc17f586
+fix some Japanese "issue" translations (#13350)
+
+Contributed by Go MAEDA.
+
+diff --git a/config/locales/ja.yml b/config/locales/ja.yml
+--- a/config/locales/ja.yml
++++ b/config/locales/ja.yml
+@@ -904,9 +904,9 @@ ja:
+   text_journal_set_to: "%{label} を %{value} にセット"
+   text_journal_deleted: "%{label} を削除 (%{old})"
+   text_journal_added: "%{label} %{value} を追加"
+-  text_tip_issue_begin_day: この日に開始するタスク
+-  text_tip_issue_end_day: この日に終了するタスク
+-  text_tip_issue_begin_end_day: この日のうちに開始して終了するタスク
++  text_tip_issue_begin_day: この日に開始するチケット
++  text_tip_issue_end_day: この日に終了するチケット
++  text_tip_issue_begin_end_day: この日に開始・終了するチケット
+   text_caracters_maximum: "最大%{count}文字です。"
+   text_caracters_minimum: "最低%{count}文字の長さが必要です"
+   text_length_between: "長さは%{min}から%{max}文字までです。"
diff --git a/test/fixtures/diffs/issue-12641-ru.diff b/test/fixtures/diffs/issue-12641-ru.diff
new file mode 100644
index 000000000..3b549e1fa
--- /dev/null
+++ b/test/fixtures/diffs/issue-12641-ru.diff
@@ -0,0 +1,19 @@
+# HG changeset patch
+# User tmaruyama
+# Date 1355872765 0
+# Node ID 8a13ebed1779c2e85fa644ecdd0de81996c969c4
+# Parent  5c3c5f917ae92f278fe42c6978366996595b0796
+Russian "about_x_hours" translation changed by Mikhail Velkin (#12640)
+
+diff --git a/config/locales/ru.yml b/config/locales/ru.yml
+--- a/config/locales/ru.yml
++++ b/config/locales/ru.yml
+@@ -115,7 +115,7 @@ ru:
+         one:   "около %{count} часа"
+         few:   "около %{count} часов"
+         many:  "около %{count} часов"
+-        other: "около %{count} часа"
++        other: "около %{count} часов"
+       x_hours:
+         one:   "1 час"
+         other: "%{count} часов"
diff --git a/test/unit/lib/redmine/unified_diff_test.rb b/test/unit/lib/redmine/unified_diff_test.rb
index e4707af9c..cbc5cec45 100644
--- a/test/unit/lib/redmine/unified_diff_test.rb
+++ b/test/unit/lib/redmine/unified_diff_test.rb
@@ -221,6 +221,29 @@ DIFF
     assert_equal "test02.txt", diff[0].file_name
   end
 
+  def test_utf8_ja
+    ja = "  text_tip_issue_end_day: "
+    ja += "\xe3\x81\x93\xe3\x81\xae\xe6\x97\xa5\xe3\x81\xab\xe7\xb5\x82\xe4\xba\x86\xe3\x81\x99\xe3\x82\x8b<span>\xe3\x82\xbf\xe3\x82\xb9\xe3\x82\xaf</span>"
+    ja.force_encoding('UTF-8') if ja.respond_to?(:force_encoding)
+    with_settings :repositories_encodings => '' do
+      diff = Redmine::UnifiedDiff.new(read_diff_fixture('issue-12641-ja.diff'), :type => 'inline')
+      assert_equal 1, diff.size
+      assert_equal 12, diff.first.size
+      assert_equal ja, diff.first[4].html_line_left
+    end
+  end
+
+  def test_utf8_ru
+    ru = "        other: &quot;\xd0\xbe\xd0\xba\xd0\xbe\xd0\xbb\xd0\xbe %{count} \xd1\x87\xd0\xb0\xd1\x81<span>\xd0\xb0</span>&quot;"
+    ru.force_encoding('UTF-8') if ru.respond_to?(:force_encoding)
+    with_settings :repositories_encodings => '' do
+      diff = Redmine::UnifiedDiff.new(read_diff_fixture('issue-12641-ru.diff'), :type => 'inline')
+      assert_equal 1, diff.size
+      assert_equal 8, diff.first.size
+      assert_equal ru, diff.first[3].html_line_left
+    end
+  end
+
   private
 
   def read_diff_fixture(filename)