Fix commenting on non-utf8 encoded files (#11916) (#11950)

* Add comment on non-unicode line to force fail

Signed-off-by: Andrew Thornton <art27@cantab.net>
* Just quote/unquote patch

Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: zeripath <art27@cantab.net>

diff --git a/integrations/api_pull_review_test.go b/integrations/api_pull_review_test.go
index c90a5c11cd129ab101cfc513a80ee5ca89f05557..611b34712caed9a265e7a70b2c6359941065c5e0 100644 (file)
--- a/integrations/api_pull_review_test.go
+++ b/integrations/api_pull_review_test.go
@@ -86,6 +86,11 @@ func TestAPIPullReview(t *testing.T) {
                        Body:       "first old line",
                        OldLineNum: 1,
                        NewLineNum: 0,
+               }, {
+                       Path:       "iso-8859-1.txt",
+                       Body:       "this line contains a non-utf-8 character",
+                       OldLineNum: 0,
+                       NewLineNum: 1,
                },
                },
        })
@@ -93,7 +98,7 @@ func TestAPIPullReview(t *testing.T) {
        DecodeJSON(t, resp, &review)
        assert.EqualValues(t, 6, review.ID)
        assert.EqualValues(t, "PENDING", review.State)
-       assert.EqualValues(t, 2, review.CodeCommentsCount)
+       assert.EqualValues(t, 3, review.CodeCommentsCount)
 
        // test SubmitPullReview
        req = NewRequestWithJSON(t, http.MethodPost, fmt.Sprintf("/api/v1/repos/%s/%s/pulls/%d/reviews/%d?token=%s", repo.OwnerName, repo.Name, pullIssue.Index, review.ID, token), &api.SubmitPullReviewOptions{
@@ -104,7 +109,7 @@ func TestAPIPullReview(t *testing.T) {
        DecodeJSON(t, resp, &review)
        assert.EqualValues(t, 6, review.ID)
        assert.EqualValues(t, "APPROVED", review.State)
-       assert.EqualValues(t, 2, review.CodeCommentsCount)
+       assert.EqualValues(t, 3, review.CodeCommentsCount)
 
        // test DeletePullReview
        req = NewRequestWithJSON(t, http.MethodPost, fmt.Sprintf("/api/v1/repos/%s/%s/pulls/%d/reviews?token=%s", repo.OwnerName, repo.Name, pullIssue.Index, token), &api.CreatePullReviewOptions{

diff --git a/integrations/gitea-repositories-meta/user2/repo1.git/objects/5f/22f7d0d95d614d25a5b68592adb345a4b5c7fd b/integrations/gitea-repositories-meta/user2/repo1.git/objects/5f/22f7d0d95d614d25a5b68592adb345a4b5c7fd
new file mode 100644 (file)
index 0000000..17fdf18
Binary files /dev/null and b/integrations/gitea-repositories-meta/user2/repo1.git/objects/5f/22f7d0d95d614d25a5b68592adb345a4b5c7fd differ

diff --git a/integrations/gitea-repositories-meta/user2/repo1.git/objects/90/dcd07da077d1e7cd6032b52d1f79ae2b5f19b2 b/integrations/gitea-repositories-meta/user2/repo1.git/objects/90/dcd07da077d1e7cd6032b52d1f79ae2b5f19b2
new file mode 100644 (file)
index 0000000..1404abd
--- /dev/null
+++ b/integrations/gitea-repositories-meta/user2/repo1.git/objects/90/dcd07da077d1e7cd6032b52d1f79ae2b5f19b2
@@ -0,0 +1,2 @@
+x\ 1e\90±NÄ0\10D©#å\1fæ\ 3\8e4 
+JÄA\ 1\ 5Å5\94\8e³\97,\97xÑzsVþ\9d\825\b\84DåÑØ»\9e7ý,=®®o.\ e\13áEå\8d¢áq5J=é\88ý\9dÈ        rÄ=\13\1e>4\e§\1dú\rO!óÂ\8aý\90ã´ðÐ6ms\988\83¾&\\1cEa¾tÍT\17\84´I¢z\94\89Ô\85!      ¢dso@a\9bÚ&ÌK5üB)\9br4\96\94Q¦`YèLÚ\ 1¯²b \9b\8f¾o`Ûaä3¹@(î\8a¶\8dÒeýÔ­5\rô\9dÂH\97\sÔHÿ9\9f9Rª3)Îë@\8eS\ 3ùã_"§\914sE0\94Rºñ§¤.\91U|/\80m¦Û¿]U÷\13Ì\15zÀ
\ No newline at end of file

diff --git a/integrations/gitea-repositories-meta/user2/repo1.git/objects/dd/59742c0f6672911f2b64cba5711ac00593ed32 b/integrations/gitea-repositories-meta/user2/repo1.git/objects/dd/59742c0f6672911f2b64cba5711ac00593ed32
new file mode 100644 (file)
index 0000000..f9137c5
Binary files /dev/null and b/integrations/gitea-repositories-meta/user2/repo1.git/objects/dd/59742c0f6672911f2b64cba5711ac00593ed32 differ

diff --git a/integrations/gitea-repositories-meta/user2/repo1.git/refs/pull/3/head b/integrations/gitea-repositories-meta/user2/repo1.git/refs/pull/3/head
index 98593d653703970b1c377933c19af00a0121fadc..33a9eaa7f162fbc125fdc92131117bfc3211c138 100644 (file)
--- a/integrations/gitea-repositories-meta/user2/repo1.git/refs/pull/3/head
+++ b/integrations/gitea-repositories-meta/user2/repo1.git/refs/pull/3/head
@@ -1 +1 @@
-4a357436d925b5c974181ff12a994538ddc5a269
+5f22f7d0d95d614d25a5b68592adb345a4b5c7fd

diff --git a/models/issue_comment.go b/models/issue_comment.go
index bb3c4be7ba722a686c0538f8cd9614186f155755..28f45b59f6e4231ba1c4136af4769f316824fc6b 100644 (file)
--- a/models/issue_comment.go
+++ b/models/issue_comment.go
@@ -9,7 +9,9 @@ package models
 import (
        "fmt"
        "regexp"
+       "strconv"
        "strings"
+       "unicode/utf8"
 
        "code.gitea.io/gitea/modules/git"
        "code.gitea.io/gitea/modules/log"
@@ -139,7 +141,8 @@ type Comment struct {
        RenderedContent string `xorm:"-"`
 
        // Path represents the 4 lines of code cemented by this comment
-       Patch string `xorm:"TEXT"`
+       Patch       string `xorm:"-"`
+       PatchQuoted string `xorm:"TEXT patch"`
 
        CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
        UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
@@ -183,6 +186,33 @@ func (c *Comment) loadIssue(e Engine) (err error) {
        return
 }
 
+// BeforeInsert will be invoked by XORM before inserting a record
+func (c *Comment) BeforeInsert() {
+       c.PatchQuoted = c.Patch
+       if !utf8.ValidString(c.Patch) {
+               c.PatchQuoted = strconv.Quote(c.Patch)
+       }
+}
+
+// BeforeUpdate will be invoked by XORM before updating a record
+func (c *Comment) BeforeUpdate() {
+       c.PatchQuoted = c.Patch
+       if !utf8.ValidString(c.Patch) {
+               c.PatchQuoted = strconv.Quote(c.Patch)
+       }
+}
+
+// AfterLoad is invoked from XORM after setting the values of all fields of this object.
+func (c *Comment) AfterLoad(session *xorm.Session) {
+       c.Patch = c.PatchQuoted
+       if len(c.PatchQuoted) > 0 && c.PatchQuoted[0] == '"' {
+               unquoted, err := strconv.Unquote(c.PatchQuoted)
+               if err == nil {
+                       c.Patch = unquoted
+               }
+       }
+}
+
 func (c *Comment) loadPoster(e Engine) (err error) {
        if c.PosterID <= 0 || c.Poster != nil {
                return nil