diff options
| author | guillep2k <18600385+guillep2k@users.noreply.github.com> | 2019-08-15 09:07:28 -0300 |
|---|---|---|
| committer | Lunny Xiao <xiaolunwen@gmail.com> | 2019-08-15 20:07:28 +0800 |
| commit | 5a44be627c055d3e9eb406ec4a91579de78b6910 (patch) | |
| tree | 1a9d4ce34f16bdc02e0ac79a63a81d955ead8e6e /models/repo_indexer.go | |
| parent | c2c35d169c819439b78c8c2c7f8877e7bf053cd1 (diff) | |
| download | gitea-5a44be627c055d3e9eb406ec4a91579de78b6910.tar.gz gitea-5a44be627c055d3e9eb406ec4a91579de78b6910.zip | |
Convert files to utf-8 for indexing (#7814)
* Convert files to utf-8 for indexing
* Move utf8 functions to modules/base
* Bump repoIndexerLatestVersion to 3
* Add tests for base/encoding.go
* Changes to pass gosimple
* Move UTF8 funcs into new modules/charset package
Diffstat (limited to 'models/repo_indexer.go')
| -rw-r--r-- | models/repo_indexer.go | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/models/repo_indexer.go b/models/repo_indexer.go index c991b1aac9..f625b80389 100644 --- a/models/repo_indexer.go +++ b/models/repo_indexer.go @@ -10,6 +10,7 @@ import ( "strings" "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/log" @@ -207,6 +208,7 @@ func addUpdate(update fileUpdate, repo *Repository, batch rupture.FlushingBatch) if err != nil { return err } else if !base.IsTextFile(fileContents) { + // FIXME: UTF-16 files will probably fail here return nil } indexerUpdate := indexer.RepoIndexerUpdate{ @@ -214,7 +216,7 @@ func addUpdate(update fileUpdate, repo *Repository, batch rupture.FlushingBatch) Op: indexer.RepoIndexerOpUpdate, Data: &indexer.RepoIndexerData{ RepoID: repo.ID, - Content: string(fileContents), + Content: string(charset.ToUTF8DropErrors(fileContents)), }, } return indexerUpdate.AddToFlushingBatch(batch) |