Implements request #14320 The rendering of CSV files does match the diff style.
* Moved CSV logic into base package.
* Added method to create a tabular diff.
* Added CSV compare context.
* Added CSV diff template.
* Use new table style in CSV markup.
* Added file size limit for CSV rendering.
* Display CSV parser errors in diff.
* Lazy read single file.
* Lazy read rows for full diff.
* Added unit tests for various CSV changes.
; Whether to render SVG files as images. If SVG rendering is disabled, SVG files are displayed as text and cannot be embedded in markdown files as images.
ENABLE_RENDER = true
+[ui.csv]
+; Maximum allowed file size in bytes to render CSV files as table. (Set to 0 for no limit).
+MAX_FILE_SIZE = 524288
+
[markdown]
; Render soft line breaks as hard line breaks, which means a single newline character between
; paragraphs will cause a line break and adding trailing whitespace to paragraphs is not
- `ENABLE_RENDER`: **true**: Whether to render SVG files as images. If SVG rendering is disabled, SVG files are displayed as text and cannot be embedded in markdown files as images.
+### UI - CSV Files (`ui.csv`)
+
+- `MAX_FILE_SIZE`: **524288** (512kb): Maximum allowed file size in bytes to render CSV files as table. (Set to 0 for no limit).
+
## Markdown (`markdown`)
- `ENABLE_HARD_LINE_BREAK_IN_COMMENTS`: **true**: Render soft line breaks as hard line breaks in comments, which
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package csv
+
+import (
+ "bytes"
+ "encoding/csv"
+ "errors"
+ "regexp"
+ "strings"
+
+ "code.gitea.io/gitea/modules/translation"
+ "code.gitea.io/gitea/modules/util"
+)
+
+var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
+
+// CreateReader creates a csv.Reader with the given delimiter.
+func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader {
+ rd := csv.NewReader(bytes.NewReader(rawBytes))
+ rd.Comma = delimiter
+ rd.TrimLeadingSpace = true
+ return rd
+}
+
+// CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
+func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader {
+ delimiter := guessDelimiter(rawBytes)
+ return CreateReader(rawBytes, delimiter)
+}
+
+// guessDelimiter scores the input CSV data against delimiters, and returns the best match.
+// Reads at most 10k bytes & 10 lines.
+func guessDelimiter(data []byte) rune {
+ maxLines := 10
+ maxBytes := util.Min(len(data), 1e4)
+ text := string(data[:maxBytes])
+ text = quoteRegexp.ReplaceAllLiteralString(text, "")
+ lines := strings.SplitN(text, "\n", maxLines+1)
+ lines = lines[:util.Min(maxLines, len(lines))]
+
+ delimiters := []rune{',', ';', '\t', '|', '@'}
+ bestDelim := delimiters[0]
+ bestScore := 0.0
+ for _, delim := range delimiters {
+ score := scoreDelimiter(lines, delim)
+ if score > bestScore {
+ bestScore = score
+ bestDelim = delim
+ }
+ }
+
+ return bestDelim
+}
+
+// scoreDelimiter uses a count & regularity metric to evaluate a delimiter against lines of CSV.
+func scoreDelimiter(lines []string, delim rune) float64 {
+ countTotal := 0
+ countLineMax := 0
+ linesNotEqual := 0
+
+ for _, line := range lines {
+ if len(line) == 0 {
+ continue
+ }
+
+ countLine := strings.Count(line, string(delim))
+ countTotal += countLine
+ if countLine != countLineMax {
+ if countLineMax != 0 {
+ linesNotEqual++
+ }
+ countLineMax = util.Max(countLine, countLineMax)
+ }
+ }
+
+ return float64(countTotal) * (1 - float64(linesNotEqual)/float64(len(lines)))
+}
+
+// FormatError converts csv errors into readable messages.
+func FormatError(err error, locale translation.Locale) (string, error) {
+ var perr *csv.ParseError
+ if errors.As(err, &perr) {
+ if perr.Err == csv.ErrFieldCount {
+ return locale.Tr("repo.error.csv.invalid_field_count", perr.Line), nil
+ }
+ return locale.Tr("repo.error.csv.unexpected", perr.Line, perr.Column), nil
+ }
+
+ return "", err
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package csv
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestCreateReader(t *testing.T) {
+ rd := CreateReader([]byte{}, ',')
+ assert.Equal(t, ',', rd.Comma)
+}
+
+func TestCreateReaderAndGuessDelimiter(t *testing.T) {
+ input := "a;b;c\n1;2;3\n4;5;6"
+
+ rd := CreateReaderAndGuessDelimiter([]byte(input))
+ assert.Equal(t, ';', rd.Comma)
+}
+
+func TestGuessDelimiter(t *testing.T) {
+ var kases = map[string]rune{
+ "a": ',',
+ "1,2": ',',
+ "1;2": ';',
+ "1\t2": '\t',
+ "1|2": '|',
+ "1,2,3;4,5,6;7,8,9\na;b;c": ';',
+ "\"1,2,3,4\";\"a\nb\"\nc;d": ';',
+ "<br/>": ',',
+ }
+
+ for k, v := range kases {
+ assert.EqualValues(t, guessDelimiter([]byte(k)), v)
+ }
+}
import (
"bytes"
- "encoding/csv"
"html"
"io"
- "regexp"
- "strings"
+ "strconv"
+ "code.gitea.io/gitea/modules/csv"
"code.gitea.io/gitea/modules/markup"
- "code.gitea.io/gitea/modules/util"
+ "code.gitea.io/gitea/modules/setting"
)
-var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
-
func init() {
markup.RegisterParser(Parser{})
-
}
-// Parser implements markup.Parser for orgmode
+// Parser implements markup.Parser for csv files
type Parser struct {
}
}
// Render implements markup.Parser
-func (p Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
- rd := csv.NewReader(bytes.NewReader(rawBytes))
- rd.Comma = p.bestDelimiter(rawBytes)
+func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
var tmpBlock bytes.Buffer
- tmpBlock.WriteString(`<table class="table">`)
+
+ if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < int64(len(rawBytes)) {
+ tmpBlock.WriteString("<pre>")
+ tmpBlock.WriteString(html.EscapeString(string(rawBytes)))
+ tmpBlock.WriteString("</pre>")
+ return tmpBlock.Bytes()
+ }
+
+ rd := csv.CreateReaderAndGuessDelimiter(rawBytes)
+
+ writeField := func(element, class, field string) {
+ tmpBlock.WriteString("<")
+ tmpBlock.WriteString(element)
+ if len(class) > 0 {
+ tmpBlock.WriteString(" class=\"")
+ tmpBlock.WriteString(class)
+ tmpBlock.WriteString("\"")
+ }
+ tmpBlock.WriteString(">")
+ tmpBlock.WriteString(html.EscapeString(field))
+ tmpBlock.WriteString("</")
+ tmpBlock.WriteString(element)
+ tmpBlock.WriteString(">")
+ }
+
+ tmpBlock.WriteString(`<table class="data-table">`)
+ row := 1
for {
fields, err := rd.Read()
if err == io.EOF {
continue
}
tmpBlock.WriteString("<tr>")
+ element := "td"
+ if row == 1 {
+ element = "th"
+ }
+ writeField(element, "line-num", strconv.Itoa(row))
for _, field := range fields {
- tmpBlock.WriteString("<td>")
- tmpBlock.WriteString(html.EscapeString(field))
- tmpBlock.WriteString("</td>")
+ writeField(element, "", field)
}
tmpBlock.WriteString("</tr>")
+
+ row++
}
tmpBlock.WriteString("</table>")
return tmpBlock.Bytes()
}
-
-// bestDelimiter scores the input CSV data against delimiters, and returns the best match.
-// Reads at most 10k bytes & 10 lines.
-func (p Parser) bestDelimiter(data []byte) rune {
- maxLines := 10
- maxBytes := util.Min(len(data), 1e4)
- text := string(data[:maxBytes])
- text = quoteRegexp.ReplaceAllLiteralString(text, "")
- lines := strings.SplitN(text, "\n", maxLines+1)
- lines = lines[:util.Min(maxLines, len(lines))]
-
- delimiters := []rune{',', ';', '\t', '|'}
- bestDelim := delimiters[0]
- bestScore := 0.0
- for _, delim := range delimiters {
- score := p.scoreDelimiter(lines, delim)
- if score > bestScore {
- bestScore = score
- bestDelim = delim
- }
- }
-
- return bestDelim
-}
-
-// scoreDelimiter uses a count & regularity metric to evaluate a delimiter against lines of CSV
-func (Parser) scoreDelimiter(lines []string, delim rune) (score float64) {
- countTotal := 0
- countLineMax := 0
- linesNotEqual := 0
-
- for _, line := range lines {
- if len(line) == 0 {
- continue
- }
-
- countLine := strings.Count(line, string(delim))
- countTotal += countLine
- if countLine != countLineMax {
- if countLineMax != 0 {
- linesNotEqual++
- }
- countLineMax = util.Max(countLine, countLineMax)
- }
- }
-
- return float64(countTotal) * (1 - float64(linesNotEqual)/float64(len(lines)))
-}
func TestRenderCSV(t *testing.T) {
var parser Parser
var kases = map[string]string{
- "a": "<table class=\"table\"><tr><td>a</td></tr></table>",
- "1,2": "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
- "1;2": "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
- "1\t2": "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
- "1|2": "<table class=\"table\"><tr><td>1</td><td>2</td></tr></table>",
- "1,2,3;4,5,6;7,8,9\na;b;c": "<table class=\"table\"><tr><td>1,2,3</td><td>4,5,6</td><td>7,8,9</td></tr><tr><td>a</td><td>b</td><td>c</td></tr></table>",
- "\"1,2,3,4\";\"a\nb\"\nc;d": "<table class=\"table\"><tr><td>1,2,3,4</td><td>a\nb</td></tr><tr><td>c</td><td>d</td></tr></table>",
- "<br/>": "<table class=\"table\"><tr><td><br/></td></tr></table>",
+ "a": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>a</th></tr></table>",
+ "1,2": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>1</th><th>2</th></tr></table>",
+ "1;2\n3;4": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th>1</th><th>2</th></tr><tr><td class=\"line-num\">2</td><td>3</td><td>4</td></tr></table>",
+ "<br/>": "<table class=\"data-table\"><tr><th class=\"line-num\">1</th><th><br/></th></tr></table>",
}
for k, v := range kases {
// Allow icons, emojis, and chroma syntax on span
sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`^((icon(\s+[\p{L}\p{N}_-]+)+)|(emoji))$|^([a-z][a-z0-9]{0,2})$`)).OnElements("span")
+ // Allow data tables
+ sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`data-table`)).OnElements("table")
+ sanitizer.policy.AllowAttrs("class").Matching(regexp.MustCompile(`line-num`)).OnElements("th", "td")
+
// Allow generally safe attributes
generalSafeAttrs := []string{"abbr", "accept", "accept-charset",
"accesskey", "action", "align", "alt",
Enabled bool `ini:"ENABLE_RENDER"`
} `ini:"ui.svg"`
+ CSV struct {
+ MaxFileSize int64
+ } `ini:"ui.csv"`
+
Admin struct {
UserPagingNum int
RepoPagingNum int
}{
Enabled: true,
},
+ CSV: struct {
+ MaxFileSize int64
+ }{
+ MaxFileSize: 524288,
+ },
Admin: struct {
UserPagingNum int
RepoPagingNum int
diff.stats_desc = <strong> %d changed files</strong> with <strong>%d additions</strong> and <strong>%d deletions</strong>
diff.stats_desc_file = %d changes: %d additions and %d deletions
diff.bin = BIN
+diff.bin_not_shown = Binary file not shown.
diff.view_file = View File
diff.file_before = Before
diff.file_after = After
topic.count_prompt = You can not select more than 25 topics
topic.format_prompt = Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
+error.csv.too_large = Can't render this file because it is too large.
+error.csv.unexpected = Can't render this file because it contains an unexpected character in line %d and column %d.
+error.csv.invalid_field_count = Can't render this file because it has a wrong number of fields in line %d.
+
[org]
org_name_holder = Organization Name
org_full_name_holder = Organization Full Name
return
}
}
- setImageCompareContext(ctx, parentCommit, commit)
headTarget := path.Join(userName, repoName)
- setPathsCompareContext(ctx, parentCommit, commit, headTarget)
+ setCompareContext(ctx, parentCommit, commit, headTarget)
ctx.Data["Title"] = commit.Summary() + " · " + base.ShortSha(commitID)
ctx.Data["Commit"] = commit
verification := models.ParseCommitWithSignature(commit)
import (
"bufio"
+ "encoding/csv"
+ "errors"
"fmt"
"html"
+ "io/ioutil"
"path"
+ "path/filepath"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/context"
+ csv_module "code.gitea.io/gitea/modules/csv"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
tplBlobExcerpt base.TplName = "repo/diff/blob_excerpt"
)
+// setCompareContext sets context data.
+func setCompareContext(ctx *context.Context, base *git.Commit, head *git.Commit, headTarget string) {
+ ctx.Data["BaseCommit"] = base
+ ctx.Data["HeadCommit"] = head
+
+ setPathsCompareContext(ctx, base, head, headTarget)
+ setImageCompareContext(ctx, base, head)
+ setCsvCompareContext(ctx)
+}
+
// setPathsCompareContext sets context data for source and raw paths
func setPathsCompareContext(ctx *context.Context, base *git.Commit, head *git.Commit, headTarget string) {
sourcePath := setting.AppSubURL + "/%s/src/commit/%s"
}
}
+// setCsvCompareContext sets context data that is required by the CSV compare template
+func setCsvCompareContext(ctx *context.Context) {
+ ctx.Data["IsCsvFile"] = func(diffFile *gitdiff.DiffFile) bool {
+ extension := strings.ToLower(filepath.Ext(diffFile.Name))
+ return extension == ".csv" || extension == ".tsv"
+ }
+
+ type CsvDiffResult struct {
+ Sections []*gitdiff.TableDiffSection
+ Error string
+ }
+
+ ctx.Data["CreateCsvDiff"] = func(diffFile *gitdiff.DiffFile, baseCommit *git.Commit, headCommit *git.Commit) CsvDiffResult {
+ if diffFile == nil || baseCommit == nil || headCommit == nil {
+ return CsvDiffResult{nil, ""}
+ }
+
+ errTooLarge := errors.New(ctx.Locale.Tr("repo.error.csv.too_large"))
+
+ csvReaderFromCommit := func(c *git.Commit) (*csv.Reader, error) {
+ blob, err := c.GetBlobByPath(diffFile.Name)
+ if err != nil {
+ return nil, err
+ }
+
+ if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < blob.Size() {
+ return nil, errTooLarge
+ }
+
+ reader, err := blob.DataAsync()
+ if err != nil {
+ return nil, err
+ }
+ defer reader.Close()
+
+ b, err := ioutil.ReadAll(reader)
+ if err != nil {
+ return nil, err
+ }
+
+ b = charset.ToUTF8WithFallback(b)
+
+ return csv_module.CreateReaderAndGuessDelimiter(b), nil
+ }
+
+ baseReader, err := csvReaderFromCommit(baseCommit)
+ if err == errTooLarge {
+ return CsvDiffResult{nil, err.Error()}
+ }
+ headReader, err := csvReaderFromCommit(headCommit)
+ if err == errTooLarge {
+ return CsvDiffResult{nil, err.Error()}
+ }
+
+ sections, err := gitdiff.CreateCsvDiff(diffFile, baseReader, headReader)
+ if err != nil {
+ errMessage, err := csv_module.FormatError(err, ctx.Locale)
+ if err != nil {
+ log.Error("RenderCsvDiff failed: %v", err)
+ return CsvDiffResult{nil, ""}
+ }
+ return CsvDiffResult{nil, errMessage}
+ }
+ return CsvDiffResult{sections, ""}
+ }
+}
+
// ParseCompareInfo parse compare info between two commit for preparing comparing references
func ParseCompareInfo(ctx *context.Context) (*models.User, *models.Repository, *git.Repository, *git.CompareInfo, string, string) {
baseRepo := ctx.Repo.Repository
ctx.Data["Username"] = headUser.Name
ctx.Data["Reponame"] = headRepo.Name
- setImageCompareContext(ctx, baseCommit, headCommit)
headTarget := path.Join(headUser.Name, repo.Name)
- setPathsCompareContext(ctx, baseCommit, headCommit, headTarget)
+ setCompareContext(ctx, baseCommit, headCommit, headTarget)
return false
}
gitRepo *git.Repository
)
- var headTarget string
var prInfo *git.CompareInfo
if pull.HasMerged {
prInfo = PrepareMergedViewPullInfo(ctx, issue)
startCommitID = prInfo.MergeBase
endCommitID = headCommitID
- headTarget = path.Join(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name)
ctx.Data["Username"] = ctx.Repo.Owner.Name
ctx.Data["Reponame"] = ctx.Repo.Repository.Name
ctx.Data["AfterCommitID"] = endCommitID
}
}
- setImageCompareContext(ctx, baseCommit, commit)
- setPathsCompareContext(ctx, baseCommit, commit, headTarget)
+ headTarget := path.Join(ctx.Repo.Owner.Name, ctx.Repo.Repository.Name)
+ setCompareContext(ctx, baseCommit, commit, headTarget)
ctx.Data["RequireHighlightJS"] = true
ctx.Data["RequireSimpleMDE"] = true
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package gitdiff
+
+import (
+ "encoding/csv"
+ "errors"
+ "io"
+
+ "code.gitea.io/gitea/modules/util"
+)
+
+const unmappedColumn = -1
+const maxRowsToInspect int = 10
+const minRatioToMatch float32 = 0.8
+
+// TableDiffCellType represents the type of a TableDiffCell.
+type TableDiffCellType uint8
+
+// TableDiffCellType possible values.
+const (
+ TableDiffCellEqual TableDiffCellType = iota + 1
+ TableDiffCellChanged
+ TableDiffCellAdd
+ TableDiffCellDel
+)
+
+// TableDiffCell represents a cell of a TableDiffRow
+type TableDiffCell struct {
+ LeftCell string
+ RightCell string
+ Type TableDiffCellType
+}
+
+// TableDiffRow represents a row of a TableDiffSection.
+type TableDiffRow struct {
+ RowIdx int
+ Cells []*TableDiffCell
+}
+
+// TableDiffSection represents a section of a DiffFile.
+type TableDiffSection struct {
+ Rows []*TableDiffRow
+}
+
+// csvReader wraps a csv.Reader which buffers the first rows.
+type csvReader struct {
+ reader *csv.Reader
+ buffer [][]string
+ line int
+ eof bool
+}
+
+// createCsvReader creates a csvReader and fills the buffer
+func createCsvReader(reader *csv.Reader, bufferRowCount int) (*csvReader, error) {
+ csv := &csvReader{reader: reader}
+ csv.buffer = make([][]string, bufferRowCount)
+ for i := 0; i < bufferRowCount && !csv.eof; i++ {
+ row, err := csv.readNextRow()
+ if err != nil {
+ return nil, err
+ }
+ csv.buffer[i] = row
+ }
+ csv.line = bufferRowCount
+ return csv, nil
+}
+
+// GetRow gets a row from the buffer if present or advances the reader to the requested row. On the end of the file only nil gets returned.
+func (csv *csvReader) GetRow(row int) ([]string, error) {
+ if row < len(csv.buffer) {
+ return csv.buffer[row], nil
+ }
+ if csv.eof {
+ return nil, nil
+ }
+ for {
+ fields, err := csv.readNextRow()
+ if err != nil {
+ return nil, err
+ }
+ if csv.eof {
+ return nil, nil
+ }
+ csv.line++
+ if csv.line-1 == row {
+ return fields, nil
+ }
+ }
+}
+
+func (csv *csvReader) readNextRow() ([]string, error) {
+ if csv.eof {
+ return nil, nil
+ }
+ row, err := csv.reader.Read()
+ if err != nil {
+ if err != io.EOF {
+ return nil, err
+ }
+ csv.eof = true
+ }
+ return row, nil
+}
+
+// CreateCsvDiff creates a tabular diff based on two CSV readers.
+func CreateCsvDiff(diffFile *DiffFile, baseReader *csv.Reader, headReader *csv.Reader) ([]*TableDiffSection, error) {
+ if baseReader != nil && headReader != nil {
+ return createCsvDiff(diffFile, baseReader, headReader)
+ }
+
+ if baseReader != nil {
+ return createCsvDiffSingle(baseReader, TableDiffCellDel)
+ }
+ return createCsvDiffSingle(headReader, TableDiffCellAdd)
+}
+
+// createCsvDiffSingle creates a tabular diff based on a single CSV reader. All cells are added or deleted.
+func createCsvDiffSingle(reader *csv.Reader, celltype TableDiffCellType) ([]*TableDiffSection, error) {
+ var rows []*TableDiffRow
+ i := 1
+ for {
+ row, err := reader.Read()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, err
+ }
+ cells := make([]*TableDiffCell, len(row))
+ for j := 0; j < len(row); j++ {
+ cells[j] = &TableDiffCell{LeftCell: row[j], Type: celltype}
+ }
+ rows = append(rows, &TableDiffRow{RowIdx: i, Cells: cells})
+ i++
+ }
+
+ return []*TableDiffSection{{Rows: rows}}, nil
+}
+
+func createCsvDiff(diffFile *DiffFile, baseReader *csv.Reader, headReader *csv.Reader) ([]*TableDiffSection, error) {
+ a, err := createCsvReader(baseReader, maxRowsToInspect)
+ if err != nil {
+ return nil, err
+ }
+
+ b, err := createCsvReader(headReader, maxRowsToInspect)
+ if err != nil {
+ return nil, err
+ }
+
+ a2b, b2a := getColumnMapping(a, b)
+
+ columns := len(a2b) + countUnmappedColumns(b2a)
+ if len(a2b) < len(b2a) {
+ columns = len(b2a) + countUnmappedColumns(a2b)
+ }
+
+ createDiffRow := func(aline int, bline int) (*TableDiffRow, error) {
+ cells := make([]*TableDiffCell, columns)
+
+ if aline == 0 || bline == 0 {
+ var (
+ row []string
+ celltype TableDiffCellType
+ err error
+ )
+ if bline == 0 {
+ row, err = a.GetRow(aline - 1)
+ celltype = TableDiffCellDel
+ } else {
+ row, err = b.GetRow(bline - 1)
+ celltype = TableDiffCellAdd
+ }
+ if err != nil {
+ return nil, err
+ }
+ if row == nil {
+ return nil, nil
+ }
+ for i := 0; i < len(row); i++ {
+ cells[i] = &TableDiffCell{LeftCell: row[i], Type: celltype}
+ }
+ return &TableDiffRow{RowIdx: bline, Cells: cells}, nil
+ }
+
+ arow, err := a.GetRow(aline - 1)
+ if err != nil {
+ return nil, err
+ }
+ brow, err := b.GetRow(bline - 1)
+ if err != nil {
+ return nil, err
+ }
+ if len(arow) == 0 && len(brow) == 0 {
+ return nil, nil
+ }
+
+ for i := 0; i < len(a2b); i++ {
+ acell, _ := getCell(arow, i)
+ if a2b[i] == unmappedColumn {
+ cells[i] = &TableDiffCell{LeftCell: acell, Type: TableDiffCellDel}
+ } else {
+ bcell, _ := getCell(brow, a2b[i])
+
+ celltype := TableDiffCellChanged
+ if acell == bcell {
+ celltype = TableDiffCellEqual
+ }
+
+ cells[i] = &TableDiffCell{LeftCell: acell, RightCell: bcell, Type: celltype}
+ }
+ }
+ for i := 0; i < len(b2a); i++ {
+ if b2a[i] == unmappedColumn {
+ bcell, _ := getCell(brow, i)
+ cells[i] = &TableDiffCell{LeftCell: bcell, Type: TableDiffCellAdd}
+ }
+ }
+
+ return &TableDiffRow{RowIdx: bline, Cells: cells}, nil
+ }
+
+ var sections []*TableDiffSection
+
+ for i, section := range diffFile.Sections {
+ var rows []*TableDiffRow
+ lines := tryMergeLines(section.Lines)
+ for j, line := range lines {
+ if i == 0 && j == 0 && (line[0] != 1 || line[1] != 1) {
+ diffRow, err := createDiffRow(1, 1)
+ if err != nil {
+ return nil, err
+ }
+ if diffRow != nil {
+ rows = append(rows, diffRow)
+ }
+ }
+ diffRow, err := createDiffRow(line[0], line[1])
+ if err != nil {
+ return nil, err
+ }
+ if diffRow != nil {
+ rows = append(rows, diffRow)
+ }
+ }
+
+ if len(rows) > 0 {
+ sections = append(sections, &TableDiffSection{Rows: rows})
+ }
+ }
+
+ return sections, nil
+}
+
+// getColumnMapping creates a mapping of columns between a and b
+func getColumnMapping(a *csvReader, b *csvReader) ([]int, []int) {
+ arow, _ := a.GetRow(0)
+ brow, _ := b.GetRow(0)
+
+ a2b := []int{}
+ b2a := []int{}
+
+ if arow != nil {
+ a2b = make([]int, len(arow))
+ }
+ if brow != nil {
+ b2a = make([]int, len(brow))
+ }
+
+ for i := 0; i < len(b2a); i++ {
+ b2a[i] = unmappedColumn
+ }
+
+ bcol := 0
+ for i := 0; i < len(a2b); i++ {
+ a2b[i] = unmappedColumn
+
+ acell, ea := getCell(arow, i)
+ if ea == nil {
+ for j := bcol; j < len(b2a); j++ {
+ bcell, eb := getCell(brow, j)
+ if eb == nil && acell == bcell {
+ a2b[i] = j
+ b2a[j] = i
+ bcol = j + 1
+ break
+ }
+ }
+ }
+ }
+
+ tryMapColumnsByContent(a, a2b, b, b2a)
+ tryMapColumnsByContent(b, b2a, a, a2b)
+
+ return a2b, b2a
+}
+
+// tryMapColumnsByContent tries to map missing columns by the content of the first lines.
+func tryMapColumnsByContent(a *csvReader, a2b []int, b *csvReader, b2a []int) {
+ start := 0
+ for i := 0; i < len(a2b); i++ {
+ if a2b[i] == unmappedColumn {
+ if b2a[start] == unmappedColumn {
+ rows := util.Min(maxRowsToInspect, util.Max(0, util.Min(len(a.buffer), len(b.buffer))-1))
+ same := 0
+ for j := 1; j <= rows; j++ {
+ acell, ea := getCell(a.buffer[j], i)
+ bcell, eb := getCell(b.buffer[j], start+1)
+ if ea == nil && eb == nil && acell == bcell {
+ same++
+ }
+ }
+ if (float32(same) / float32(rows)) > minRatioToMatch {
+ a2b[i] = start + 1
+ b2a[start+1] = i
+ }
+ }
+ }
+ start = a2b[i]
+ }
+}
+
+// getCell returns the specific cell or nil if not present.
+func getCell(row []string, column int) (string, error) {
+ if column < len(row) {
+ return row[column], nil
+ }
+ return "", errors.New("Undefined column")
+}
+
+// countUnmappedColumns returns the count of unmapped columns.
+func countUnmappedColumns(mapping []int) int {
+ count := 0
+ for i := 0; i < len(mapping); i++ {
+ if mapping[i] == unmappedColumn {
+ count++
+ }
+ }
+ return count
+}
+
+// tryMergeLines maps the separated line numbers of a git diff. The result is assumed to be ordered.
+func tryMergeLines(lines []*DiffLine) [][2]int {
+ ids := make([][2]int, len(lines))
+
+ i := 0
+ for _, line := range lines {
+ if line.Type != DiffLineSection {
+ ids[i][0] = line.LeftIdx
+ ids[i][1] = line.RightIdx
+ i++
+ }
+ }
+
+ ids = ids[:i]
+
+ result := make([][2]int, len(ids))
+
+ j := 0
+ for i = 0; i < len(ids); i++ {
+ if ids[i][0] == 0 {
+ if j > 0 && result[j-1][1] == 0 {
+ temp := j
+ for temp > 0 && result[temp-1][1] == 0 {
+ temp--
+ }
+ result[temp][1] = ids[i][1]
+ continue
+ }
+ }
+ result[j] = ids[i]
+ j++
+ }
+
+ return result[:j]
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package gitdiff
+
+import (
+ "encoding/csv"
+ "strings"
+ "testing"
+
+ csv_module "code.gitea.io/gitea/modules/csv"
+ "code.gitea.io/gitea/modules/setting"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestCSVDiff(t *testing.T) {
+ var cases = []struct {
+ diff string
+ base string
+ head string
+ cells [][2]TableDiffCellType
+ }{
+ // case 0
+ {
+ diff: `diff --git a/unittest.csv b/unittest.csv
+--- a/unittest.csv
++++ b/unittest.csv
+@@ -0,0 +1,2 @@
++col1,col2
++a,a`,
+ base: "",
+ head: "col1,col2\na,a",
+ cells: [][2]TableDiffCellType{{TableDiffCellAdd, TableDiffCellAdd}, {TableDiffCellAdd, TableDiffCellAdd}},
+ },
+ // case 1
+ {
+ diff: `diff --git a/unittest.csv b/unittest.csv
+--- a/unittest.csv
++++ b/unittest.csv
+@@ -1,2 +1,3 @@
+ col1,col2
+-a,a
++a,a
++b,b`,
+ base: "col1,col2\na,a",
+ head: "col1,col2\na,a\nb,b",
+ cells: [][2]TableDiffCellType{{TableDiffCellEqual, TableDiffCellEqual}, {TableDiffCellEqual, TableDiffCellEqual}, {TableDiffCellAdd, TableDiffCellAdd}},
+ },
+ // case 2
+ {
+ diff: `diff --git a/unittest.csv b/unittest.csv
+--- a/unittest.csv
++++ b/unittest.csv
+@@ -1,3 +1,2 @@
+ col1,col2
+-a,a
+ b,b`,
+ base: "col1,col2\na,a\nb,b",
+ head: "col1,col2\nb,b",
+ cells: [][2]TableDiffCellType{{TableDiffCellEqual, TableDiffCellEqual}, {TableDiffCellDel, TableDiffCellDel}, {TableDiffCellEqual, TableDiffCellEqual}},
+ },
+ // case 3
+ {
+ diff: `diff --git a/unittest.csv b/unittest.csv
+--- a/unittest.csv
++++ b/unittest.csv
+@@ -1,2 +1,2 @@
+ col1,col2
+-b,b
++b,c`,
+ base: "col1,col2\nb,b",
+ head: "col1,col2\nb,c",
+ cells: [][2]TableDiffCellType{{TableDiffCellEqual, TableDiffCellEqual}, {TableDiffCellEqual, TableDiffCellChanged}},
+ },
+ // case 4
+ {
+ diff: `diff --git a/unittest.csv b/unittest.csv
+--- a/unittest.csv
++++ b/unittest.csv
+@@ -1,2 +0,0 @@
+-col1,col2
+-b,c`,
+ base: "col1,col2\nb,c",
+ head: "",
+ cells: [][2]TableDiffCellType{{TableDiffCellDel, TableDiffCellDel}, {TableDiffCellDel, TableDiffCellDel}},
+ },
+ }
+
+ for n, c := range cases {
+ diff, err := ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, strings.NewReader(c.diff))
+ if err != nil {
+ t.Errorf("ParsePatch failed: %s", err)
+ }
+
+ var baseReader *csv.Reader
+ if len(c.base) > 0 {
+ baseReader = csv_module.CreateReaderAndGuessDelimiter([]byte(c.base))
+ }
+ var headReader *csv.Reader
+ if len(c.head) > 0 {
+ headReader = csv_module.CreateReaderAndGuessDelimiter([]byte(c.head))
+ }
+
+ result, err := CreateCsvDiff(diff.Files[0], baseReader, headReader)
+ assert.NoError(t, err)
+ assert.Equal(t, 1, len(result), "case %d: should be one section", n)
+
+ section := result[0]
+ assert.Equal(t, len(c.cells), len(section.Rows), "case %d: should be %d rows", n, len(c.cells))
+
+ for i, row := range section.Rows {
+ assert.Equal(t, 2, len(row.Cells), "case %d: row %d should have two cells", n, i)
+ for j, cell := range row.Cells {
+ assert.Equal(t, c.cells[i][j], cell.Type, "case %d: row %d cell %d should be equal", n, i, j)
+ }
+ }
+ }
+}
{{else}}
{{$isImage = (call $.IsImageFileInHead $file.Name)}}
{{end}}
+ {{$isCsv := (call $.IsCsvFile $file)}}
+ {{$showFileViewToggle := or $isImage $isCsv}}
<a role="button" class="fold-file muted mr-2">
{{svg "octicon-chevron-down" 18}}
</a>
<span class="file mono">{{if $file.IsRenamed}}{{$file.OldName}} → {{end}}{{$file.Name}}{{if .IsLFSFile}} ({{$.i18n.Tr "repo.stored_lfs"}}){{end}}</span>
</div>
<div class="diff-file-header-actions df ac">
+ {{if $showFileViewToggle}}
+ <div class="ui compact icon buttons">
+ <span class="ui tiny basic button poping up active file-view-toggle" data-toggle-selector="#diff-source-{{$i}}" data-content="{{$.i18n.Tr "repo.file_view_source"}}" data-position="bottom center" data-variation="tiny inverted">{{svg "octicon-code"}}</span>
+ <span class="ui tiny basic button poping up file-view-toggle" data-toggle-selector="#diff-rendered-{{$i}}" data-content="{{$.i18n.Tr "repo.file_view_rendered"}}" data-position="bottom center" data-variation="tiny inverted">{{svg "octicon-file"}}</span>
+ </div>
+ {{end}}
{{if $file.IsProtected}}
<span class="ui basic label">{{$.i18n.Tr "repo.diff.protected"}}</span>
{{end}}
</h4>
<div class="diff-file-body ui attached unstackable table segment">
{{if ne $file.Type 4}}
- <div class="file-body file-code has-context-menu{{if not $isImage}} code-diff{{end}}{{if $.IsSplitStyle}} code-diff-split{{else}} code-diff-unified{{end}}{{if $isImage}} py-4{{end}}">
- <table class="chroma{{if $isImage}} w-100{{end}}">
- <tbody>
- {{if $isImage}}
- {{template "repo/diff/image_diff" dict "file" . "root" $}}
- {{else}}
- {{if $.IsSplitStyle}}
- {{template "repo/diff/section_split" dict "file" . "root" $}}
- {{else}}
- {{template "repo/diff/section_unified" dict "file" . "root" $}}
- {{end}}
- {{end}}
- </tbody>
+ <div id="diff-source-{{$i}}" class="file-body file-code has-context-menu code-diff{{if $.IsSplitStyle}} code-diff-split{{else}} code-diff-unified{{end}}">
+ {{if $file.IsBin}}
+ <div class="diff-file-body binary" style="padding: 5px 10px;">{{$.i18n.Tr "repo.diff.bin_not_shown"}}</div>
+ {{else}}
+ <table class="chroma">
+ {{if $.IsSplitStyle}}
+ {{template "repo/diff/section_split" dict "file" . "root" $}}
+ {{else}}
+ {{template "repo/diff/section_unified" dict "file" . "root" $}}
+ {{end}}
</table>
+ {{end}}
</div>
+ {{if or $isImage $isCsv}}
+ <div id="diff-rendered-{{$i}}" class="file-body file-code has-context-menu{{if $.IsSplitStyle}} code-diff-split{{else}} code-diff-unified{{end}} hide">
+ <table class="chroma w-100">
+ {{if $isImage}}
+ {{template "repo/diff/image_diff" dict "file" . "root" $}}
+ {{else}}
+ {{template "repo/diff/csv_diff" dict "file" . "root" $}}
+ {{end}}
+ </table>
+ </div>
+ {{end}}
{{end}}
</div>
</div>
--- /dev/null
+<tr>
+ <td>
+ {{$result := call .root.CreateCsvDiff .file .root.BaseCommit .root.HeadCommit}}
+ {{if $result.Error}}
+ <div class="ui center">{{$result.Error}}</div>
+ {{else if $result.Sections}}
+ <table class="data-table">
+ {{range $i, $section := $result.Sections}}
+ <tbody {{if gt $i 0}}class="section"{{end}}>
+ {{range $j, $row := $section.Rows}}
+ <tr>
+ {{if and (eq $i 0) (eq $j 0)}}
+ <th class="line-num">{{.RowIdx}}</th>
+ {{range $j, $cell := $row.Cells}}
+ {{if eq $cell.Type 2}}
+ <th class="modified"><span class="removed-code">{{.LeftCell}}</span> <span class="added-code">{{.RightCell}}</span></th>
+ {{else if eq $cell.Type 3}}
+ <th class="added"><span class="added-code">{{.LeftCell}}</span></th>
+ {{else if eq $cell.Type 4}}
+ <th class="removed"><span class="removed-code">{{.LeftCell}}</span></th>
+ {{else}}
+ <th>{{.RightCell}}</th>
+ {{end}}
+ {{end}}
+ {{else}}
+ <td class="line-num">{{if .RowIdx}}{{.RowIdx}}{{end}}</td>
+ {{range $j, $cell := $row.Cells}}
+ {{if eq $cell.Type 2}}
+ <td class="modified"><span class="removed-code">{{.LeftCell}}</span> <span class="added-code">{{.RightCell}}</span></td>
+ {{else if eq $cell.Type 3}}
+ <td class="added"><span class="added-code">{{.LeftCell}}</span></td>
+ {{else if eq $cell.Type 4}}
+ <td class="removed"><span class="removed-code">{{.LeftCell}}</span></td>
+ {{else}}
+ <td>{{.RightCell}}</td>
+ {{end}}
+ {{end}}
+ {{end}}
+ </tr>
+ {{end}}
+ </tbody>
+ {{end}}
+ </table>
+ {{end}}
+ </td>
+</tr>
\ No newline at end of file
{{end}}
</h4>
<div class="ui attached table unstackable segment">
- <div class="file-view {{if .IsMarkup}}{{.MarkupType}} markdown{{else if .IsRenderedHTML}}plain-text{{else if .IsTextSource}}code-view{{end}}">
+ <div class="file-view {{if .IsMarkup}}{{.MarkupType}} {{if ne "csv" .MarkupType}}markdown{{end}}{{else if .IsRenderedHTML}}plain-text{{else if .IsTextSource}}code-view{{end}}">
{{if .IsMarkup}}
{{if .FileContent}}{{.FileContent | Safe}}{{end}}
{{else if .IsRenderedHTML}}
});
}
+function initFileViewToggle() {
+ $('.file-view-toggle').on('click', function() {
+ const $this = $(this);
+ $this.parent().children().removeClass('active');
+ $this.addClass('active');
+
+ const $target = $($this.data('toggle-selector'));
+ $target.parent().children().addClass('hide');
+ $target.removeClass('hide');
+ });
+}
+
function initLinkAccountView() {
const $lnkUserPage = $('.page-content.user.link-account');
if ($lnkUserPage.length === 0) {
initTableSort();
initNotificationsTable();
initPullRequestMergeInstruction();
+ initFileViewToggle();
initReleaseEditor();
initRelease();
box-shadow: inset 0 -1px 0 var(--color-secondary);
}
- .csv-data td,
- .csv-data th {
- padding: 5px;
- overflow: hidden;
- font-size: 12px;
- line-height: 1;
- text-align: left;
- white-space: nowrap;
- }
-
- .csv-data .blob-num {
- padding: 10px 8px 9px;
- text-align: right;
- border: 0;
- }
-
- .csv-data tr {
- border-top: 0;
- }
-
- .csv-data th {
- font-weight: 600;
- border-top: 0;
- }
-
.ui.list .list,
ol.ui.list ol,
ul.ui.list ul {
}
}
+ .data-table {
+ width: 100%;
+
+ tr {
+ border-top: 0;
+ }
+
+ td,
+ th {
+ padding: 5px !important;
+ overflow: hidden;
+ font-size: 12px;
+ text-align: left;
+ white-space: nowrap;
+ border: 1px solid var(--color-secondary);
+ }
+
+ td {
+ white-space: pre-line;
+ }
+
+ th {
+ font-weight: 600;
+ background: var(--color-box-header);
+ border-top: 0;
+ }
+
+ td.added,
+ th.added,
+ tr.added {
+ background-color: var(--color-diff-added-row-bg) !important;
+ }
+
+ td.removed,
+ th.removed,
+ tr.removed {
+ background-color: var(--color-diff-removed-row-bg) !important;
+ }
+
+ tbody.section {
+ border-top: 2px solid var(--color-secondary);
+ }
+
+ .line-num {
+ width: 1%;
+ min-width: 50px;
+ font-family: monospace;
+ line-height: 20px;
+ color: var(--color-secondary-dark-2);
+ white-space: nowrap;
+ vertical-align: top;
+ cursor: pointer;
+ user-select: none;
+ text-align: right;
+ background: var(--color-body);
+ border: 0;
+ }
+ }
+
.diff-detail-box {
padding: 7px 0;
background: var(--color-body);
projects / gitea.git / commitdiff