]> source.dussan.org Git - gitea.git/commitdiff
Variable expansion in repository templates (#9163)
authorJohn Olheiser <42128690+jolheiser@users.noreply.github.com>
Sat, 30 Nov 2019 06:54:47 +0000 (00:54 -0600)
committertechknowlogick <techknowlogick@gitea.io>
Sat, 30 Nov 2019 06:54:47 +0000 (01:54 -0500)
* Start expansion

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* _template rather than .template

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Use ioutil

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Add descriptions to mapping

* Start globbing

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Tune globbing

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Re-arrange imports

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Don't expand git hooks

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Add glob tests for .giteatemplate

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Parse globs separately so they can be tested more easily

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Change template location and add docs

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* nit

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Update docs/content/doc/features/gitea-directory.md

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>
* Update docs/content/doc/features/gitea-directory.md

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>
* Add upper-lower case match

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Nits

Signed-off-by: jolheiser <john.olheiser@gmail.com>
* Update models/repo_generate.go

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>
docs/content/doc/features/gitea-directory.md [new file with mode: 0644]
models/repo.go
models/repo_generate.go
models/repo_generate_test.go [new file with mode: 0644]
modules/git/repo.go

diff --git a/docs/content/doc/features/gitea-directory.md b/docs/content/doc/features/gitea-directory.md
new file mode 100644 (file)
index 0000000..e598969
--- /dev/null
@@ -0,0 +1,56 @@
+---
+date: "2019-11-28:00:00+02:00"
+title: "The .gitea Directory"
+slug: "gitea-directory"
+weight: 40
+toc: true
+draft: false
+menu:
+  sidebar:
+    parent: "features"
+    name: "The .gitea Directory"
+    weight: 50
+    identifier: "gitea-directory"
+---
+
+# The .gitea directory
+Gitea repositories can include a `.gitea` directory at their base which will store settings/configurations for certain features.
+
+## Templates
+Gitea includes template repositories, and one feature implemented with them is auto-expansion of specific variables within your template files.  
+To tell Gitea which files to expand, you must include a `template` file inside the `.gitea` directory of the template repository.  
+Gitea uses [gobwas/glob](https://github.com/gobwas/glob) for its glob syntax. It closely resembles a traditional `.gitignore`, however there may be slight differences.
+
+### Example `.gitea/template` file  
+All paths are relative to the base of the repository
+```gitignore
+# All .go files, anywhere in the repository
+**.go
+
+# All text files in the text directory
+text/*.txt
+
+# A specific file
+a/b/c/d.json
+
+# Batch files in both upper or lower case can be matched
+**.[bB][aA][tT]
+```
+**NOTE:** The `template` file will be removed from the `.gitea` directory when a repository is generated from the template.
+
+### Variable Expansion
+In any file matched by the above globs, certain variables will be expanded.  
+All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}`
+
+| Variable             | Expands To                                          |
+|----------------------|-----------------------------------------------------|
+| REPO_NAME            | The name of the generated repository                |
+| TEMPLATE_NAME        | The name of the template repository                 |
+| REPO_DESCRIPTION     | The description of the generated repository         |
+| TEMPLATE_DESCRIPTION | The description of the template repository          |
+| REPO_LINK            | The URL to the generated repository                 |
+| TEMPLATE_LINK        | The URL to the template repository                  |
+| REPO_HTTPS_URL       | The HTTP(S) clone link for the generated repository |
+| TEMPLATE_HTTPS_URL   | The HTTP(S) clone link for the template repository  |
+| REPO_SSH_URL         | The SSH clone link for the generated repository     |
+| TEMPLATE_SSH_URL     | The SSH clone link for the template repository      |
index cbe1ccc4f64af75db536029cc0cda89881b4a309..0ccf786db3bfb562d8dea82baabefb00b3478c37 100644 (file)
@@ -1361,54 +1361,6 @@ func prepareRepoCommit(e Engine, repo *Repository, tmpDir, repoPath string, opts
        return nil
 }
 
-func generateRepoCommit(e Engine, repo, templateRepo *Repository, tmpDir string) error {
-       commitTimeStr := time.Now().Format(time.RFC3339)
-       authorSig := repo.Owner.NewGitSig()
-
-       // Because this may call hooks we should pass in the environment
-       env := append(os.Environ(),
-               "GIT_AUTHOR_NAME="+authorSig.Name,
-               "GIT_AUTHOR_EMAIL="+authorSig.Email,
-               "GIT_AUTHOR_DATE="+commitTimeStr,
-               "GIT_COMMITTER_NAME="+authorSig.Name,
-               "GIT_COMMITTER_EMAIL="+authorSig.Email,
-               "GIT_COMMITTER_DATE="+commitTimeStr,
-       )
-
-       // Clone to temporary path and do the init commit.
-       templateRepoPath := templateRepo.repoPath(e)
-       _, stderr, err := process.GetManager().ExecDirEnv(
-               -1, "",
-               fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath),
-               env,
-               git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir,
-       )
-       if err != nil {
-               return fmt.Errorf("git clone: %v - %s", err, stderr)
-       }
-
-       if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
-               return fmt.Errorf("remove git dir: %v", err)
-       }
-
-       if err := git.InitRepository(tmpDir, false); err != nil {
-               return err
-       }
-
-       repoPath := repo.repoPath(e)
-       _, stderr, err = process.GetManager().ExecDirEnv(
-               -1, tmpDir,
-               fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath),
-               env,
-               git.GitExecutable, "remote", "add", "origin", repoPath,
-       )
-       if err != nil {
-               return fmt.Errorf("git remote add: %v - %s", err, stderr)
-       }
-
-       return initRepoCommit(tmpDir, repo.Owner)
-}
-
 func checkInitRepository(repoPath string) (err error) {
        // Somehow the directory could exist.
        if com.IsExist(repoPath) {
index 234bdc27f2edae0ae3e7bfe75964dd734a11b23d..56a3940ac18c24579e49c5291d1f803d25dab248 100644 (file)
@@ -6,7 +6,9 @@ package models
 
 import (
        "fmt"
+       "io/ioutil"
        "os"
+       "path"
        "path/filepath"
        "strconv"
        "strings"
@@ -14,7 +16,10 @@ import (
 
        "code.gitea.io/gitea/modules/git"
        "code.gitea.io/gitea/modules/log"
+       "code.gitea.io/gitea/modules/process"
+       "code.gitea.io/gitea/modules/util"
 
+       "github.com/gobwas/glob"
        "github.com/unknwon/com"
 )
 
@@ -36,8 +41,148 @@ func (gro GenerateRepoOptions) IsValid() bool {
        return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels // or other items as they are added
 }
 
+// GiteaTemplate holds information about a .gitea/template file
+type GiteaTemplate struct {
+       Path    string
+       Content []byte
+
+       globs []glob.Glob
+}
+
+// Globs parses the .gitea/template globs or returns them if they were already parsed
+func (gt GiteaTemplate) Globs() []glob.Glob {
+       if gt.globs != nil {
+               return gt.globs
+       }
+
+       gt.globs = make([]glob.Glob, 0)
+       lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n")
+       for _, line := range lines {
+               line = strings.TrimSpace(line)
+               if line == "" || strings.HasPrefix(line, "#") {
+                       continue
+               }
+               g, err := glob.Compile(line, '/')
+               if err != nil {
+                       log.Info("Invalid glob expression '%s' (skipped): %v", line, err)
+                       continue
+               }
+               gt.globs = append(gt.globs, g)
+       }
+       return gt.globs
+}
+
+func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) {
+       gtPath := filepath.Join(tmpDir, ".gitea", "template")
+       if _, err := os.Stat(gtPath); os.IsNotExist(err) {
+               return nil, nil
+       } else if err != nil {
+               return nil, err
+       }
+
+       content, err := ioutil.ReadFile(gtPath)
+       if err != nil {
+               return nil, err
+       }
+
+       gt := &GiteaTemplate{
+               Path:    gtPath,
+               Content: content,
+       }
+
+       return gt, nil
+}
+
+func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, tmpDir string) error {
+       commitTimeStr := time.Now().Format(time.RFC3339)
+       authorSig := repo.Owner.NewGitSig()
+
+       // Because this may call hooks we should pass in the environment
+       env := append(os.Environ(),
+               "GIT_AUTHOR_NAME="+authorSig.Name,
+               "GIT_AUTHOR_EMAIL="+authorSig.Email,
+               "GIT_AUTHOR_DATE="+commitTimeStr,
+               "GIT_COMMITTER_NAME="+authorSig.Name,
+               "GIT_COMMITTER_EMAIL="+authorSig.Email,
+               "GIT_COMMITTER_DATE="+commitTimeStr,
+       )
+
+       // Clone to temporary path and do the init commit.
+       templateRepoPath := templateRepo.repoPath(e)
+       if err := git.Clone(templateRepoPath, tmpDir, git.CloneRepoOptions{
+               Depth: 1,
+       }); err != nil {
+               return fmt.Errorf("git clone: %v", err)
+       }
+
+       if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
+               return fmt.Errorf("remove git dir: %v", err)
+       }
+
+       // Variable expansion
+       gt, err := checkGiteaTemplate(tmpDir)
+       if err != nil {
+               return fmt.Errorf("checkGiteaTemplate: %v", err)
+       }
+
+       if err := os.Remove(gt.Path); err != nil {
+               return fmt.Errorf("remove .giteatemplate: %v", err)
+       }
+
+       // Avoid walking tree if there are no globs
+       if len(gt.Globs()) > 0 {
+               tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/"
+               if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error {
+                       if walkErr != nil {
+                               return walkErr
+                       }
+
+                       if info.IsDir() {
+                               return nil
+                       }
+
+                       base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash)
+                       for _, g := range gt.Globs() {
+                               if g.Match(base) {
+                                       content, err := ioutil.ReadFile(path)
+                                       if err != nil {
+                                               return err
+                                       }
+
+                                       if err := ioutil.WriteFile(path,
+                                               []byte(generateExpansion(string(content), templateRepo, generateRepo)),
+                                               0644); err != nil {
+                                               return err
+                                       }
+                                       break
+                               }
+                       }
+                       return nil
+               }); err != nil {
+                       return err
+               }
+       }
+
+       if err := git.InitRepository(tmpDir, false); err != nil {
+               return err
+       }
+
+       repoPath := repo.repoPath(e)
+       _, stderr, err := process.GetManager().ExecDirEnv(
+               -1, tmpDir,
+               fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath),
+               env,
+               git.GitExecutable, "remote", "add", "origin", repoPath,
+       )
+       if err != nil {
+               return fmt.Errorf("git remote add: %v - %s", err, stderr)
+       }
+
+       return initRepoCommit(tmpDir, repo.Owner)
+}
+
 // generateRepository initializes repository from template
-func generateRepository(e Engine, repo, templateRepo *Repository) (err error) {
+func generateRepository(e Engine, repo, templateRepo, generateRepo *Repository) (err error) {
        tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond()))
 
        if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
@@ -50,7 +195,7 @@ func generateRepository(e Engine, repo, templateRepo *Repository) (err error) {
                }
        }()
 
-       if err = generateRepoCommit(e, repo, templateRepo, tmpDir); err != nil {
+       if err = generateRepoCommit(e, repo, templateRepo, generateRepo, tmpDir); err != nil {
                return fmt.Errorf("generateRepoCommit: %v", err)
        }
 
@@ -95,7 +240,7 @@ func GenerateRepository(ctx DBContext, doer, owner *User, templateRepo *Reposito
 
 // GenerateGitContent generates git content from a template repository
 func GenerateGitContent(ctx DBContext, templateRepo, generateRepo *Repository) error {
-       if err := generateRepository(ctx.e, generateRepo, templateRepo); err != nil {
+       if err := generateRepository(ctx.e, generateRepo, templateRepo, generateRepo); err != nil {
                return err
        }
 
@@ -210,3 +355,36 @@ func GenerateIssueLabels(ctx DBContext, templateRepo, generateRepo *Repository)
        }
        return nil
 }
+
+func generateExpansion(src string, templateRepo, generateRepo *Repository) string {
+       return os.Expand(src, func(key string) string {
+               switch key {
+               case "REPO_NAME":
+                       return generateRepo.Name
+               case "TEMPLATE_NAME":
+                       return templateRepo.Name
+               case "REPO_DESCRIPTION":
+                       return generateRepo.Description
+               case "TEMPLATE_DESCRIPTION":
+                       return templateRepo.Description
+               case "REPO_OWNER":
+                       return generateRepo.MustOwnerName()
+               case "TEMPLATE_OWNER":
+                       return templateRepo.MustOwnerName()
+               case "REPO_LINK":
+                       return generateRepo.Link()
+               case "TEMPLATE_LINK":
+                       return templateRepo.Link()
+               case "REPO_HTTPS_URL":
+                       return generateRepo.CloneLink().HTTPS
+               case "TEMPLATE_HTTPS_URL":
+                       return templateRepo.CloneLink().HTTPS
+               case "REPO_SSH_URL":
+                       return generateRepo.CloneLink().SSH
+               case "TEMPLATE_SSH_URL":
+                       return templateRepo.CloneLink().SSH
+               default:
+                       return key
+               }
+       })
+}
diff --git a/models/repo_generate_test.go b/models/repo_generate_test.go
new file mode 100644 (file)
index 0000000..53ab4fc
--- /dev/null
@@ -0,0 +1,57 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import (
+       "testing"
+
+       "github.com/stretchr/testify/assert"
+)
+
+var giteaTemplate = []byte(`
+# Header
+
+# All .go files
+**.go
+
+# All text files in /text/
+text/*.txt
+
+# All files in modules folders
+**/modules/*
+`)
+
+func TestGiteaTemplate(t *testing.T) {
+       gt := GiteaTemplate{Content: giteaTemplate}
+       assert.Equal(t, len(gt.Globs()), 3)
+
+       tt := []struct {
+               Path  string
+               Match bool
+       }{
+               {Path: "main.go", Match: true},
+               {Path: "a/b/c/d/e.go", Match: true},
+               {Path: "main.txt", Match: false},
+               {Path: "a/b.txt", Match: false},
+               {Path: "text/a.txt", Match: true},
+               {Path: "text/b.txt", Match: true},
+               {Path: "text/c.json", Match: false},
+               {Path: "a/b/c/modules/README.md", Match: true},
+               {Path: "a/b/c/modules/d/README.md", Match: false},
+       }
+
+       for _, tc := range tt {
+               t.Run(tc.Path, func(t *testing.T) {
+                       match := false
+                       for _, g := range gt.Globs() {
+                               if g.Match(tc.Path) {
+                                       match = true
+                                       break
+                               }
+                       }
+                       assert.Equal(t, tc.Match, match)
+               })
+       }
+}
index e277f896bf969b3ada611963d7e8c356148e96b3..03296d56abd66c4b7750d6ddb8400201f7f6eaab 100644 (file)
@@ -161,6 +161,7 @@ type CloneRepoOptions struct {
        Branch     string
        Shared     bool
        NoCheckout bool
+       Depth      int
 }
 
 // Clone clones original repository to target path.
@@ -193,6 +194,9 @@ func CloneWithArgs(from, to string, args []string, opts CloneRepoOptions) (err e
        if opts.NoCheckout {
                cmd.AddArguments("--no-checkout")
        }
+       if opts.Depth > 0 {
+               cmd.AddArguments("--depth", strconv.Itoa(opts.Depth))
+       }
 
        if len(opts.Branch) > 0 {
                cmd.AddArguments("-b", opts.Branch)