### Summary Extend the template variable substitution to replace file paths. This can be helpful for setting up log files & directories that should match the repository name. ### PR Changes - Move files matching glob pattern when setting up repos from template - For security, added ~escaping~ sanitization for cross-platform support and to prevent directory traversal (thanks @silverwind for the reference) - Added unit testing for escaping function - Fixed the integration tests for repo template generation by passing the repo_template_id - Updated the integration testfiles to add some variable substitution & assert the outputs I had to fix the existing repo template integration test and extend it to add a check for variable substitutions. Example: ![image](https://github.com/go-gitea/gitea/assets/12700993/621feb09-0ef3-460e-afa8-da74cd84fa4e)tags/v1.21.0-rc0
@@ -51,6 +51,8 @@ a/b/c/d.json | |||
In any file matched by the above globs, certain variables will be expanded. | |||
Matching filenames and paths can also be expanded, and are conservatively sanitized to support cross-platform filesystems. | |||
All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}` | |||
| Variable | Expands To | Transformable | |
@@ -11,6 +11,7 @@ import ( | |||
"os" | |||
"path" | |||
"path/filepath" | |||
"regexp" | |||
"strings" | |||
"time" | |||
@@ -48,7 +49,7 @@ var defaultTransformers = []transformer{ | |||
{Name: "TITLE", Transform: util.ToTitleCase}, | |||
} | |||
func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository) string { | |||
func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository, sanitizeFileName bool) string { | |||
expansions := []expansion{ | |||
{Name: "REPO_NAME", Value: generateRepo.Name, Transformers: defaultTransformers}, | |||
{Name: "TEMPLATE_NAME", Value: templateRepo.Name, Transformers: defaultTransformers}, | |||
@@ -74,6 +75,9 @@ func generateExpansion(src string, templateRepo, generateRepo *repo_model.Reposi | |||
return os.Expand(src, func(key string) string { | |||
if expansion, ok := expansionMap[key]; ok { | |||
if sanitizeFileName { | |||
return fileNameSanitize(expansion) | |||
} | |||
return expansion | |||
} | |||
return key | |||
@@ -191,10 +195,24 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r | |||
} | |||
if err := os.WriteFile(path, | |||
[]byte(generateExpansion(string(content), templateRepo, generateRepo)), | |||
[]byte(generateExpansion(string(content), templateRepo, generateRepo, false)), | |||
0o644); err != nil { | |||
return err | |||
} | |||
substPath := filepath.FromSlash(filepath.Join(tmpDirSlash, | |||
generateExpansion(base, templateRepo, generateRepo, true))) | |||
// Create parent subdirectories if needed or continue silently if it exists | |||
if err := os.MkdirAll(filepath.Dir(substPath), 0o755); err != nil { | |||
return err | |||
} | |||
// Substitute filename variables | |||
if err := os.Rename(path, substPath); err != nil { | |||
return err | |||
} | |||
break | |||
} | |||
} | |||
@@ -353,3 +371,13 @@ func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templ | |||
return generateRepo, nil | |||
} | |||
// Sanitize user input to valid OS filenames | |||
// | |||
// Based on https://github.com/sindresorhus/filename-reserved-regex | |||
// Adds ".." to prevent directory traversal | |||
func fileNameSanitize(s string) string { | |||
re := regexp.MustCompile(`(?i)\.\.|[<>:\"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`) | |||
return strings.TrimSpace(re.ReplaceAllString(s, "_")) | |||
} |
@@ -54,3 +54,14 @@ func TestGiteaTemplate(t *testing.T) { | |||
}) | |||
} | |||
} | |||
func TestFileNameSanitize(t *testing.T) { | |||
assert.Equal(t, "test_CON", fileNameSanitize("test_CON")) | |||
assert.Equal(t, "test CON", fileNameSanitize("test CON ")) | |||
assert.Equal(t, "__traverse__", fileNameSanitize("../traverse/..")) | |||
assert.Equal(t, "http___localhost_3003_user_test.git", fileNameSanitize("http://localhost:3003/user/test.git")) | |||
assert.Equal(t, "_", fileNameSanitize("CON")) | |||
assert.Equal(t, "_", fileNameSanitize("con")) | |||
assert.Equal(t, "_", fileNameSanitize("\u0000")) | |||
assert.Equal(t, "目标", fileNameSanitize("目标")) | |||
} |
@@ -0,0 +1,2 @@ | |||
x�ÎAJÅ0€a×9Å\@Ij2™ÂCÞwž"™Ìh±i¥�ÞÞ·qïö‡~Þ{_¦ ì�çæ+cÔ)M�•³*rȉSD&’ŠM³û*‡l¥pm*³Ž5fE_ªP 8û˜´D™QCËÉ•aûo?«À+\>ÛèûfÛ¸¾÷²¬O¼÷HH9G"xôÑ{w¯÷;“ÿ8 | |||
iþsîÖœ£ž¶Ø0�ï²9Ý/å IH |
@@ -1 +1 @@ | |||
aacbdfe9e1c4b47f60abe81849045fa4e96f1d75 | |||
2a83b349fa234131fc5db6f2a0498d3f4d3d6038 |
@@ -7,16 +7,18 @@ import ( | |||
"fmt" | |||
"net/http" | |||
"net/http/httptest" | |||
"strings" | |||
"testing" | |||
"code.gitea.io/gitea/models/unittest" | |||
user_model "code.gitea.io/gitea/models/user" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/tests" | |||
"github.com/stretchr/testify/assert" | |||
) | |||
func testRepoGenerate(t *testing.T, session *TestSession, templateOwnerName, templateRepoName, generateOwnerName, generateRepoName string) *httptest.ResponseRecorder { | |||
func testRepoGenerate(t *testing.T, session *TestSession, templateID, templateOwnerName, templateRepoName, generateOwnerName, generateRepoName string) *httptest.ResponseRecorder { | |||
generateOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: generateOwnerName}) | |||
// Step0: check the existence of the generated repo | |||
@@ -41,16 +43,38 @@ func testRepoGenerate(t *testing.T, session *TestSession, templateOwnerName, tem | |||
_, exists = htmlDoc.doc.Find(fmt.Sprintf(".owner.dropdown .item[data-value=\"%d\"]", generateOwner.ID)).Attr("data-value") | |||
assert.True(t, exists, fmt.Sprintf("Generate owner '%s' is not present in select box", generateOwnerName)) | |||
req = NewRequestWithValues(t, "POST", link, map[string]string{ | |||
"_csrf": htmlDoc.GetCSRF(), | |||
"uid": fmt.Sprintf("%d", generateOwner.ID), | |||
"repo_name": generateRepoName, | |||
"git_content": "true", | |||
"_csrf": htmlDoc.GetCSRF(), | |||
"uid": fmt.Sprintf("%d", generateOwner.ID), | |||
"repo_name": generateRepoName, | |||
"repo_template": templateID, | |||
"git_content": "true", | |||
}) | |||
session.MakeRequest(t, req, http.StatusSeeOther) | |||
// Step4: check the existence of the generated repo | |||
req = NewRequestf(t, "GET", "/%s/%s", generateOwnerName, generateRepoName) | |||
session.MakeRequest(t, req, http.StatusOK) | |||
// Step5: check substituted values in Readme | |||
req = NewRequestf(t, "GET", "/%s/%s/raw/branch/master/README.md", generateOwnerName, generateRepoName) | |||
resp = session.MakeRequest(t, req, http.StatusOK) | |||
body := fmt.Sprintf(`# %s Readme | |||
Owner: %s | |||
Link: /%s/%s | |||
Clone URL: %s%s/%s.git`, | |||
generateRepoName, | |||
strings.ToUpper(generateOwnerName), | |||
generateOwnerName, | |||
generateRepoName, | |||
setting.AppURL, | |||
generateOwnerName, | |||
generateRepoName) | |||
assert.Equal(t, body, resp.Body.String()) | |||
// Step6: check substituted values in substituted file path ${REPO_NAME} | |||
req = NewRequestf(t, "GET", "/%s/%s/raw/branch/master/%s.log", generateOwnerName, generateRepoName, generateRepoName) | |||
resp = session.MakeRequest(t, req, http.StatusOK) | |||
assert.Equal(t, generateRepoName, resp.Body.String()) | |||
return resp | |||
} | |||
@@ -58,11 +82,11 @@ func testRepoGenerate(t *testing.T, session *TestSession, templateOwnerName, tem | |||
func TestRepoGenerate(t *testing.T) { | |||
defer tests.PrepareTestEnv(t)() | |||
session := loginUser(t, "user1") | |||
testRepoGenerate(t, session, "user27", "template1", "user1", "generated1") | |||
testRepoGenerate(t, session, "44", "user27", "template1", "user1", "generated1") | |||
} | |||
func TestRepoGenerateToOrg(t *testing.T) { | |||
defer tests.PrepareTestEnv(t)() | |||
session := loginUser(t, "user2") | |||
testRepoGenerate(t, session, "user27", "template1", "user2", "generated2") | |||
testRepoGenerate(t, session, "44", "user27", "template1", "user2", "generated2") | |||
} |