* draft * Implement extract command * Fix nits and force args on extract * Add !bindata stub, support Windows, fmt * fix vendored flag * Remove leading slash for matching * Add docs * Fix typos * Add embedded view command Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>tags/v1.10.5
@@ -0,0 +1,332 @@ | |||
// Copyright 2020 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
// +build bindata | |||
package cmd | |||
import ( | |||
"errors" | |||
"fmt" | |||
"os" | |||
"path/filepath" | |||
"sort" | |||
"strings" | |||
"code.gitea.io/gitea/modules/log" | |||
"code.gitea.io/gitea/modules/options" | |||
"code.gitea.io/gitea/modules/public" | |||
"code.gitea.io/gitea/modules/setting" | |||
"code.gitea.io/gitea/modules/templates" | |||
"github.com/gobwas/glob" | |||
"github.com/urfave/cli" | |||
) | |||
// Cmdembedded represents the available extract sub-command. | |||
var ( | |||
Cmdembedded = cli.Command{ | |||
Name: "embedded", | |||
Usage: "Extract embedded resources", | |||
Description: "A command for extracting embedded resources, like templates and images", | |||
Subcommands: []cli.Command{ | |||
subcmdList, | |||
subcmdView, | |||
subcmdExtract, | |||
}, | |||
} | |||
subcmdList = cli.Command{ | |||
Name: "list", | |||
Usage: "List files matching the given pattern", | |||
Action: runList, | |||
Flags: []cli.Flag{ | |||
cli.BoolFlag{ | |||
Name: "include-vendored,vendor", | |||
Usage: "Include files under public/vendor as well", | |||
}, | |||
}, | |||
} | |||
subcmdView = cli.Command{ | |||
Name: "view", | |||
Usage: "View a file matching the given pattern", | |||
Action: runView, | |||
Flags: []cli.Flag{ | |||
cli.BoolFlag{ | |||
Name: "include-vendored,vendor", | |||
Usage: "Include files under public/vendor as well", | |||
}, | |||
}, | |||
} | |||
subcmdExtract = cli.Command{ | |||
Name: "extract", | |||
Usage: "Extract resources", | |||
Action: runExtract, | |||
Flags: []cli.Flag{ | |||
cli.BoolFlag{ | |||
Name: "include-vendored,vendor", | |||
Usage: "Include files under public/vendor as well", | |||
}, | |||
cli.BoolFlag{ | |||
Name: "overwrite", | |||
Usage: "Overwrite files if they already exist", | |||
}, | |||
cli.BoolFlag{ | |||
Name: "rename", | |||
Usage: "Rename files as {name}.bak if they already exist (overwrites previous .bak)", | |||
}, | |||
cli.BoolFlag{ | |||
Name: "custom", | |||
Usage: "Extract to the 'custom' directory as per app.ini", | |||
}, | |||
cli.StringFlag{ | |||
Name: "destination,dest-dir", | |||
Usage: "Extract to the specified directory", | |||
}, | |||
}, | |||
} | |||
sections map[string]*section | |||
assets []asset | |||
) | |||
type section struct { | |||
Path string | |||
Names func() []string | |||
IsDir func(string) (bool, error) | |||
Asset func(string) ([]byte, error) | |||
} | |||
type asset struct { | |||
Section *section | |||
Name string | |||
Path string | |||
} | |||
func initEmbeddedExtractor(c *cli.Context) error { | |||
// Silence the console logger | |||
log.DelNamedLogger("console") | |||
log.DelNamedLogger(log.DEFAULT) | |||
// Read configuration file | |||
setting.NewContext() | |||
pats, err := getPatterns(c.Args()) | |||
if err != nil { | |||
return err | |||
} | |||
sections := make(map[string]*section, 3) | |||
sections["public"] = §ion{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset} | |||
sections["options"] = §ion{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset} | |||
sections["templates"] = §ion{Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir, Asset: templates.Asset} | |||
for _, sec := range sections { | |||
assets = append(assets, buildAssetList(sec, pats, c)...) | |||
} | |||
// Sort assets | |||
sort.SliceStable(assets, func(i, j int) bool { | |||
return assets[i].Path < assets[j].Path | |||
}) | |||
return nil | |||
} | |||
func runList(c *cli.Context) error { | |||
if err := runListDo(c); err != nil { | |||
fmt.Fprintf(os.Stderr, "%v\n", err) | |||
return err | |||
} | |||
return nil | |||
} | |||
func runView(c *cli.Context) error { | |||
if err := runViewDo(c); err != nil { | |||
fmt.Fprintf(os.Stderr, "%v\n", err) | |||
return err | |||
} | |||
return nil | |||
} | |||
func runExtract(c *cli.Context) error { | |||
if err := runExtractDo(c); err != nil { | |||
fmt.Fprintf(os.Stderr, "%v\n", err) | |||
return err | |||
} | |||
return nil | |||
} | |||
func runListDo(c *cli.Context) error { | |||
if err := initEmbeddedExtractor(c); err != nil { | |||
return err | |||
} | |||
for _, a := range assets { | |||
fmt.Println(a.Path) | |||
} | |||
return nil | |||
} | |||
func runViewDo(c *cli.Context) error { | |||
if err := initEmbeddedExtractor(c); err != nil { | |||
return err | |||
} | |||
if len(assets) == 0 { | |||
return fmt.Errorf("No files matched the given pattern") | |||
} else if len(assets) > 1 { | |||
return fmt.Errorf("Too many files matched the given pattern; try to be more specific") | |||
} | |||
data, err := assets[0].Section.Asset(assets[0].Name) | |||
if err != nil { | |||
return fmt.Errorf("%s: %v", assets[0].Path, err) | |||
} | |||
if _, err = os.Stdout.Write(data); err != nil { | |||
return fmt.Errorf("%s: %v", assets[0].Path, err) | |||
} | |||
return nil | |||
} | |||
func runExtractDo(c *cli.Context) error { | |||
if err := initEmbeddedExtractor(c); err != nil { | |||
return err | |||
} | |||
if len(c.Args()) == 0 { | |||
return fmt.Errorf("A list of pattern of files to extract is mandatory (e.g. '**' for all)") | |||
} | |||
destdir := "." | |||
if c.IsSet("destination") { | |||
destdir = c.String("destination") | |||
} else if c.Bool("custom") { | |||
destdir = setting.CustomPath | |||
fmt.Println("Using app.ini at", setting.CustomConf) | |||
} | |||
fi, err := os.Stat(destdir) | |||
if errors.Is(err, os.ErrNotExist) { | |||
// In case Windows users attempt to provide a forward-slash path | |||
wdestdir := filepath.FromSlash(destdir) | |||
if wfi, werr := os.Stat(wdestdir); werr == nil { | |||
destdir = wdestdir | |||
fi = wfi | |||
err = nil | |||
} | |||
} | |||
if err != nil { | |||
return fmt.Errorf("%s: %s", destdir, err) | |||
} else if !fi.IsDir() { | |||
return fmt.Errorf("%s is not a directory.", destdir) | |||
} | |||
fmt.Printf("Extracting to %s:\n", destdir) | |||
overwrite := c.Bool("overwrite") | |||
rename := c.Bool("rename") | |||
for _, a := range assets { | |||
if err := extractAsset(destdir, a, overwrite, rename); err != nil { | |||
// Non-fatal error | |||
fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err) | |||
} | |||
} | |||
return nil | |||
} | |||
func extractAsset(d string, a asset, overwrite, rename bool) error { | |||
dest := filepath.Join(d, filepath.FromSlash(a.Path)) | |||
dir := filepath.Dir(dest) | |||
data, err := a.Section.Asset(a.Name) | |||
if err != nil { | |||
return fmt.Errorf("%s: %v", a.Path, err) | |||
} | |||
if err := os.MkdirAll(dir, os.ModePerm); err != nil { | |||
return fmt.Errorf("%s: %v", dir, err) | |||
} | |||
perms := os.ModePerm & 0666 | |||
fi, err := os.Lstat(dest) | |||
if err != nil { | |||
if !errors.Is(err, os.ErrNotExist) { | |||
return fmt.Errorf("%s: %v", dest, err) | |||
} | |||
} else if !overwrite && !rename { | |||
fmt.Printf("%s already exists; skipped.\n", dest) | |||
return nil | |||
} else if !fi.Mode().IsRegular() { | |||
return fmt.Errorf("%s already exists, but it's not a regular file", dest) | |||
} else if rename { | |||
if err := os.Rename(dest, dest+".bak"); err != nil { | |||
return fmt.Errorf("Error creating backup for %s: %v", dest, err) | |||
} | |||
// Attempt to respect file permissions mask (even if user:group will be set anew) | |||
perms = fi.Mode() | |||
} | |||
file, err := os.OpenFile(dest, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, perms) | |||
if err != nil { | |||
return fmt.Errorf("%s: %v", dest, err) | |||
} | |||
defer file.Close() | |||
if _, err = file.Write(data); err != nil { | |||
return fmt.Errorf("%s: %v", dest, err) | |||
} | |||
fmt.Println(dest) | |||
return nil | |||
} | |||
func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset { | |||
var results = make([]asset, 0, 64) | |||
for _, name := range sec.Names() { | |||
if isdir, err := sec.IsDir(name); !isdir && err == nil { | |||
if sec.Path == "public" && | |||
strings.HasPrefix(name, "vendor/") && | |||
!c.Bool("include-vendored") { | |||
continue | |||
} | |||
matchName := sec.Path + "/" + name | |||
for _, g := range globs { | |||
if g.Match(matchName) { | |||
results = append(results, asset{Section: sec, | |||
Name: name, | |||
Path: sec.Path + "/" + name}) | |||
break | |||
} | |||
} | |||
} | |||
} | |||
return results | |||
} | |||
func getPatterns(args []string) ([]glob.Glob, error) { | |||
if len(args) == 0 { | |||
args = []string{"**"} | |||
} | |||
pat := make([]glob.Glob, len(args)) | |||
for i := range args { | |||
if g, err := glob.Compile(args[i], '/'); err != nil { | |||
return nil, fmt.Errorf("'%s': Invalid glob pattern: %v", args[i], err) | |||
} else { | |||
pat[i] = g | |||
} | |||
} | |||
return pat, nil | |||
} |
@@ -0,0 +1,30 @@ | |||
// Copyright 2020 The Gitea Authors. All rights reserved. | |||
// Use of this source code is governed by a MIT-style | |||
// license that can be found in the LICENSE file. | |||
// +build !bindata | |||
package cmd | |||
import ( | |||
"fmt" | |||
"os" | |||
"github.com/urfave/cli" | |||
) | |||
// Cmdembedded represents the available extract sub-command. | |||
var ( | |||
Cmdembedded = cli.Command{ | |||
Name: "embedded", | |||
Usage: "Extract embedded resources", | |||
Description: "A command for extracting embedded resources, like templates and images", | |||
Action: extractorNotImplemented, | |||
} | |||
) | |||
func extractorNotImplemented(c *cli.Context) error { | |||
err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in builds without bindata") | |||
fmt.Fprintf(os.Stderr, "%s\n", err) | |||
return err | |||
} |
@@ -0,0 +1,115 @@ | |||
--- | |||
date: "2020-01-25T21:00:00-03:00" | |||
title: "Embedded data extraction tool" | |||
slug: "cmd-embedded" | |||
weight: 40 | |||
toc: true | |||
draft: false | |||
menu: | |||
sidebar: | |||
parent: "advanced" | |||
name: "Embedded data extraction tool" | |||
weight: 40 | |||
identifier: "cmd-embedded" | |||
--- | |||
# Embedded data extraction tool | |||
Gitea's executable contains all the resources required to run: templates, images, style-sheets | |||
and translations. Any of them can be overridden by placing a replacement in a matching path | |||
inside the `custom` directory (see [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}})). | |||
To obtain a copy of the embedded resources ready for editing, the `embedded` command from the CLI | |||
can be used from the OS shell interface. | |||
## Listing resources | |||
To list resources embedded in Gitea's executable, use the following syntax: | |||
``` | |||
gitea embedded list [--include-vendored] [patterns...] | |||
``` | |||
The `--include-vendored` flag makes the command include vendored files, which are | |||
normally excluded; that is, files from external libraries that are required for Gitea | |||
(e.g. [font-awesome](https://fontawesome.com/), [octicons](https://octicons.github.com/), etc). | |||
A list of file search patterns can be provided. Gitea uses [gobwas/glob](https://github.com/gobwas/glob) | |||
for its glob syntax. Here are some examples: | |||
- List all template files, in any virtual directory: `**.tmpl` | |||
- List all mail template files: `templates/mail/**.tmpl` | |||
- List all files inside `public/img`: `public/img/**` | |||
Don't forget to use quotes for the patterns, as spaces, `*` and other characters might have | |||
a special meaning for your command shell. | |||
If no pattern is provided, all files are listed. | |||
#### Example | |||
Listing all embedded files with `openid` in their path: | |||
``` | |||
$ gitea embedded list '**openid**' | |||
public/img/auth/openid_connect.png | |||
public/img/openid-16x16.png | |||
templates/user/auth/finalize_openid.tmpl | |||
templates/user/auth/signin_openid.tmpl | |||
templates/user/auth/signup_openid_connect.tmpl | |||
templates/user/auth/signup_openid_navbar.tmpl | |||
templates/user/auth/signup_openid_register.tmpl | |||
templates/user/settings/security_openid.tmpl | |||
``` | |||
## Extracting resources | |||
To extract resources embedded in Gitea's executable, use the following syntax: | |||
``` | |||
gitea [--config {file}] embedded extract [--destination {dir}|--custom] [--overwrite|--rename] [--include-vendored] {patterns...} | |||
``` | |||
The `--config` option tells gitea the location of the `app.ini` configuration file if | |||
it's not in its default location. This option is only used with the `--custom` flag. | |||
The `--destination` option tells gitea the directory where the files must be extracted to. | |||
The default is the current directory. | |||
The `--custom` flag tells gitea to extract the files directly into the `custom` directory. | |||
For this to work, the command needs to know the location of the `app.ini` configuration | |||
file (`--config`) and, depending of the configuration, be ran from the directory where | |||
gitea normally starts. See [Customizing Gitea]({{< relref "doc/advanced/customizing-gitea.en-us.md" >}}) for details. | |||
The `--overwrite` flag allows any existing files in the destination directory to be overwritten. | |||
The `--rename` flag tells gitea to rename any existing files in the destination directory | |||
as `filename.bak`. Previous `.bak` files are overwritten. | |||
At least one file search pattern must be provided; see `list` subcomand above for pattern | |||
syntax and examples. | |||
#### Important notice | |||
Make sure to **only extract those files that require customization**. Files that | |||
are present in the `custom` directory are not upgraded by Gitea's upgrade process. | |||
When Gitea is upgraded to a new version (by replacing the executable), many of the | |||
embedded files will suffer changes. Gitea will honor and use any files found | |||
in the `custom` directory, even if they are old and incompatible. | |||
#### Example | |||
Extracting mail templates to a temporary directory: | |||
``` | |||
$ mkdir tempdir | |||
$ gitea embedded extract --destination tempdir 'templates/mail/**.tmpl' | |||
Extracting to tempdir: | |||
tempdir/templates/mail/auth/activate.tmpl | |||
tempdir/templates/mail/auth/activate_email.tmpl | |||
tempdir/templates/mail/auth/register_notify.tmpl | |||
tempdir/templates/mail/auth/reset_passwd.tmpl | |||
tempdir/templates/mail/issue/assigned.tmpl | |||
tempdir/templates/mail/issue/default.tmpl | |||
tempdir/templates/mail/notify/collaborator.tmpl | |||
``` |
@@ -57,14 +57,21 @@ the url `http://gitea.domain.tld/image.png`. | |||
Place the png image at the following path: `custom/public/img/avatar_default.png` | |||
## Customizing Gitea pages | |||
## Customizing Gitea pages and resources | |||
The `custom/templates` folder allows changing every single page of Gitea. Templates | |||
to override can be found in the [`templates`](https://github.com/go-gitea/gitea/tree/master/templates) directory of Gitea source (Note: the example link is from `master` branch. Make sure to copy templates from same release you are using). Override by | |||
making a copy of the file under `custom/templates` using a full path structure | |||
matching source. | |||
Gitea's executable contains all the resources required to run: templates, images, style-sheets | |||
and translations. Any of them can be overridden by placing a replacement in a matching path | |||
inside the `custom` directory. For example, to replace the default `.gitignore` provided | |||
for C++ repositories, we want to replace `options/gitignore/C++`. To do this, a replacement | |||
must be placed in `custom/options/gitignore/C++` (see about the location of the `custom` | |||
directory at the top of this document). | |||
Any statement contained inside `{{` and `}}` are Gitea's template syntax and | |||
Every single page of Gitea can be changed. Dynamic content is generated using [go templates](https://golang.org/pkg/html/template/), | |||
which can be modified by placing replacements below the `custom/templates` directory. | |||
To obtain any embedded file (including templates), the [`gitea embedded` tool]({{< relref "doc/advanced/cmd-embedded.en-us.md" >}}) can be used. Alternatively, they can be found in the [`templates`](https://github.com/go-gitea/gitea/tree/master/templates) directory of Gitea source (Note: the example link is from the `master` branch. Make sure to use templates compatible with the release you are using). | |||
Be aware that any statement contained inside `{{` and `}}` are Gitea's template syntax and | |||
shouldn't be touched without fully understanding these components. | |||
### Customizing startpage / homepage |
@@ -70,6 +70,7 @@ arguments - which can alternatively be run by running the subcommand web.` | |||
cmd.CmdConvert, | |||
cmd.CmdDoctor, | |||
cmd.CmdManager, | |||
cmd.Cmdembedded, | |||
} | |||
// Now adjust these commands to add our global configuration options | |||
@@ -113,6 +113,37 @@ func fileFromDir(name string) ([]byte, error) { | |||
return ioutil.ReadAll(f) | |||
} | |||
func Asset(name string) ([]byte, error) { | |||
f, err := Assets.Open("/" + name) | |||
if err != nil { | |||
return nil, err | |||
} | |||
defer f.Close() | |||
return ioutil.ReadAll(f) | |||
} | |||
func AssetNames() []string { | |||
realFS := Assets.(vfsgen۰FS) | |||
var results = make([]string, 0, len(realFS)) | |||
for k := range realFS { | |||
results = append(results, k[1:]) | |||
} | |||
return results | |||
} | |||
func AssetIsDir(name string) (bool, error) { | |||
if f, err := Assets.Open("/" + name); err != nil { | |||
return false, err | |||
} else { | |||
defer f.Close() | |||
if fi, err := f.Stat(); err != nil { | |||
return false, err | |||
} else { | |||
return fi.IsDir(), nil | |||
} | |||
} | |||
} | |||
// IsDynamic will return false when using embedded data (-tags bindata) | |||
func IsDynamic() bool { | |||
return false |
@@ -7,6 +7,8 @@ | |||
package public | |||
import ( | |||
"io/ioutil" | |||
"gitea.com/macaron/macaron" | |||
) | |||
@@ -17,3 +19,34 @@ func Static(opts *Options) macaron.Handler { | |||
// used when in the options there is no FileSystem. | |||
return opts.staticHandler("") | |||
} | |||
func Asset(name string) ([]byte, error) { | |||
f, err := Assets.Open("/" + name) | |||
if err != nil { | |||
return nil, err | |||
} | |||
defer f.Close() | |||
return ioutil.ReadAll(f) | |||
} | |||
func AssetNames() []string { | |||
realFS := Assets.(vfsgen۰FS) | |||
var results = make([]string, 0, len(realFS)) | |||
for k := range realFS { | |||
results = append(results, k[1:]) | |||
} | |||
return results | |||
} | |||
func AssetIsDir(name string) (bool, error) { | |||
if f, err := Assets.Open("/" + name); err != nil { | |||
return false, err | |||
} else { | |||
defer f.Close() | |||
if fi, err := f.Stat(); err != nil { | |||
return false, err | |||
} else { | |||
return fi.IsDir(), nil | |||
} | |||
} | |||
} |
@@ -229,3 +229,16 @@ func AssetNames() []string { | |||
} | |||
return results | |||
} | |||
func AssetIsDir(name string) (bool, error) { | |||
if f, err := Assets.Open("/" + name); err != nil { | |||
return false, err | |||
} else { | |||
defer f.Close() | |||
if fi, err := f.Stat(); err != nil { | |||
return false, err | |||
} else { | |||
return fi.IsDir(), nil | |||
} | |||
} | |||
} |