mirror/gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2024-11-27 12:39:29 +08:00
Code Packages Projects Releases Wiki Activity

Use a general approach to access custom/static/builtin assets (#24022)

Browse Source 
The idea is to use a Layered Asset File-system (modules/assetfs/layered.go)

For example: when there are 2 layers: "custom", "builtin", when access
to asset "my/page.tmpl", the Layered Asset File-system will first try to
use "custom" assets, if not found, then use "builtin" assets.

This approach will hugely simplify a lot of code, make them testable.

Other changes:

* Simplify the AssetsHandlerFunc code
* Simplify the `gitea embedded` sub-command code

---------

Co-authored-by: Jason Song <i@wolfogre.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
wxiaoguang 2023-04-12 18:16:45 +08:00 committed by GitHub
parent 42919ccb7c
commit 50a72e7a83
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 689 additions and 1055 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
cmd/embedded.go
View File

@ -1,8 +1,6 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build bindata
package cmd
import (
@ -10,9 +8,9 @@ import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/public"
@ -89,24 +87,20 @@ var (
},
}
sections map[string]*section
assets []asset
matchedAssetFiles []assetFile
)
type section struct {
Path string
Names func() []string
IsDir func(string) (bool, error)
Asset func(string) ([]byte, error)
}
type asset struct {
Section *section
Name string
Path string
type assetFile struct {
fs *assetfs.LayeredFS
name string
path string
}
func initEmbeddedExtractor(c *cli.Context) error {
// FIXME: there is a bug, if the user runs `gitea embedded` with a different user or root,
// The setting.Init (loadRunModeFrom) will fail and do log.Fatal
// But the console logger has been deleted, so nothing is printed, the user sees nothing and Gitea just exits.
// Silence the console logger
log.DelNamedLogger("console")
log.DelNamedLogger(log.DEFAULT)
@ -115,24 +109,14 @@ func initEmbeddedExtractor(c *cli.Context) error {
setting.InitProviderAllowEmpty()
setting.LoadCommonSettings()
pats, err := getPatterns(c.Args())
patterns, err := compileCollectPatterns(c.Args())
if err != nil {
return err
}
sections := make(map[string]*section, 3)
sections["public"] = &section{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset}
sections["options"] = &section{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset}
sections["templates"] = &section{Path: "templates", Names: templates.BuiltinAssetNames, IsDir: templates.BuiltinAssetIsDir, Asset: templates.BuiltinAsset}
for _, sec := range sections {
assets = append(assets, buildAssetList(sec, pats, c)...)
}
// Sort assets
sort.SliceStable(assets, func(i, j int) bool {
return assets[i].Path < assets[j].Path
})
collectAssetFilesByPattern(c, patterns, "options", options.BuiltinAssets())
collectAssetFilesByPattern(c, patterns, "public", public.BuiltinAssets())
collectAssetFilesByPattern(c, patterns, "templates", templates.BuiltinAssets())
return nil
}
@ -166,8 +150,8 @@ func runListDo(c *cli.Context) error {
return err
}
for _, a := range assets {
fmt.Println(a.Path)
for _, a := range matchedAssetFiles {
fmt.Println(a.path)
}
return nil
@ -178,19 +162,19 @@ func runViewDo(c *cli.Context) error {
return err
}
if len(assets) == 0 {
return fmt.Errorf("No files matched the given pattern")
} else if len(assets) > 1 {
return fmt.Errorf("Too many files matched the given pattern; try to be more specific")
if len(matchedAssetFiles) == 0 {
return fmt.Errorf("no files matched the given pattern")
} else if len(matchedAssetFiles) > 1 {
return fmt.Errorf("too many files matched the given pattern, try to be more specific")
}
data, err := assets[0].Section.Asset(assets[0].Name)
data, err := matchedAssetFiles[0].fs.ReadFile(matchedAssetFiles[0].name)
if err != nil {
return fmt.Errorf("%s: %w", assets[0].Path, err)
return fmt.Errorf("%s: %w", matchedAssetFiles[0].path, err)
}
if _, err = os.Stdout.Write(data); err != nil {
return fmt.Errorf("%s: %w", assets[0].Path, err)
return fmt.Errorf("%s: %w", matchedAssetFiles[0].path, err)
}
return nil
@ -202,7 +186,7 @@ func runExtractDo(c *cli.Context) error {
}
if len(c.Args()) == 0 {
return fmt.Errorf("A list of pattern of files to extract is mandatory (e.g. '**' for all)")
return fmt.Errorf("a list of pattern of files to extract is mandatory (e.g. '**' for all)")
}
destdir := "."
@ -227,7 +211,7 @@ func runExtractDo(c *cli.Context) error {
if err != nil {
return fmt.Errorf("%s: %s", destdir, err)
} else if !fi.IsDir() {
return fmt.Errorf("%s is not a directory.", destdir)
return fmt.Errorf("destination %q is not a directory", destdir)
}
fmt.Printf("Extracting to %s:\n", destdir)
@ -235,23 +219,23 @@ func runExtractDo(c *cli.Context) error {
overwrite := c.Bool("overwrite")
rename := c.Bool("rename")
for _, a := range assets {
for _, a := range matchedAssetFiles {
if err := extractAsset(destdir, a, overwrite, rename); err != nil {
// Non-fatal error
fmt.Fprintf(os.Stderr, "%s: %v", a.Path, err)
fmt.Fprintf(os.Stderr, "%s: %v", a.path, err)
}
}
return nil
}
func extractAsset(d string, a asset, overwrite, rename bool) error {
dest := filepath.Join(d, filepath.FromSlash(a.Path))
func extractAsset(d string, a assetFile, overwrite, rename bool) error {
dest := filepath.Join(d, filepath.FromSlash(a.path))
dir := filepath.Dir(dest)
data, err := a.Section.Asset(a.Name)
data, err := a.fs.ReadFile(a.name)
if err != nil {
return fmt.Errorf("%s: %w", a.Path, err)
return fmt.Errorf("%s: %w", a.path, err)
}
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
@ -272,7 +256,7 @@ func extractAsset(d string, a asset, overwrite, rename bool) error {
return fmt.Errorf("%s already exists, but it's not a regular file", dest)
} else if rename {
if err := util.Rename(dest, dest+".bak"); err != nil {
return fmt.Errorf("Error creating backup for %s: %w", dest, err)
return fmt.Errorf("error creating backup for %s: %w", dest, err)
}
// Attempt to respect file permissions mask (even if user:group will be set anew)
perms = fi.Mode()
@ -293,32 +277,30 @@ func extractAsset(d string, a asset, overwrite, rename bool) error {
return nil
}
func buildAssetList(sec *section, globs []glob.Glob, c *cli.Context) []asset {
results := make([]asset, 0, 64)
for _, name := range sec.Names() {
if isdir, err := sec.IsDir(name); !isdir && err == nil {
if sec.Path == "public" &&
strings.HasPrefix(name, "vendor/") &&
!c.Bool("include-vendored") {
continue
}
matchName := sec.Path + "/" + name
for _, g := range globs {
if g.Match(matchName) {
results = append(results, asset{
Section: sec,
Name: name,
Path: sec.Path + "/" + name,
})
break
}
func collectAssetFilesByPattern(c *cli.Context, globs []glob.Glob, path string, layer *assetfs.Layer) {
fs := assetfs.Layered(layer)
files, err := fs.ListAllFiles(".", true)
if err != nil {
log.Error("Error listing files in %q: %v", path, err)
return
}
for _, name := range files {
if path == "public" &&
strings.HasPrefix(name, "vendor/") &&
!c.Bool("include-vendored") {
continue
}
matchName := path + "/" + name
for _, g := range globs {
if g.Match(matchName) {
matchedAssetFiles = append(matchedAssetFiles, assetFile{fs: fs, name: name, path: path + "/" + name})
break
}
}
}
return results
}
func getPatterns(args []string) ([]glob.Glob, error) {
func compileCollectPatterns(args []string) ([]glob.Glob, error) {
if len(args) == 0 {
args = []string{"**"}
}
@ -326,7 +308,7 @@ func getPatterns(args []string) ([]glob.Glob, error) {
for i := range args {
if g, err := glob.Compile(args[i], '/'); err != nil {
return nil, fmt.Errorf("'%s': Invalid glob pattern: %w", args[i], err)
} else {
} else { //nolint:revive
pat[i] = g
}
}

29
cmd/embedded_stub.go
View File

@ -1,29 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !bindata
package cmd
import (
"fmt"
"os"
"github.com/urfave/cli"
)
// Cmdembedded represents the available extract sub-command.
var (
Cmdembedded = cli.Command{
Name: "embedded",
Usage: "Extract embedded resources",
Description: "A command for extracting embedded resources, like templates and images",
Action: extractorNotImplemented,
}
)
func extractorNotImplemented(c *cli.Context) error {
err := fmt.Errorf("Sorry: the 'embedded' subcommand is not available in builds without bindata")
fmt.Fprintf(os.Stderr, "%s\n", err)
return err
}

260
modules/assetfs/layered.go Normal file
View File

@ -0,0 +1,260 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package assetfs
import (
"context"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"sort"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/util"
"github.com/fsnotify/fsnotify"
)
// Layer represents a layer in a layered asset file-system. It has a name and works like http.FileSystem
type Layer struct {
name string
fs http.FileSystem
localPath string
}
func (l *Layer) Name() string {
return l.name
}
// Open opens the named file. The caller is responsible for closing the file.
func (l *Layer) Open(name string) (http.File, error) {
return l.fs.Open(name)
}
// Local returns a new Layer with the given name, it serves files from the given local path.
func Local(name, base string, sub ...string) *Layer {
// TODO: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before
// Ideally, the caller should guarantee the base is absolute, guessing a relative path based on the current working directory is unreliable.
base, err := filepath.Abs(base)
if err != nil {
// This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths.
panic(fmt.Sprintf("Unable to get absolute path for %q: %v", base, err))
}
root := util.FilePathJoinAbs(base, sub...)
return &Layer{name: name, fs: http.Dir(root), localPath: root}
}
// Bindata returns a new Layer with the given name, it serves files from the given bindata asset.
func Bindata(name string, fs http.FileSystem) *Layer {
return &Layer{name: name, fs: fs}
}
// LayeredFS is a layered asset file-system. It works like http.FileSystem, but it can have multiple layers.
// The first layer is the top layer, and it will be used first.
// If the file is not found in the top layer, it will be searched in the next layer.
type LayeredFS struct {
layers []*Layer
}
// Layered returns a new LayeredFS with the given layers. The first layer is the top layer.
func Layered(layers ...*Layer) *LayeredFS {
return &LayeredFS{layers: layers}
}
// Open opens the named file. The caller is responsible for closing the file.
func (l *LayeredFS) Open(name string) (http.File, error) {
for _, layer := range l.layers {
f, err := layer.Open(name)
if err == nil || !os.IsNotExist(err) {
return f, err
}
}
return nil, fs.ErrNotExist
}
// ReadFile reads the named file.
func (l *LayeredFS) ReadFile(elems ...string) ([]byte, error) {
bs, _, err := l.ReadLayeredFile(elems...)
return bs, err
}
// ReadLayeredFile reads the named file, and returns the layer name.
func (l *LayeredFS) ReadLayeredFile(elems ...string) ([]byte, string, error) {
name := util.PathJoinRel(elems...)
for _, layer := range l.layers {
f, err := layer.Open(name)
if os.IsNotExist(err) {
continue
} else if err != nil {
return nil, layer.name, err
}
bs, err := io.ReadAll(f)
_ = f.Close()
return bs, layer.name, err
}
return nil, "", fs.ErrNotExist
}
func shouldInclude(info fs.FileInfo, fileMode ...bool) bool {
if util.CommonSkip(info.Name()) {
return false
}
if len(fileMode) == 0 {
return true
} else if len(fileMode) == 1 {
return fileMode[0] == !info.Mode().IsDir()
}
panic("too many arguments for fileMode in shouldInclude")
}
func readDir(layer *Layer, name string) ([]fs.FileInfo, error) {
f, err := layer.Open(name)
if os.IsNotExist(err) {
return nil, nil
} else if err != nil {
return nil, err
}
defer f.Close()
return f.Readdir(-1)
}
// ListFiles lists files/directories in the given directory. The fileMode controls the returned files.
// * omitted: all files and directories will be returned.
// * true: only files will be returned.
// * false: only directories will be returned.
// The returned files are sorted by name.
func (l *LayeredFS) ListFiles(name string, fileMode ...bool) ([]string, error) {
fileMap := map[string]bool{}
for _, layer := range l.layers {
infos, err := readDir(layer, name)
if err != nil {
return nil, err
}
for _, info := range infos {
if shouldInclude(info, fileMode...) {
fileMap[info.Name()] = true
}
}
}
files := make([]string, 0, len(fileMap))
for file := range fileMap {
files = append(files, file)
}
sort.Strings(files)
return files, nil
}
// ListAllFiles returns files/directories in the given directory, including subdirectories, recursively.
// The fileMode controls the returned files:
// * omitted: all files and directories will be returned.
// * true: only files will be returned.
// * false: only directories will be returned.
// The returned files are sorted by name.
func (l *LayeredFS) ListAllFiles(name string, fileMode ...bool) ([]string, error) {
return listAllFiles(l.layers, name, fileMode...)
}
func listAllFiles(layers []*Layer, name string, fileMode ...bool) ([]string, error) {
fileMap := map[string]bool{}
var list func(dir string) error
list = func(dir string) error {
for _, layer := range layers {
infos, err := readDir(layer, dir)
if err != nil {
return err
}
for _, info := range infos {
path := util.PathJoinRelX(dir, info.Name())
if shouldInclude(info, fileMode...) {
fileMap[path] = true
}
if info.IsDir() {
if err = list(path); err != nil {
return err
}
}
}
}
return nil
}
if err := list(name); err != nil {
return nil, err
}
var files []string
for file := range fileMap {
files = append(files, file)
}
sort.Strings(files)
return files, nil
}
// WatchLocalChanges watches local changes in the file-system. It's used to help to reload assets when the local file-system changes.
func (l *LayeredFS) WatchLocalChanges(ctx context.Context, callback func()) {
ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Asset Local FileSystem Watcher", process.SystemProcessType, true)
defer finished()
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Error("Unable to create watcher for asset local file-system: %v", err)
return
}
defer watcher.Close()
for _, layer := range l.layers {
if layer.localPath == "" {
continue
}
layerDirs, err := listAllFiles([]*Layer{layer}, ".", false)
if err != nil {
log.Error("Unable to list directories for asset local file-system %q: %v", layer.localPath, err)
continue
}
for _, dir := range layerDirs {
if err = watcher.Add(util.FilePathJoinAbs(layer.localPath, dir)); err != nil {
log.Error("Unable to watch directory %s: %v", dir, err)
}
}
}
debounce := util.Debounce(100 * time.Millisecond)
for {
select {
case <-ctx.Done():
return
case event, ok := <-watcher.Events:
if !ok {
return
}
log.Trace("Watched asset local file-system had event: %v", event)
debounce(callback)
case err, ok := <-watcher.Errors:
if !ok {
return
}
log.Error("Watched asset local file-system had error: %v", err)
}
}
}
// GetFileLayerName returns the name of the first-seen layer that contains the given file.
func (l *LayeredFS) GetFileLayerName(elems ...string) string {
name := util.PathJoinRel(elems...)
for _, layer := range l.layers {
f, err := layer.Open(name)
if os.IsNotExist(err) {
continue
} else if err != nil {
return ""
}
_ = f.Close()
return layer.name
}
return ""
}

109
modules/assetfs/layered_test.go Normal file
View File

@ -0,0 +1,109 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package assetfs
import (
"io"
"io/fs"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLayered(t *testing.T) {
dir := filepath.Join(t.TempDir(), "assetfs-layers")
dir1 := filepath.Join(dir, "l1")
dir2 := filepath.Join(dir, "l2")
mkdir := func(elems ...string) {
assert.NoError(t, os.MkdirAll(filepath.Join(elems...), 0o755))
}
write := func(content string, elems ...string) {
assert.NoError(t, os.WriteFile(filepath.Join(elems...), []byte(content), 0o644))
}
// d1 & f1: only in "l1"; d2 & f2: only in "l2"
// da & fa: in both "l1" and "l2"
mkdir(dir1, "d1")
mkdir(dir1, "da")
mkdir(dir1, "da/sub1")
mkdir(dir2, "d2")
mkdir(dir2, "da")
mkdir(dir2, "da/sub2")
write("dummy", dir1, ".DS_Store")
write("f1", dir1, "f1")
write("fa-1", dir1, "fa")
write("d1-f", dir1, "d1/f")
write("da-f-1", dir1, "da/f")
write("f2", dir2, "f2")
write("fa-2", dir2, "fa")
write("d2-f", dir2, "d2/f")
write("da-f-2", dir2, "da/f")
assets := Layered(Local("l1", dir1), Local("l2", dir2))
f, err := assets.Open("f1")
assert.NoError(t, err)
bs, err := io.ReadAll(f)
assert.NoError(t, err)
assert.EqualValues(t, "f1", string(bs))
_ = f.Close()
assertRead := func(expected string, expectedErr error, elems ...string) {
bs, err := assets.ReadFile(elems...)
if err != nil {
assert.ErrorAs(t, err, &expectedErr)
} else {
assert.NoError(t, err)
assert.Equal(t, expected, string(bs))
}
}
assertRead("f1", nil, "f1")
assertRead("f2", nil, "f2")
assertRead("fa-1", nil, "fa")
assertRead("d1-f", nil, "d1/f")
assertRead("d2-f", nil, "d2/f")
assertRead("da-f-1", nil, "da/f")
assertRead("", fs.ErrNotExist, "no-such")
files, err := assets.ListFiles(".", true)
assert.NoError(t, err)
assert.EqualValues(t, []string{"f1", "f2", "fa"}, files)
files, err = assets.ListFiles(".", false)
assert.NoError(t, err)
assert.EqualValues(t, []string{"d1", "d2", "da"}, files)
files, err = assets.ListFiles(".")
assert.NoError(t, err)
assert.EqualValues(t, []string{"d1", "d2", "da", "f1", "f2", "fa"}, files)
files, err = assets.ListAllFiles(".", true)
assert.NoError(t, err)
assert.EqualValues(t, []string{"d1/f", "d2/f", "da/f", "f1", "f2", "fa"}, files)
files, err = assets.ListAllFiles(".", false)
assert.NoError(t, err)
assert.EqualValues(t, []string{"d1", "d2", "da", "da/sub1", "da/sub2"}, files)
files, err = assets.ListAllFiles(".")
assert.NoError(t, err)
assert.EqualValues(t, []string{
"d1", "d1/f",
"d2", "d2/f",
"da", "da/f", "da/sub1", "da/sub2",
"f1", "f2", "fa",
}, files)
assert.Empty(t, assets.GetFileLayerName("no-such"))
assert.EqualValues(t, "l1", assets.GetFileLayerName("f1"))
assert.EqualValues(t, "l2", assets.GetFileLayerName("f2"))
}

6
modules/auth/pam/pam_stub.go
View File

@ -14,5 +14,9 @@ var Supported = false
// Auth not supported lack of pam tag
func Auth(serviceName, userName, passwd string) (string, error) {
return "", errors.New("PAM not supported")
// bypass the lint on callers: SA4023: this comparison is always true (staticcheck)
if !Supported {
return "", errors.New("PAM not supported")
}
return "", nil
}

12
modules/context/context.go
View File

@ -240,19 +240,15 @@ func (ctx *Context) HTML(status int, name base.TplName) {
}
line, _ := strconv.Atoi(lineStr) // Cannot error out as groups[2] is [1-9][0-9]*
pos, _ := strconv.Atoi(posStr) // Cannot error out as groups[3] is [1-9][0-9]*
filename, filenameErr := templates.GetAssetFilename("templates/" + errorTemplateName + ".tmpl")
if filenameErr != nil {
filename = "(template) " + errorTemplateName
}
assetLayerName := templates.AssetFS().GetFileLayerName(errorTemplateName + ".tmpl")
filename := fmt.Sprintf("(%s) %s", assetLayerName, errorTemplateName)
if errorTemplateName != string(name) {
filename += " (subtemplate of " + string(name) + ")"
}
err = fmt.Errorf("failed to render %s, error: %w:\n%s", filename, err, templates.GetLineFromTemplate(errorTemplateName, line, target, pos))
} else {
filename, filenameErr := templates.GetAssetFilename("templates/" + execErr.Name + ".tmpl")
if filenameErr != nil {
filename = "(template) " + execErr.Name
}
assetLayerName := templates.AssetFS().GetFileLayerName(execErr.Name + ".tmpl")
filename := fmt.Sprintf("(%s) %s", assetLayerName, execErr.Name)
if execErr.Name != string(name) {
filename += " (subtemplate of " + string(name) + ")"
}

3
modules/doctor/paths.go
View File

@ -9,7 +9,6 @@ import (
"os"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/setting"
)
@ -79,7 +78,7 @@ func checkConfigurationFiles(ctx context.Context, logger log.Logger, autofix boo
{"Log Root Path", setting.Log.RootPath, true, true, true},
}
if options.IsDynamic() {
if !setting.HasBuiltinBindata {
configurationFiles = append(configurationFiles, configurationFile{"Static File Root Path", setting.StaticRootPath, true, true, false})
}

118
modules/options/base.go
View File

@ -4,131 +4,39 @@
package options
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
)
var directories = make(directorySet)
func CustomAssets() *assetfs.Layer {
return assetfs.Local("custom", setting.CustomPath, "options")
}
func AssetFS() *assetfs.LayeredFS {
return assetfs.Layered(CustomAssets(), BuiltinAssets())
}
// Locale reads the content of a specific locale from static/bindata or custom path.
func Locale(name string) ([]byte, error) {
return fileFromOptionsDir("locale", name)
return AssetFS().ReadFile("locale", name)
}
// Readme reads the content of a specific readme from static/bindata or custom path.
func Readme(name string) ([]byte, error) {
return fileFromOptionsDir("readme", name)
return AssetFS().ReadFile("readme", name)
}
// Gitignore reads the content of a gitignore locale from static/bindata or custom path.
func Gitignore(name string) ([]byte, error) {
return fileFromOptionsDir("gitignore", name)
return AssetFS().ReadFile("gitignore", name)
}
// License reads the content of a specific license from static/bindata or custom path.
func License(name string) ([]byte, error) {
return fileFromOptionsDir("license", name)
return AssetFS().ReadFile("license", name)
}
// Labels reads the content of a specific labels from static/bindata or custom path.
func Labels(name string) ([]byte, error) {
return fileFromOptionsDir("label", name)
}
// WalkLocales reads the content of a specific locale
func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error {
if IsDynamic() {
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to walk locales. Error: %w", err)
}
}
if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to walk locales. Error: %w", err)
}
return nil
}
func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
// name is the path relative to the root
name := path[len(root):]
if len(name) > 0 && name[0] == '/' {
name = name[1:]
}
if err != nil {
if os.IsNotExist(err) {
return callback(path, name, d, err)
}
return err
}
if util.CommonSkip(d.Name()) {
if d.IsDir() {
return fs.SkipDir
}
return nil
}
return callback(path, name, d, err)
}); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("unable to get files for assets in %s: %w", root, err)
}
return nil
}
// mustLocalPathAbs coverts a path to absolute path
// FIXME: the old behavior (StaticRootPath might not be absolute), not ideal, just keep the same as before
func mustLocalPathAbs(s string) string {
abs, err := filepath.Abs(s)
if err != nil {
// This should never happen in a real system. If it happens, the user must have already been in trouble: the system is not able to resolve its own paths.
log.Fatal("Unable to get absolute path for %q: %v", s, err)
}
return abs
}
func joinLocalPaths(baseDirs []string, subDir string, elems ...string) (paths []string) {
abs := make([]string, len(elems)+2)
abs[1] = subDir
copy(abs[2:], elems)
for _, baseDir := range baseDirs {
abs[0] = mustLocalPathAbs(baseDir)
paths = append(paths, util.FilePathJoinAbs(abs...))
}
return paths
}
func listLocalDirIfExist(baseDirs []string, subDir string, elems ...string) (files []string, err error) {
for _, localPath := range joinLocalPaths(baseDirs, subDir, elems...) {
isDir, err := util.IsDir(localPath)
if err != nil {
return nil, fmt.Errorf("unable to check if path %q is a directory. %w", localPath, err)
} else if !isDir {
continue
}
dirFiles, err := util.StatDir(localPath, true)
if err != nil {
return nil, fmt.Errorf("unable to read directory %q. %w", localPath, err)
}
files = append(files, dirFiles...)
}
return files, nil
}
func readLocalFile(baseDirs []string, subDir string, elems ...string) ([]byte, error) {
for _, localPath := range joinLocalPaths(baseDirs, subDir, elems...) {
data, err := os.ReadFile(localPath)
if err == nil {
return data, nil
} else if !os.IsNotExist(err) {
log.Error("Unable to read file %q. Error: %v", localPath, err)
}
}
return nil, os.ErrNotExist
return AssetFS().ReadFile("label", name)
}

25
modules/options/dynamic.go
View File

@ -6,29 +6,10 @@
package options
import (
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/setting"
)
// Dir returns all files from static or custom directory.
func Dir(name string) ([]string, error) {
if directories.Filled(name) {
return directories.Get(name), nil
}
result, err := listLocalDirIfExist([]string{setting.CustomPath, setting.StaticRootPath}, "options", name)
if err != nil {
return nil, err
}
return directories.AddAndGet(name, result), nil
}
// fileFromOptionsDir is a helper to read files from custom or static path.
func fileFromOptionsDir(elems ...string) ([]byte, error) {
return readLocalFile([]string{setting.CustomPath, setting.StaticRootPath}, "options", elems...)
}
// IsDynamic will return false when using embedded data (-tags bindata)
func IsDynamic() bool {
return true
func BuiltinAssets() *assetfs.Layer {
return assetfs.Local("builtin(static)", setting.StaticRootPath, "options")
}

45
modules/options/options.go
View File

@ -1,45 +0,0 @@
// Copyright 2016 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package options
type directorySet map[string][]string
func (s directorySet) Add(key string, value []string) {
_, ok := s[key]
if !ok {
s[key] = make([]string, 0, len(value))
}
s[key] = append(s[key], value...)
}
func (s directorySet) Get(key string) []string {
_, ok := s[key]
if ok {
result := []string{}
seen := map[string]string{}
for _, val := range s[key] {
if _, ok := seen[val]; !ok {
result = append(result, val)
seen[val] = val
}
}
return result
}
return []string{}
}
func (s directorySet) AddAndGet(key string, value []string) []string {
s.Add(key, value)
return s.Get(key)
}
func (s directorySet) Filled(key string) bool {
return len(s[key]) > 0
}

95
modules/options/static.go
View File

@ -6,98 +6,9 @@
package options
import (
"fmt"
"io"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/assetfs"
)
// Dir returns all files from custom directory or bindata.
func Dir(name string) ([]string, error) {
if directories.Filled(name) {
return directories.Get(name), nil
}
result, err := listLocalDirIfExist([]string{setting.CustomPath}, "options", name)
if err != nil {
return nil, err
}
files, err := AssetDir(name)
if err != nil {
return []string{}, fmt.Errorf("unable to read embedded directory %q. %w", name, err)
}
result = append(result, files...)
return directories.AddAndGet(name, result), nil
}
func AssetDir(dirName string) ([]string, error) {
d, err := Assets.Open(dirName)
if err != nil {
return nil, err
}
defer d.Close()
files, err := d.Readdir(-1)
if err != nil {
return nil, err
}
results := make([]string, 0, len(files))
for _, file := range files {
results = append(results, file.Name())
}
return results, nil
}
// fileFromOptionsDir is a helper to read files from custom path or bindata.
func fileFromOptionsDir(elems ...string) ([]byte, error) {
// only try custom dir, no static dir
if data, err := readLocalFile([]string{setting.CustomPath}, "options", elems...); err == nil {
return data, nil
}
f, err := Assets.Open(util.PathJoinRelX(elems...))
if err != nil {
return nil, err
}
defer f.Close()
return io.ReadAll(f)
}
func Asset(name string) ([]byte, error) {
f, err := Assets.Open("/" + name)
if err != nil {
return nil, err
}
defer f.Close()
return io.ReadAll(f)
}
func AssetNames() []string {
realFS := Assets.(vfsgen۰FS)
results := make([]string, 0, len(realFS))
for k := range realFS {
results = append(results, k[1:])
}
return results
}
func AssetIsDir(name string) (bool, error) {
if f, err := Assets.Open("/" + name); err != nil {
return false, err
} else {
defer f.Close()
if fi, err := f.Stat(); err != nil {
return false, err
} else {
return fi.IsDir(), nil
}
}
}
// IsDynamic will return false when using embedded data (-tags bindata)
func IsDynamic() bool {
return false
func BuiltinAssets() *assetfs.Layer {
return assetfs.Bindata("builtin(bindata)", Assets)
}

94
modules/public/public.go
View File

@ -4,11 +4,15 @@
package public
import (
"bytes"
"io"
"net/http"
"os"
"path/filepath"
"path"
"strings"
"time"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/log"
@ -16,55 +20,31 @@ import (
"code.gitea.io/gitea/modules/util"
)
// Options represents the available options to configure the handler.
type Options struct {
Directory string
Prefix string
CorsHandler func(http.Handler) http.Handler
func CustomAssets() *assetfs.Layer {
return assetfs.Local("custom", setting.CustomPath, "public")
}
// AssetsURLPathPrefix is the path prefix for static asset files
const AssetsURLPathPrefix = "/assets/"
func AssetFS() *assetfs.LayeredFS {
return assetfs.Layered(CustomAssets(), BuiltinAssets())
}
// AssetsHandlerFunc implements the static handler for serving custom or original assets.
func AssetsHandlerFunc(opts *Options) http.HandlerFunc {
custPath := filepath.Join(setting.CustomPath, "public")
if !filepath.IsAbs(custPath) {
custPath = filepath.Join(setting.AppWorkPath, custPath)
}
if !filepath.IsAbs(opts.Directory) {
opts.Directory = filepath.Join(setting.AppWorkPath, opts.Directory)
}
if !strings.HasSuffix(opts.Prefix, "/") {
opts.Prefix += "/"
}
func AssetsHandlerFunc(prefix string) http.HandlerFunc {
assetFS := AssetFS()
prefix = strings.TrimSuffix(prefix, "/") + "/"
return func(resp http.ResponseWriter, req *http.Request) {
subPath := req.URL.Path
if !strings.HasPrefix(subPath, prefix) {
return
}
subPath = strings.TrimPrefix(subPath, prefix)
if req.Method != "GET" && req.Method != "HEAD" {
resp.WriteHeader(http.StatusNotFound)
return
}
if opts.CorsHandler != nil {
var corsSent bool
opts.CorsHandler(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
corsSent = true
})).ServeHTTP(resp, req)
// If CORS is not sent, the response must have been written by other handlers
if !corsSent {
return
}
}
file := req.URL.Path[len(opts.Prefix):]
// custom files
if opts.handle(resp, req, http.Dir(custPath), file) {
return
}
// internal files
if opts.handle(resp, req, fileSystem(opts.Directory), file) {
if handleRequest(resp, req, assetFS, subPath) {
return
}
@ -85,13 +65,13 @@ func parseAcceptEncoding(val string) container.Set[string] {
// setWellKnownContentType will set the Content-Type if the file is a well-known type.
// See the comments of detectWellKnownMimeType
func setWellKnownContentType(w http.ResponseWriter, file string) {
mimeType := detectWellKnownMimeType(filepath.Ext(file))
mimeType := detectWellKnownMimeType(path.Ext(file))
if mimeType != "" {
w.Header().Set("Content-Type", mimeType)
}
}
func (opts *Options) handle(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool {
func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool {
// actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here
f, err := fs.Open(util.PathJoinRelX(file))
if err != nil {
@ -121,8 +101,34 @@ func (opts *Options) handle(w http.ResponseWriter, req *http.Request, fs http.Fi
return true
}
setWellKnownContentType(w, file)
serveContent(w, req, fi, fi.ModTime(), f)
return true
}
type GzipBytesProvider interface {
GzipBytes() []byte
}
// serveContent serve http content
func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) {
setWellKnownContentType(w, fi.Name())
encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding"))
if encodings.Contains("gzip") {
// try to provide gzip content directly from bindata (provided by vfsgen۰CompressedFileInfo)
if compressed, ok := fi.(GzipBytesProvider); ok {
rdGzip := bytes.NewReader(compressed.GzipBytes())
// all gzipped static files (from bindata) are managed by Gitea, so we can make sure every file has the correct ext name
// then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data
if w.Header().Get("Content-Type") == "" {
w.Header().Set("Content-Type", "application/octet-stream")
}
w.Header().Set("Content-Encoding", "gzip")
http.ServeContent(w, req, fi.Name(), modtime, rdGzip)
return
}
}
http.ServeContent(w, req, fi.Name(), modtime, content)
return
}

15
modules/public/serve_dynamic.go
View File

@ -6,17 +6,10 @@
package public
import (
"io"
"net/http"
"os"
"time"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/setting"
)
func fileSystem(dir string) http.FileSystem {
return http.Dir(dir)
}
// serveContent serve http content
func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) {
http.ServeContent(w, req, fi.Name(), modtime, content)
func BuiltinAssets() *assetfs.Layer {
return assetfs.Local("builtin(static)", setting.StaticRootPath, "public")
}

66
modules/public/serve_static.go
View File

@ -6,75 +6,19 @@
package public
import (
"bytes"
"io"
"net/http"
"os"
"path/filepath"
"time"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/timeutil"
)
var _ GzipBytesProvider = (*vfsgen۰CompressedFileInfo)(nil)
// GlobalModTime provide a global mod time for embedded asset files
func GlobalModTime(filename string) time.Time {
return timeutil.GetExecutableModTime()
}
func fileSystem(dir string) http.FileSystem {
return Assets
}
func Asset(name string) ([]byte, error) {
f, err := Assets.Open("/" + name)
if err != nil {
return nil, err
}
defer f.Close()
return io.ReadAll(f)
}
func AssetNames() []string {
realFS := Assets.(vfsgen۰FS)
results := make([]string, 0, len(realFS))
for k := range realFS {
results = append(results, k[1:])
}
return results
}
func AssetIsDir(name string) (bool, error) {
if f, err := Assets.Open("/" + name); err != nil {
return false, err
} else {
defer f.Close()
if fi, err := f.Stat(); err != nil {
return false, err
} else {
return fi.IsDir(), nil
}
}
}
// serveContent serve http content
func serveContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) {
encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding"))
if encodings.Contains("gzip") {
if cf, ok := fi.(*vfsgen۰CompressedFileInfo); ok {
rdGzip := bytes.NewReader(cf.GzipBytes())
// all static files are managed by Gitea, so we can make sure every file has the correct ext name
// then we can get the correct Content-Type, we do not need to do http.DetectContentType on the decompressed data
mimeType := detectWellKnownMimeType(filepath.Ext(fi.Name()))
if mimeType == "" {
mimeType = "application/octet-stream"
}
w.Header().Set("Content-Type", mimeType)
w.Header().Set("Content-Encoding", "gzip")
http.ServeContent(w, req, fi.Name(), modtime, rdGzip)
return
}
}
http.ServeContent(w, req, fi.Name(), modtime, content)
return
func BuiltinAssets() *assetfs.Layer {
return assetfs.Bindata("builtin(bindata)", Assets)
}

2
modules/repository/init.go
View File

@ -79,7 +79,7 @@ func LoadRepoConfig() error {
typeFiles := make([]optionFileList, len(types))
for i, t := range types {
var err error
if typeFiles[i].all, err = options.Dir(t); err != nil {
if typeFiles[i].all, err = options.AssetFS().ListFiles(t, true); err != nil {
return fmt.Errorf("failed to list %s files: %w", t, err)
}
sort.Strings(typeFiles[i].all)

8
modules/setting/asset_dynamic.go Normal file
View File

@ -0,0 +1,8 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !bindata
package setting
const HasBuiltinBindata = false

8
modules/setting/asset_static.go Normal file
View File

@ -0,0 +1,8 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build bindata
package setting
const HasBuiltinBindata = true

30
modules/svg/discover_bindata.go
View File

@ -1,30 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build bindata
package svg
import (
"path/filepath"
"code.gitea.io/gitea/modules/public"
)
// Discover returns a map of discovered SVG icons in bindata
func Discover() map[string]string {
svgs := make(map[string]string)
for _, file := range public.AssetNames() {
matched, _ := filepath.Match("img/svg/*.svg", file)
if matched {
content, err := public.Asset(file)
if err == nil {
filename := filepath.Base(file)
svgs[filename[:len(filename)-4]] = string(content)
}
}
}
return svgs
}

29
modules/svg/discover_nobindata.go
View File

@ -1,29 +0,0 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !bindata
package svg
import (
"os"
"path/filepath"
"code.gitea.io/gitea/modules/setting"
)
// Discover returns a map of discovered SVG icons in the file system
func Discover() map[string]string {
svgs := make(map[string]string)
files, _ := filepath.Glob(filepath.Join(setting.StaticRootPath, "public", "img", "svg", "*.svg"))
for _, file := range files {
content, err := os.ReadFile(file)
if err == nil {
filename := filepath.Base(file)
svgs[filename[:len(filename)-4]] = string(content)
}
}
return svgs
}

29
modules/svg/svg.go
View File

@ -6,15 +6,18 @@ package svg
import (
"fmt"
"html/template"
"path"
"regexp"
"strings"
"code.gitea.io/gitea/modules/html"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/public"
)
var (
// SVGs contains discovered SVGs
SVGs map[string]string
SVGs = map[string]string{}
widthRe = regexp.MustCompile(`width="[0-9]+?"`)
heightRe = regexp.MustCompile(`height="[0-9]+?"`)
@ -23,17 +26,29 @@ var (
const defaultSize = 16
// Init discovers SVGs and populates the `SVGs` variable
func Init() {
SVGs = Discover()
func Init() error {
files, err := public.AssetFS().ListFiles("img/svg")
if err != nil {
return err
}
// Remove `xmlns` because inline SVG does not need it
r := regexp.MustCompile(`(<svg\b[^>]*?)\s+xmlns="[^"]*"`)
for name, svg := range SVGs {
SVGs[name] = r.ReplaceAllString(svg, "$1")
reXmlns := regexp.MustCompile(`(<svg\b[^>]*?)\s+xmlns="[^"]*"`)
for _, file := range files {
if path.Ext(file) != ".svg" {
continue
}
bs, err := public.AssetFS().ReadFile("img/svg", file)
if err != nil {
log.Error("Failed to read SVG file %s: %v", file, err)
} else {
SVGs[file[:len(file)-4]] = reXmlns.ReplaceAllString(string(bs), "$1")
}
}
return nil
}
// Render render icons - arguments icon name (string), size (int), class (string)
// RenderHTML renders icons - arguments icon name (string), size (int), class (string)
func RenderHTML(icon string, others ...interface{}) template.HTML {
size, class := html.ParseSizeAndClass(defaultSize, "", others...)

93
modules/templates/base.go
View File

@ -4,14 +4,10 @@
package templates
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
)
@ -47,81 +43,30 @@ func BaseVars() Vars {
}
}
func getDirTemplateAssetNames(dir string) []string {
return getDirAssetNames(dir, false)
func AssetFS() *assetfs.LayeredFS {
return assetfs.Layered(CustomAssets(), BuiltinAssets())
}
func getDirAssetNames(dir string, mailer bool) []string {
var tmpls []string
func CustomAssets() *assetfs.Layer {
return assetfs.Local("custom", setting.CustomPath, "templates")
}
if mailer {
dir += filepath.Join(dir, "mail")
}
f, err := os.Stat(dir)
func ListWebTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) {
files, err := assets.ListAllFiles(".", true)
if err != nil {
if os.IsNotExist(err) {
return tmpls
}
log.Warn("Unable to check if templates dir %s is a directory. Error: %v", dir, err)
return tmpls
}
if !f.IsDir() {
log.Warn("Templates dir %s is a not directory.", dir)
return tmpls
return nil, err
}
return util.SliceRemoveAllFunc(files, func(file string) bool {
return strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl")
}), nil
}
files, err := util.StatDir(dir)
func ListMailTemplateAssetNames(assets *assetfs.LayeredFS) ([]string, error) {
files, err := assets.ListAllFiles(".", true)
if err != nil {
log.Warn("Failed to read %s templates dir. %v", dir, err)
return tmpls
return nil, err
}
prefix := "templates/"
if mailer {
prefix += "mail/"
}
for _, filePath := range files {
if !mailer && strings.HasPrefix(filePath, "mail/") {
continue
}
if !strings.HasSuffix(filePath, ".tmpl") {
continue
}
tmpls = append(tmpls, prefix+filePath)
}
return tmpls
}
func walkAssetDir(root string, skipMail bool, callback func(path, name string, d fs.DirEntry, err error) error) error {
mailRoot := filepath.Join(root, "mail")
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
name := path[len(root):]
if len(name) > 0 && name[0] == '/' {
name = name[1:]
}
if err != nil {
if os.IsNotExist(err) {
return callback(path, name, d, err)
}
return err
}
if skipMail && path == mailRoot && d.IsDir() {
return fs.SkipDir
}
if util.CommonSkip(d.Name()) {
if d.IsDir() {
return fs.SkipDir
}
return nil
}
if strings.HasSuffix(d.Name(), ".tmpl") || d.IsDir() {
return callback(path, name, d, err)
}
return nil
}); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("unable to get files for template assets in %s: %w", root, err)
}
return nil
return util.SliceRemoveAllFunc(files, func(file string) bool {
return !strings.HasPrefix(file, "mail/") || !strings.HasSuffix(file, ".tmpl")
}), nil
}

72
modules/templates/dynamic.go
View File

@ -6,76 +6,10 @@
package templates
import (
"io/fs"
"os"
"path/filepath"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/setting"
)
// GetAsset returns asset content via name
func GetAsset(name string) ([]byte, error) {
bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name))
if err != nil && !os.IsNotExist(err) {
return nil, err
} else if err == nil {
return bs, nil
}
return os.ReadFile(filepath.Join(setting.StaticRootPath, name))
}
// GetAssetFilename returns the filename of the provided asset
func GetAssetFilename(name string) (string, error) {
filename := filepath.Join(setting.CustomPath, name)
_, err := os.Stat(filename)
if err != nil && !os.IsNotExist(err) {
return filename, err
} else if err == nil {
return filename, nil
}
filename = filepath.Join(setting.StaticRootPath, name)
_, err = os.Stat(filename)
return filename, err
}
// walkTemplateFiles calls a callback for each template asset
func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
return err
}
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// GetTemplateAssetNames returns list of template names
func GetTemplateAssetNames() []string {
tmpls := getDirTemplateAssetNames(filepath.Join(setting.CustomPath, "templates"))
tmpls2 := getDirTemplateAssetNames(filepath.Join(setting.StaticRootPath, "templates"))
return append(tmpls, tmpls2...)
}
func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
return err
}
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// BuiltinAsset will read the provided asset from the embedded assets
// (This always returns os.ErrNotExist)
func BuiltinAsset(name string) ([]byte, error) {
return nil, os.ErrNotExist
}
// BuiltinAssetNames returns the names of the embedded assets
// (This always returns nil)
func BuiltinAssetNames() []string {
return nil
func BuiltinAssets() *assetfs.Layer {
return assetfs.Local("builtin(static)", setting.StaticRootPath, "templates")
}

62
modules/templates/htmlrenderer.go
View File

@ -21,7 +21,6 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/watcher"
)
var (
@ -66,20 +65,23 @@ func (h *HTMLRender) TemplateLookup(name string) (*template.Template, error) {
}
func (h *HTMLRender) CompileTemplates() error {
dirPrefix := "templates/"
extSuffix := ".tmpl"
tmpls := template.New("")
for _, path := range GetTemplateAssetNames() {
if !strings.HasSuffix(path, extSuffix) {
assets := AssetFS()
files, err := ListWebTemplateAssetNames(assets)
if err != nil {
return nil
}
for _, file := range files {
if !strings.HasSuffix(file, extSuffix) {
continue
}
name := strings.TrimPrefix(path, dirPrefix)
name = strings.TrimSuffix(name, extSuffix)
name := strings.TrimSuffix(file, extSuffix)
tmpl := tmpls.New(filepath.ToSlash(name))
for _, fm := range NewFuncMap() {
tmpl.Funcs(fm)
}
buf, err := GetAsset(path)
buf, err := assets.ReadFile(file)
if err != nil {
return err
}
@ -112,13 +114,10 @@ func HTMLRenderer(ctx context.Context) (context.Context, *HTMLRender) {
log.Fatal("HTMLRenderer error: %v", err)
}
if !setting.IsProd {
watcher.CreateWatcher(ctx, "HTML Templates", &watcher.CreateWatcherOpts{
PathsCallback: walkTemplateFiles,
BetweenCallback: func() {
if err := renderer.CompileTemplates(); err != nil {
log.Error("Template error: %v\n%s", err, log.Stack(2))
}
},
go AssetFS().WatchLocalChanges(ctx, func() {
if err := renderer.CompileTemplates(); err != nil {
log.Error("Template error: %v\n%s", err, log.Stack(2))
}
})
}
return context.WithValue(ctx, rendererKey, renderer), renderer
@ -138,14 +137,8 @@ func handleGenericTemplateError(err error) (string, []interface{}) {
}
templateName, lineNumberStr, message := groups[1], groups[2], groups[3]
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
if assetErr != nil {
return "", nil
}
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
lineNumber, _ := strconv.Atoi(lineNumberStr)
line := GetLineFromTemplate(templateName, lineNumber, "", -1)
return "PANIC: Unable to compile templates!\n%s in template file %s at line %d:\n\n%s\nStacktrace:\n\n%s", []interface{}{message, filename, lineNumber, log.NewColoredValue(line, log.Reset), log.Stack(2)}
@ -158,16 +151,9 @@ func handleNotDefinedPanicError(err error) (string, []interface{}) {
}
templateName, lineNumberStr, functionName := groups[1], groups[2], groups[3]
functionName, _ = strconv.Unquote(`"` + functionName + `"`)
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
if assetErr != nil {
return "", nil
}
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
lineNumber, _ := strconv.Atoi(lineNumberStr)
line := GetLineFromTemplate(templateName, lineNumber, functionName, -1)
return "PANIC: Unable to compile templates!\nUndefined function %q in template file %s at line %d:\n\n%s", []interface{}{functionName, filename, lineNumber, log.NewColoredValue(line, log.Reset)}
@ -181,14 +167,8 @@ func handleUnexpected(err error) (string, []interface{}) {
templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3]
unexpected, _ = strconv.Unquote(`"` + unexpected + `"`)
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
if assetErr != nil {
return "", nil
}
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
lineNumber, _ := strconv.Atoi(lineNumberStr)
line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1)
return "PANIC: Unable to compile templates!\nUnexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)}
@ -201,14 +181,8 @@ func handleExpectedEnd(err error) (string, []interface{}) {
}
templateName, lineNumberStr, unexpected := groups[1], groups[2], groups[3]
filename, assetErr := GetAssetFilename("templates/" + templateName + ".tmpl")
if assetErr != nil {
return "", nil
}
filename := fmt.Sprintf("%s (provided by %s)", templateName, AssetFS().GetFileLayerName(templateName+".tmpl"))
lineNumber, _ := strconv.Atoi(lineNumberStr)
line := GetLineFromTemplate(templateName, lineNumber, unexpected, -1)
return "PANIC: Unable to compile templates!\nMissing end with unexpected %q in template file %s at line %d:\n\n%s", []interface{}{unexpected, filename, lineNumber, log.NewColoredValue(line, log.Reset)}
@ -218,7 +192,7 @@ const dashSeparator = "---------------------------------------------------------
// GetLineFromTemplate returns a line from a template with some context
func GetLineFromTemplate(templateName string, targetLineNum int, target string, position int) string {
bs, err := GetAsset("templates/" + templateName + ".tmpl")
bs, err := AssetFS().ReadFile(templateName + ".tmpl")
if err != nil {
return fmt.Sprintf("(unable to read template file: %v)", err)
}

63
modules/templates/mailer.go
View File

@ -6,15 +6,12 @@ package templates
import (
"context"
"html/template"
"io/fs"
"os"
"strings"
texttmpl "text/template"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/watcher"
)
// mailSubjectTextFuncMap returns functions for injecting to text templates, it's only used for mail subject
@ -62,54 +59,23 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) {
bodyTemplates.Funcs(funcs)
}
assetFS := AssetFS()
refreshTemplates := func() {
for _, assetPath := range BuiltinAssetNames() {
if !strings.HasPrefix(assetPath, "mail/") {
continue
}
if !strings.HasSuffix(assetPath, ".tmpl") {
continue
}
content, err := BuiltinAsset(assetPath)
if err != nil {
log.Warn("Failed to read embedded %s template. %v", assetPath, err)
continue
}
assetName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/")
log.Trace("Adding built-in mailer template for %s", assetName)
buildSubjectBodyTemplate(subjectTemplates,
bodyTemplates,
assetName,
content)
assetPaths, err := ListMailTemplateAssetNames(assetFS)
if err != nil {
log.Error("Failed to list mail templates: %v", err)
return
}
if err := walkMailerTemplates(func(path, name string, d fs.DirEntry, err error) error {
for _, assetPath := range assetPaths {
content, layerName, err := assetFS.ReadLayeredFile(assetPath)
if err != nil {
return err
log.Warn("Failed to read mail template %s by %s: %v", assetPath, layerName, err)
continue
}
if d.IsDir() {
return nil
}
content, err := os.ReadFile(path)
if err != nil {
log.Warn("Failed to read custom %s template. %v", path, err)
return nil
}
assetName := strings.TrimSuffix(name, ".tmpl")
log.Trace("Adding mailer template for %s from %q", assetName, path)
buildSubjectBodyTemplate(subjectTemplates,
bodyTemplates,
assetName,
content)
return nil
}); err != nil && !os.IsNotExist(err) {
log.Warn("Error whilst walking mailer templates directories. %v", err)
tmplName := strings.TrimPrefix(strings.TrimSuffix(assetPath, ".tmpl"), "mail/")
log.Trace("Adding mail template %s: %s by %s", tmplName, assetPath, layerName)
buildSubjectBodyTemplate(subjectTemplates, bodyTemplates, tmplName, content)
}
}
@ -118,10 +84,7 @@ func Mailer(ctx context.Context) (*texttmpl.Template, *template.Template) {
if !setting.IsProd {
// Now subjectTemplates and bodyTemplates are both synchronized
// thus it is safe to call refresh from a different goroutine
watcher.CreateWatcher(ctx, "Mailer Templates", &watcher.CreateWatcherOpts{
PathsCallback: walkMailerTemplates,
BetweenCallback: refreshTemplates,
})
go assetFS.WatchLocalChanges(ctx, refreshTemplates)
}
return subjectTemplates, bodyTemplates

103
modules/templates/static.go
View File

@ -6,114 +6,17 @@
package templates
import (
"html/template"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
texttmpl "text/template"
"time"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/assetfs"
"code.gitea.io/gitea/modules/timeutil"
)
var (
subjectTemplates = texttmpl.New("")
bodyTemplates = template.New("")
)
// GlobalModTime provide a global mod time for embedded asset files
func GlobalModTime(filename string) time.Time {
return timeutil.GetExecutableModTime()
}
// GetAssetFilename returns the filename of the provided asset
func GetAssetFilename(name string) (string, error) {
filename := filepath.Join(setting.CustomPath, name)
_, err := os.Stat(filename)
if err != nil && !os.IsNotExist(err) {
return name, err
} else if err == nil {
return filename, nil
}
return "(builtin) " + name, nil
}
// GetAsset get a special asset, only for chi
func GetAsset(name string) ([]byte, error) {
bs, err := os.ReadFile(filepath.Join(setting.CustomPath, name))
if err != nil && !os.IsNotExist(err) {
return nil, err
} else if err == nil {
return bs, nil
}
return BuiltinAsset(strings.TrimPrefix(name, "templates/"))
}
// GetFiles calls a callback for each template asset
func walkTemplateFiles(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates"), true, callback); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// GetTemplateAssetNames only for chi
func GetTemplateAssetNames() []string {
realFS := Assets.(vfsgen۰FS)
tmpls := make([]string, 0, len(realFS))
for k := range realFS {
if strings.HasPrefix(k, "/mail/") {
continue
}
tmpls = append(tmpls, "templates/"+k[1:])
}
customDir := path.Join(setting.CustomPath, "templates")
customTmpls := getDirTemplateAssetNames(customDir)
return append(tmpls, customTmpls...)
}
func walkMailerTemplates(callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := walkAssetDir(filepath.Join(setting.CustomPath, "templates", "mail"), false, callback); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// BuiltinAsset reads the provided asset from the builtin embedded assets
func BuiltinAsset(name string) ([]byte, error) {
f, err := Assets.Open("/" + name)
if err != nil {
return nil, err
}
defer f.Close()
return io.ReadAll(f)
}
// BuiltinAssetNames returns the names of the built-in embedded assets
func BuiltinAssetNames() []string {
realFS := Assets.(vfsgen۰FS)
results := make([]string, 0, len(realFS))
for k := range realFS {
results = append(results, k[1:])
}
return results
}
// BuiltinAssetIsDir returns if a provided asset is a directory
func BuiltinAssetIsDir(name string) (bool, error) {
if f, err := Assets.Open("/" + name); err != nil {
return false, err
} else {
defer f.Close()
if fi, err := f.Stat(); err != nil {
return false, err
} else {
return fi.IsDir(), nil
}
}
func BuiltinAssets() *assetfs.Layer {
return assetfs.Bindata("builtin(bindata)", Assets)
}

14
modules/translation/translation.go
View File

@ -13,7 +13,6 @@ import (
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/translation/i18n"
"code.gitea.io/gitea/modules/watcher"
"golang.org/x/text/language"
)
@ -58,7 +57,7 @@ func InitLocales(ctx context.Context) {
refreshLocales := func() {
i18n.ResetDefaultLocales()
localeNames, err := options.Dir("locale")
localeNames, err := options.AssetFS().ListFiles("locale", true)
if err != nil {
log.Fatal("Failed to list locale files: %v", err)
}
@ -118,13 +117,10 @@ func InitLocales(ctx context.Context) {
})
if !setting.IsProd {
watcher.CreateWatcher(ctx, "Locales", &watcher.CreateWatcherOpts{
PathsCallback: options.WalkLocales,
BetweenCallback: func() {
lock.Lock()
defer lock.Unlock()
refreshLocales()
},
go options.AssetFS().WatchLocalChanges(ctx, func() {
lock.Lock()
defer lock.Unlock()
refreshLocales()
})
}
}

19
modules/util/path.go
View File

@ -74,29 +74,28 @@ const pathSeparator = string(os.PathSeparator)
//
// {`/foo`, ``, `bar`} => `/foo/bar`
// {`/foo`, `..`, `bar`} => `/foo/bar`
func FilePathJoinAbs(elem ...string) string {
elems := make([]string, len(elem))
func FilePathJoinAbs(base string, sub ...string) string {
elems := make([]string, 1, len(sub)+1)
// POISX filesystem can have `\` in file names. Windows: `\` and `/` are both used for path separators
// POSIX filesystem can have `\` in file names. Windows: `\` and `/` are both used for path separators
// to keep the behavior consistent, we do not allow `\` in file names, replace all `\` with `/`
if isOSWindows() {
elems[0] = filepath.Clean(elem[0])
elems[0] = filepath.Clean(base)
} else {
elems[0] = filepath.Clean(strings.ReplaceAll(elem[0], "\\", pathSeparator))
elems[0] = filepath.Clean(strings.ReplaceAll(base, "\\", pathSeparator))
}
if !filepath.IsAbs(elems[0]) {
// This shouldn't happen. If there is really necessary to pass in relative path, return the full path with filepath.Abs() instead
panic(fmt.Sprintf("FilePathJoinAbs: %q (for path %v) is not absolute, do not guess a relative path based on current working directory", elems[0], elems))
}
for i := 1; i < len(elem); i++ {
if elem[i] == "" {
for _, s := range sub {
if s == "" {
continue
}
if isOSWindows() {
elems[i] = filepath.Clean(pathSeparator + elem[i])
elems = append(elems, filepath.Clean(pathSeparator+s))
} else {
elems[i] = filepath.Clean(pathSeparator + strings.ReplaceAll(elem[i], "\\", pathSeparator))
elems = append(elems, filepath.Clean(pathSeparator+strings.ReplaceAll(s, "\\", pathSeparator)))
}
}
// the elems[0] must be an absolute path, just join them together

2
modules/util/path_test.go
View File

@ -207,6 +207,6 @@ func TestCleanPath(t *testing.T) {
}
}
for _, c := range cases {
assert.Equal(t, c.expected, FilePathJoinAbs(c.elems...), "case: %v", c.elems)
assert.Equal(t, c.expected, FilePathJoinAbs(c.elems[0], c.elems[1:]...), "case: %v", c.elems)
}
}

28
modules/util/timer.go
View File

@ -4,6 +4,7 @@
package util
import (
"sync"
"time"
)
@ -18,3 +19,30 @@ func StopTimer(t *time.Timer) bool {
}
return stopped
}
func Debounce(d time.Duration) func(f func()) {
type debouncer struct {
mu sync.Mutex
t *time.Timer
}
db := &debouncer{}
return func(f func()) {
db.mu.Lock()
defer db.mu.Unlock()
if db.t != nil {
db.t.Stop()
}
var trigger *time.Timer
trigger = time.AfterFunc(d, func() {
db.mu.Lock()
defer db.mu.Unlock()
if trigger == db.t {
f()
db.t = nil
}
})
db.t = trigger
}
}

30
modules/util/timer_test.go Normal file
View File

@ -0,0 +1,30 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package util
import (
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestDebounce(t *testing.T) {
var c int64
d := Debounce(50 * time.Millisecond)
d(func() { atomic.AddInt64(&c, 1) })
assert.EqualValues(t, 0, atomic.LoadInt64(&c))
d(func() { atomic.AddInt64(&c, 1) })
d(func() { atomic.AddInt64(&c, 1) })
time.Sleep(100 * time.Millisecond)
assert.EqualValues(t, 1, atomic.LoadInt64(&c))
d(func() { atomic.AddInt64(&c, 1) })
assert.EqualValues(t, 1, atomic.LoadInt64(&c))
d(func() { atomic.AddInt64(&c, 1) })
d(func() { atomic.AddInt64(&c, 1) })
d(func() { atomic.AddInt64(&c, 1) })
time.Sleep(100 * time.Millisecond)
assert.EqualValues(t, 2, atomic.LoadInt64(&c))
}

114
modules/watcher/watcher.go
View File

@ -1,114 +0,0 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package watcher
import (
"context"
"io/fs"
"os"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"github.com/fsnotify/fsnotify"
)
// CreateWatcherOpts are options to configure the watcher
type CreateWatcherOpts struct {
// PathsCallback is used to set the required paths to watch
PathsCallback func(func(path, name string, d fs.DirEntry, err error) error) error
// BeforeCallback is called before any files are watched
BeforeCallback func()
// Between Callback is called between after a watched event has occurred
BetweenCallback func()
// AfterCallback is called as this watcher ends
AfterCallback func()
}
// CreateWatcher creates a watcher labelled with the provided description and running with the provided options.
// The created watcher will create a subcontext from the provided ctx and register it with the process manager.
func CreateWatcher(ctx context.Context, desc string, opts *CreateWatcherOpts) {
go run(ctx, desc, opts)
}
func run(ctx context.Context, desc string, opts *CreateWatcherOpts) {
if opts.BeforeCallback != nil {
opts.BeforeCallback()
}
if opts.AfterCallback != nil {
defer opts.AfterCallback()
}
ctx, _, finished := process.GetManager().AddTypedContext(ctx, "Watcher: "+desc, process.SystemProcessType, true)
defer finished()
log.Trace("Watcher loop starting for %s", desc)
defer log.Trace("Watcher loop ended for %s", desc)
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Error("Unable to create watcher for %s: %v", desc, err)
return
}
if err := opts.PathsCallback(func(path, _ string, d fs.DirEntry, err error) error {
if err != nil && !os.IsNotExist(err) {
return err
}
log.Trace("Watcher: %s watching %q", desc, path)
_ = watcher.Add(path)
return nil
}); err != nil {
log.Error("Unable to create watcher for %s: %v", desc, err)
_ = watcher.Close()
return
}
// Note we don't call the BetweenCallback here
for {
select {
case event, ok := <-watcher.Events:
if !ok {
_ = watcher.Close()
return
}
log.Debug("Watched file for %s had event: %v", desc, event)
case err, ok := <-watcher.Errors:
if !ok {
_ = watcher.Close()
return
}
log.Error("Error whilst watching files for %s: %v", desc, err)
case <-ctx.Done():
_ = watcher.Close()
return
}
// Recreate the watcher - only call the BetweenCallback after the new watcher is set-up
_ = watcher.Close()
watcher, err = fsnotify.NewWatcher()
if err != nil {
log.Error("Unable to create watcher for %s: %v", desc, err)
return
}
if err := opts.PathsCallback(func(path, _ string, _ fs.DirEntry, err error) error {
if err != nil {
return err
}
_ = watcher.Add(path)
return nil
}); err != nil {
log.Error("Unable to create watcher for %s: %v", desc, err)
_ = watcher.Close()
return
}
// Inform our BetweenCallback that there has been an event
if opts.BetweenCallback != nil {
opts.BetweenCallback()
}
}
}

9
routers/init.go
View File

@ -71,13 +71,6 @@ func mustInitCtx(ctx context.Context, fn func(ctx context.Context) error) {
}
}
// InitGitServices init new services for git, this is also called in `contrib/pr/checkout.go`
func InitGitServices() {
setting.LoadSettings()
mustInit(storage.Init)
mustInit(repo_service.Init)
}
func syncAppConfForGit(ctx context.Context) error {
runtimeState := new(system.RuntimeState)
if err := system.AppState.Get(runtimeState); err != nil {
@ -172,7 +165,7 @@ func GlobalInitInstalled(ctx context.Context) {
mustInit(ssh.Init)
auth.Init()
svg.Init()
mustInit(svg.Init)
actions_service.Init()

6
routers/install/routes.go
View File

@ -8,7 +8,6 @@ import (
"fmt"
"html"
"net/http"
"path"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/log"
@ -89,10 +88,7 @@ func Routes(ctx goctx.Context) *web.Route {
r.Use(middle)
}
r.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{
Directory: path.Join(setting.StaticRootPath, "public"),
Prefix: public.AssetsURLPathPrefix,
}), "InstallAssetsHandler"))
r.Use(web.WrapWithPrefix("/assets/", public.AssetsHandlerFunc("/assets/"), "AssetsHandler"))
r.Use(session.Sessioner(session.Options{
Provider: setting.SessionConfig.Provider,

3
routers/install/setting.go
View File

@ -30,7 +30,7 @@ func PreloadSettings(ctx context.Context) bool {
}
setting.LoadSettingsForInstall()
svg.Init()
_ = svg.Init()
}
return !setting.InstallLock
@ -47,6 +47,5 @@ func reloadSettings(ctx context.Context) {
} else {
log.Fatal("ORM engine initialization failed: %v", err)
}
svg.Init()
}
}

15
routers/web/devtest/devtest.go
View File

@ -15,15 +15,16 @@ import (
// List all devtest templates, they will be used for e2e tests for the UI components
func List(ctx *context.Context) {
templateNames := templates.GetTemplateAssetNames()
templateNames, err := templates.AssetFS().ListFiles("devtest", true)
if err != nil {
ctx.ServerError("AssetFS().ListFiles", err)
return
}
var subNames []string
const prefix = "templates/devtest/"
for _, tmplName := range templateNames {
if strings.HasPrefix(tmplName, prefix) {
subName := strings.TrimSuffix(strings.TrimPrefix(tmplName, prefix), ".tmpl")
if subName != "list" {
subNames = append(subNames, subName)
}
subName := strings.TrimSuffix(tmplName, ".tmpl")
if subName != "list" {
subNames = append(subNames, subName)
}
}
ctx.Data["SubNames"] = subNames

6
routers/web/web.go
View File

@ -103,11 +103,7 @@ func buildAuthGroup() *auth_service.Group {
func Routes(ctx gocontext.Context) *web.Route {
routes := web.NewRoute()
routes.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{
Directory: path.Join(setting.StaticRootPath, "public"),
Prefix: public.AssetsURLPathPrefix,
CorsHandler: CorsHandler(),
}), "AssetsHandler"))
routes.Use(web.WrapWithPrefix("/assets/", web.Wrap(CorsHandler(), public.AssetsHandlerFunc("/assets/")), "AssetsHandler"))
sessioner := session.Sessioner(session.Options{
Provider: setting.SessionConfig.Provider,