package files import ( "archive/zip" "bufio" "bytes" "context" "crypto/tls" "encoding/json" "fmt" "io" "io/fs" "net/http" "os" "path" "path/filepath" "strconv" "strings" "time" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/utils/cmd" http2 "github.com/1Panel-dev/1Panel/agent/utils/http" cZip "github.com/klauspost/compress/zip" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/transform" "github.com/1Panel-dev/1Panel/agent/global" "github.com/mholt/archiver/v4" "github.com/pkg/errors" "github.com/spf13/afero" ) type FileOp struct { Fs afero.Fs } func NewFileOp() FileOp { return FileOp{ Fs: afero.NewOsFs(), } } func (f FileOp) OpenFile(dst string) (fs.File, error) { return f.Fs.Open(dst) } func (f FileOp) GetContent(dst string) ([]byte, error) { afs := &afero.Afero{Fs: f.Fs} cByte, err := afs.ReadFile(dst) if err != nil { return nil, err } return cByte, nil } func (f FileOp) CreateDir(dst string, mode fs.FileMode) error { return f.Fs.MkdirAll(dst, mode) } func (f FileOp) CreateDirWithMode(dst string, mode fs.FileMode) error { if err := f.Fs.MkdirAll(dst, mode); err != nil { return err } return f.ChmodRWithMode(dst, mode, true) } func (f FileOp) CreateFile(dst string) error { if _, err := f.Fs.Create(dst); err != nil { return err } return nil } func (f FileOp) CreateFileWithMode(dst string, mode fs.FileMode) error { file, err := f.Fs.OpenFile(dst, os.O_CREATE, mode) if err != nil { return err } return file.Close() } func (f FileOp) LinkFile(source string, dst string, isSymlink bool) error { if isSymlink { osFs := afero.OsFs{} return osFs.SymlinkIfPossible(source, dst) } else { return os.Link(source, dst) } } func (f FileOp) DeleteDir(dst string) error { return f.Fs.RemoveAll(dst) } func (f FileOp) Stat(dst string) bool { info, _ := f.Fs.Stat(dst) return info != nil } func (f FileOp) DeleteFile(dst string) error { return f.Fs.Remove(dst) } func (f FileOp) CleanDir(dst string) error { return cmd.ExecCmd(fmt.Sprintf("rm -rf %s/*", dst)) } func (f FileOp) RmRf(dst string) error { return cmd.ExecCmd(fmt.Sprintf("rm -rf %s", dst)) } func (f FileOp) WriteFile(dst string, in io.Reader, mode fs.FileMode) error { file, err := f.Fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return err } defer file.Close() if _, err = io.Copy(file, in); err != nil { return err } if _, err = file.Stat(); err != nil { return err } return nil } func (f FileOp) SaveFile(dst string, content string, mode fs.FileMode) error { if !f.Stat(path.Dir(dst)) { _ = f.CreateDir(path.Dir(dst), mode.Perm()) } file, err := f.Fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return err } defer file.Close() write := bufio.NewWriter(file) _, _ = write.WriteString(content) write.Flush() return nil } func (f FileOp) SaveFileWithByte(dst string, content []byte, mode fs.FileMode) error { if !f.Stat(path.Dir(dst)) { _ = f.CreateDir(path.Dir(dst), mode.Perm()) } file, err := f.Fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return err } defer file.Close() write := bufio.NewWriter(file) _, _ = write.Write(content) write.Flush() return nil } func (f FileOp) ChownR(dst string, uid string, gid string, sub bool) error { cmdStr := fmt.Sprintf(`chown %s:%s "%s"`, uid, gid, dst) if sub { cmdStr = fmt.Sprintf(`chown -R %s:%s "%s"`, uid, gid, dst) } if cmd.HasNoPasswordSudo() { cmdStr = fmt.Sprintf("sudo %s", cmdStr) } if msg, err := cmd.ExecWithTimeOut(cmdStr, 10*time.Second); err != nil { if msg != "" { return errors.New(msg) } return err } return nil } func (f FileOp) ChmodR(dst string, mode int64, sub bool) error { cmdStr := fmt.Sprintf(`chmod %v "%s"`, fmt.Sprintf("%04o", mode), dst) if sub { cmdStr = fmt.Sprintf(`chmod -R %v "%s"`, fmt.Sprintf("%04o", mode), dst) } if cmd.HasNoPasswordSudo() { cmdStr = fmt.Sprintf("sudo %s", cmdStr) } if msg, err := cmd.ExecWithTimeOut(cmdStr, 10*time.Second); err != nil { if msg != "" { return errors.New(msg) } return err } return nil } func (f FileOp) ChmodRWithMode(dst string, mode fs.FileMode, sub bool) error { cmdStr := fmt.Sprintf(`chmod %v "%s"`, fmt.Sprintf("%o", mode.Perm()), dst) if sub { cmdStr = fmt.Sprintf(`chmod -R %v "%s"`, fmt.Sprintf("%o", mode.Perm()), dst) } if cmd.HasNoPasswordSudo() { cmdStr = fmt.Sprintf("sudo %s", cmdStr) } if msg, err := cmd.ExecWithTimeOut(cmdStr, 10*time.Second); err != nil { if msg != "" { return errors.New(msg) } return err } return nil } func (f FileOp) Rename(oldName string, newName string) error { return f.Fs.Rename(oldName, newName) } type WriteCounter struct { Total uint64 Written uint64 Key string Name string } type Process struct { Total uint64 `json:"total"` Written uint64 `json:"written"` Percent float64 `json:"percent"` Name string `json:"name"` } func (w *WriteCounter) Write(p []byte) (n int, err error) { n = len(p) w.Written += uint64(n) w.SaveProcess() return n, nil } func (w *WriteCounter) SaveProcess() { percentValue := 0.0 if w.Total > 0 { percent := float64(w.Written) / float64(w.Total) * 100 percentValue, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", percent), 64) } process := Process{ Total: w.Total, Written: w.Written, Percent: percentValue, Name: w.Name, } by, _ := json.Marshal(process) if percentValue < 100 { if err := global.CACHE.Set(w.Key, string(by)); err != nil { global.LOG.Errorf("save cache error, err %s", err.Error()) } } else { if err := global.CACHE.SetWithTTL(w.Key, string(by), time.Second*time.Duration(10)); err != nil { global.LOG.Errorf("save cache error, err %s", err.Error()) } } } func (f FileOp) DownloadFileWithProcess(url, dst, key string, ignoreCertificate bool) error { client := &http.Client{} if ignoreCertificate { client.Transport = &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } } request, err := http.NewRequest("GET", url, nil) if err != nil { return nil } request.Header.Set("Accept-Encoding", "identity") resp, err := client.Do(request) if err != nil { global.LOG.Errorf("get download file [%s] error, err %s", dst, err.Error()) return err } out, err := os.Create(dst) if err != nil { global.LOG.Errorf("create download file [%s] error, err %s", dst, err.Error()) return err } go func() { counter := &WriteCounter{} counter.Key = key if resp.ContentLength > 0 { counter.Total = uint64(resp.ContentLength) } counter.Name = filepath.Base(dst) if _, err = io.Copy(out, io.TeeReader(resp.Body, counter)); err != nil { global.LOG.Errorf("save download file [%s] error, err %s", dst, err.Error()) } out.Close() resp.Body.Close() value, err := global.CACHE.Get(counter.Key) if err != nil { global.LOG.Errorf("get cache error,err %s", err.Error()) return } process := &Process{} _ = json.Unmarshal(value, process) process.Percent = 100 process.Name = counter.Name process.Total = process.Written by, _ := json.Marshal(process) if err := global.CACHE.SetWithTTL(counter.Key, string(by), time.Second*time.Duration(10)); err != nil { global.LOG.Errorf("save cache error, err %s", err.Error()) } }() return nil } func (f FileOp) DownloadFile(url, dst string) error { resp, err := http2.GetHttpRes(url) if err != nil { return err } defer resp.Body.Close() out, err := os.Create(dst) if err != nil { return fmt.Errorf("create download file [%s] error, err %s", dst, err.Error()) } defer out.Close() if _, err = io.Copy(out, resp.Body); err != nil { return fmt.Errorf("save download file [%s] error, err %s", dst, err.Error()) } return nil } func (f FileOp) DownloadFileWithProxy(url, dst string) error { _, resp, err := http2.HandleGet(url, http.MethodGet, constant.TimeOut5m) if err != nil { return err } out, err := os.Create(dst) if err != nil { return fmt.Errorf("create download file [%s] error, err %s", dst, err.Error()) } defer out.Close() reader := bytes.NewReader(resp) if _, err = io.Copy(out, reader); err != nil { return fmt.Errorf("save download file [%s] error, err %s", dst, err.Error()) } return nil } func (f FileOp) Cut(oldPaths []string, dst, name string, cover bool) error { for _, p := range oldPaths { var dstPath string if name != "" { dstPath = filepath.Join(dst, name) if f.Stat(dstPath) { dstPath = dst } } else { base := filepath.Base(p) dstPath = filepath.Join(dst, base) } coverFlag := "" if cover { coverFlag = "-f" } cmdStr := fmt.Sprintf(`mv %s '%s' '%s'`, coverFlag, p, dstPath) if err := cmd.ExecCmd(cmdStr); err != nil { return err } } return nil } func (f FileOp) Mv(oldPath, dstPath string) error { cmdStr := fmt.Sprintf(`mv '%s' '%s'`, oldPath, dstPath) if err := cmd.ExecCmd(cmdStr); err != nil { return err } return nil } func (f FileOp) Copy(src, dst string) error { if src = path.Clean("/" + src); src == "" { return os.ErrNotExist } if dst = path.Clean("/" + dst); dst == "" { return os.ErrNotExist } if src == "/" || dst == "/" { return os.ErrInvalid } if dst == src { return os.ErrInvalid } info, err := f.Fs.Stat(src) if err != nil { return err } if info.IsDir() { return f.CopyDir(src, dst) } return f.CopyFile(src, dst) } func (f FileOp) CopyAndReName(src, dst, name string, cover bool) error { if src = path.Clean("/" + src); src == "" { return os.ErrNotExist } if dst = path.Clean("/" + dst); dst == "" { return os.ErrNotExist } if src == "/" || dst == "/" { return os.ErrInvalid } if dst == src { return os.ErrInvalid } srcInfo, err := f.Fs.Stat(src) if err != nil { return err } if srcInfo.IsDir() { dstPath := dst if name != "" && !cover { dstPath = filepath.Join(dst, name) } return cmd.ExecCmd(fmt.Sprintf(`cp -rf '%s' '%s'`, src, dstPath)) } else { dstPath := filepath.Join(dst, name) if cover { dstPath = dst } return cmd.ExecCmd(fmt.Sprintf(`cp -f '%s' '%s'`, src, dstPath)) } } func (f FileOp) CopyDirWithNewName(src, dst, newName string) error { dstDir := filepath.Join(dst, newName) str := fmt.Sprintf(`cp -rf '%s' '%s'`, src, dstDir) return cmd.ExecCmd(str) } func (f FileOp) CopyDir(src, dst string) error { srcInfo, err := f.Fs.Stat(src) if err != nil { return err } dstDir := filepath.Join(dst, srcInfo.Name()) if err = f.Fs.MkdirAll(dstDir, srcInfo.Mode()); err != nil { return err } return cmd.ExecCmd(fmt.Sprintf(`cp -rf '%s' '%s'`, src, dst+"/")) } func (f FileOp) CopyDirWithExclude(src, dst string, excludeNames []string) error { srcInfo, err := f.Fs.Stat(src) if err != nil { return err } dstDir := filepath.Join(dst, srcInfo.Name()) if err = f.Fs.MkdirAll(dstDir, srcInfo.Mode()); err != nil { return err } if len(excludeNames) == 0 { return cmd.ExecCmd(fmt.Sprintf(`cp -rf '%s' '%s'`, src, dst+"/")) } tmpFiles, err := os.ReadDir(src) if err != nil { return err } for _, item := range tmpFiles { isExclude := false for _, name := range excludeNames { if item.Name() == name { isExclude = true break } } if isExclude { continue } if item.IsDir() { if err := f.CopyDir(path.Join(src, item.Name()), dstDir); err != nil { return err } continue } if err := f.CopyFile(path.Join(src, item.Name()), dstDir); err != nil { return err } } return nil } func (f FileOp) CopyFile(src, dst string) error { dst = filepath.Clean(dst) + string(filepath.Separator) return cmd.ExecCmd(fmt.Sprintf(`cp -f '%s' '%s'`, src, dst+"/")) } func (f FileOp) GetDirSize(path string) (float64, error) { var size int64 err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() { size += info.Size() } return nil }) if err != nil { return 0, err } return float64(size), nil } func getFormat(cType CompressType) archiver.CompressedArchive { format := archiver.CompressedArchive{} switch cType { case Tar: format.Archival = archiver.Tar{} case TarGz, Gz: format.Compression = archiver.Gz{} format.Archival = archiver.Tar{} case SdkTarGz: format.Compression = archiver.Gz{} format.Archival = archiver.Tar{} case SdkZip, Zip: format.Archival = archiver.Zip{ Compression: zip.Deflate, } case Bz2: format.Compression = archiver.Bz2{} format.Archival = archiver.Tar{} case Xz: format.Compression = archiver.Xz{} format.Archival = archiver.Tar{} } return format } func (f FileOp) Compress(srcRiles []string, dst string, name string, cType CompressType, secret string) error { format := getFormat(cType) fileMaps := make(map[string]string, len(srcRiles)) for _, s := range srcRiles { base := filepath.Base(s) fileMaps[s] = base } if !f.Stat(dst) { _ = f.CreateDir(dst, 0755) } files, err := archiver.FilesFromDisk(nil, fileMaps) if err != nil { return err } dstFile := filepath.Join(dst, name) out, err := f.Fs.Create(dstFile) if err != nil { return err } switch cType { case Zip: if err := ZipFile(files, out); err == nil { return nil } _ = f.DeleteFile(dstFile) return NewZipArchiver().Compress(srcRiles, dstFile, "") case TarGz: err = NewTarGzArchiver().Compress(srcRiles, dstFile, secret) if err != nil { _ = f.DeleteFile(dstFile) return err } default: err = format.Archive(context.Background(), out, files) if err != nil { _ = f.DeleteFile(dstFile) return err } } return nil } func isIgnoreFile(name string) bool { return strings.HasPrefix(name, "__MACOSX") || strings.HasSuffix(name, ".DS_Store") || strings.HasPrefix(name, "._") } func decodeGBK(input string) (string, error) { decoder := simplifiedchinese.GBK.NewDecoder() decoded, _, err := transform.String(decoder, input) if err != nil { return "", err } return decoded, nil } func (f FileOp) decompressWithSDK(srcFile string, dst string, cType CompressType) error { format := getFormat(cType) handler := func(ctx context.Context, archFile archiver.File) error { info := archFile.FileInfo if isIgnoreFile(archFile.Name()) { return nil } fileName := archFile.NameInArchive var err error if header, ok := archFile.Header.(cZip.FileHeader); ok { if header.NonUTF8 && header.Flags == 0 { fileName, err = decodeGBK(fileName) if err != nil { return err } } } filePath := filepath.Join(dst, fileName) if archFile.FileInfo.IsDir() { if err := f.Fs.MkdirAll(filePath, info.Mode()); err != nil { return err } return nil } else { parentDir := path.Dir(filePath) if !f.Stat(parentDir) { if err := f.Fs.MkdirAll(parentDir, info.Mode()); err != nil { return err } } } fr, err := archFile.Open() if err != nil { return err } defer fr.Close() fw, err := f.Fs.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode()) if err != nil { return err } defer fw.Close() if _, err := io.Copy(fw, fr); err != nil { return err } return nil } input, err := f.Fs.Open(srcFile) if err != nil { return err } return format.Extract(context.Background(), input, nil, handler) } func (f FileOp) Decompress(srcFile string, dst string, cType CompressType, secret string) error { if cType == Tar || cType == Zip || cType == TarGz { shellArchiver, err := NewShellArchiver(cType) if err == nil { if err = shellArchiver.Extract(srcFile, dst, secret); err == nil { return nil } } } return f.decompressWithSDK(srcFile, dst, cType) } func ZipFile(files []archiver.File, dst afero.File) error { zw := zip.NewWriter(dst) defer zw.Close() for _, file := range files { hdr, err := zip.FileInfoHeader(file) if err != nil { return err } hdr.Method = zip.Deflate hdr.Name = file.NameInArchive if file.IsDir() { if !strings.HasSuffix(hdr.Name, "/") { hdr.Name += "/" } } w, err := zw.CreateHeader(hdr) if err != nil { return err } if file.IsDir() { continue } if file.LinkTarget != "" { _, err = w.Write([]byte(filepath.ToSlash(file.LinkTarget))) if err != nil { return err } } else { fileReader, err := file.Open() if err != nil { return err } _, err = io.Copy(w, fileReader) if err != nil { return err } } } return nil } func (f FileOp) TarGzCompressPro(withDir bool, src, dst, secret, exclusionRules string) error { workdir := src srcItem := "." if withDir { workdir = path.Dir(src) srcItem = path.Base(src) } commands := "" exMap := make(map[string]struct{}) exStr := "" excludes := strings.Split(exclusionRules, ";") excludes = append(excludes, "*.sock") for _, exclude := range excludes { if len(exclude) == 0 { continue } if _, ok := exMap[exclude]; ok { continue } exStr += " --exclude " exStr += exclude exMap[exclude] = struct{}{} } if len(secret) != 0 { commands = fmt.Sprintf("tar -zcf - %s | openssl enc -aes-256-cbc -salt -pbkdf2 -k '%s' -out %s", srcItem, secret, dst) global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) } else { commands = fmt.Sprintf("tar zcf %s %s %s", dst, exStr, srcItem) global.LOG.Debug(commands) } return cmd.ExecCmdWithDir(commands, workdir) } func (f FileOp) TarGzExtractPro(src, dst string, secret string) error { if _, err := os.Stat(path.Dir(dst)); err != nil && os.IsNotExist(err) { if err = os.MkdirAll(path.Dir(dst), os.ModePerm); err != nil { return err } } commands := "" if len(secret) != 0 { commands = fmt.Sprintf("openssl enc -d -aes-256-cbc -salt -pbkdf2 -k '%s' -in %s | tar -zxf - > /root/log", secret, src) global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) } else { commands = fmt.Sprintf("tar zxvf %s", src) global.LOG.Debug(commands) } return cmd.ExecCmdWithDir(commands, dst) }