mirror of
https://github.com/1Panel-dev/1Panel.git
synced 2024-11-23 18:49:21 +08:00
pref: 移除部分废弃代码 (#5910)
This commit is contained in:
ssongliu
GitHub
parent
075879b5b5
commit
2873632a40
@ -1,303 +0,0 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/1Panel-dev/1Panel/backend/constant"
|
||||
"github.com/1Panel-dev/1Panel/backend/global"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func init() {}
|
||||
|
||||
type dumpOption struct {
|
||||
isData bool
|
||||
|
||||
tables []string
|
||||
isAllTable bool
|
||||
isDropTable bool
|
||||
writer io.Writer
|
||||
}
|
||||
|
||||
type DumpOption func(*dumpOption)
|
||||
|
||||
func WithDropTable() DumpOption {
|
||||
return func(option *dumpOption) {
|
||||
option.isDropTable = true
|
||||
}
|
||||
}
|
||||
|
||||
func WithData() DumpOption {
|
||||
return func(option *dumpOption) {
|
||||
option.isData = true
|
||||
}
|
||||
}
|
||||
|
||||
func WithWriter(writer io.Writer) DumpOption {
|
||||
return func(option *dumpOption) {
|
||||
option.writer = writer
|
||||
}
|
||||
}
|
||||
|
||||
func Dump(dns string, opts ...DumpOption) error {
|
||||
start := time.Now()
|
||||
global.LOG.Infof("dump start at %s\n", start.Format(constant.DateTimeLayout))
|
||||
defer func() {
|
||||
end := time.Now()
|
||||
global.LOG.Infof("dump end at %s, cost %s\n", end.Format(constant.DateTimeLayout), end.Sub(start))
|
||||
}()
|
||||
|
||||
var err error
|
||||
|
||||
var o dumpOption
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
|
||||
if len(o.tables) == 0 {
|
||||
o.isAllTable = true
|
||||
}
|
||||
|
||||
if o.writer == nil {
|
||||
o.writer = os.Stdout
|
||||
}
|
||||
|
||||
buf := bufio.NewWriter(o.writer)
|
||||
defer buf.Flush()
|
||||
|
||||
itemFile, lineNumber := "", 0
|
||||
|
||||
itemFile += "-- ----------------------------\n"
|
||||
itemFile += "-- MySQL Database Dump\n"
|
||||
itemFile += "-- Start Time: " + start.Format(constant.DateTimeLayout) + "\n"
|
||||
itemFile += "-- ----------------------------\n\n\n"
|
||||
|
||||
db, err := sql.Open("mysql", dns)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("open mysql db failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
dbName, err := getDBNameFromDNS(dns)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("get db name from dns failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
_, err = db.Exec(fmt.Sprintf("USE `%s`", dbName))
|
||||
if err != nil {
|
||||
global.LOG.Errorf("exec `use %s` failed, err: %v", dbName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
var tables []string
|
||||
if o.isAllTable {
|
||||
tmp, err := getAllTables(db)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("get all tables failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
tables = tmp
|
||||
} else {
|
||||
tables = o.tables
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
if o.isDropTable {
|
||||
itemFile += fmt.Sprintf("DROP TABLE IF EXISTS `%s`;\n", table)
|
||||
}
|
||||
|
||||
itemFile += "-- ----------------------------\n"
|
||||
itemFile += fmt.Sprintf("-- Table structure for %s\n", table)
|
||||
itemFile += "-- ----------------------------\n"
|
||||
|
||||
createTableSQL, err := getCreateTableSQL(db, table)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("get create table sql failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
itemFile += createTableSQL
|
||||
itemFile += ";\n\n\n\n"
|
||||
|
||||
if o.isData {
|
||||
itemFile += "-- ----------------------------\n"
|
||||
itemFile += fmt.Sprintf("-- Records of %s\n", table)
|
||||
itemFile += "-- ----------------------------\n"
|
||||
|
||||
lineRows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table))
|
||||
if err != nil {
|
||||
global.LOG.Errorf("exec `select * from %s` failed, err: %v", table, err)
|
||||
return err
|
||||
}
|
||||
defer lineRows.Close()
|
||||
|
||||
var columns []string
|
||||
columns, err = lineRows.Columns()
|
||||
if err != nil {
|
||||
global.LOG.Errorf("get columes failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
columnTypes, err := lineRows.ColumnTypes()
|
||||
if err != nil {
|
||||
global.LOG.Errorf("get colume types failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
for lineRows.Next() {
|
||||
row := make([]interface{}, len(columns))
|
||||
rowPointers := make([]interface{}, len(columns))
|
||||
for i := range columns {
|
||||
rowPointers[i] = &row[i]
|
||||
}
|
||||
if err = lineRows.Scan(rowPointers...); err != nil {
|
||||
global.LOG.Errorf("scan row data failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
ssql := loadDataSql(row, columnTypes, table)
|
||||
if len(ssql) != 0 {
|
||||
itemFile += ssql
|
||||
lineNumber++
|
||||
}
|
||||
if lineNumber > 500 {
|
||||
_, _ = buf.WriteString(itemFile)
|
||||
itemFile = ""
|
||||
lineNumber = 0
|
||||
}
|
||||
}
|
||||
|
||||
itemFile += "\n\n"
|
||||
}
|
||||
}
|
||||
|
||||
itemFile += "-- ----------------------------\n"
|
||||
itemFile += "-- Dumped by mysqldump\n"
|
||||
itemFile += "-- Cost Time: " + time.Since(start).String() + "\n"
|
||||
itemFile += "-- ----------------------------\n"
|
||||
|
||||
_, _ = buf.WriteString(itemFile)
|
||||
_ = buf.Flush()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getCreateTableSQL(db *sql.DB, table string) (string, error) {
|
||||
var createTableSQL string
|
||||
err := db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE `%s`", table)).Scan(&table, &createTableSQL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
createTableSQL = strings.Replace(createTableSQL, "CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1)
|
||||
return createTableSQL, nil
|
||||
}
|
||||
|
||||
func getAllTables(db *sql.DB) ([]string, error) {
|
||||
var tables []string
|
||||
rows, err := db.Query("SHOW TABLES")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var table string
|
||||
err = rows.Scan(&table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tables = append(tables, table)
|
||||
}
|
||||
return tables, nil
|
||||
}
|
||||
|
||||
func loadDataSql(row []interface{}, columnTypes []*sql.ColumnType, table string) string {
|
||||
ssql := "INSERT INTO `" + table + "` VALUES ("
|
||||
for i, col := range row {
|
||||
if col == nil {
|
||||
ssql += "NULL"
|
||||
} else {
|
||||
Type := columnTypes[i].DatabaseTypeName()
|
||||
Type = strings.Replace(Type, "UNSIGNED", "", -1)
|
||||
Type = strings.Replace(Type, " ", "", -1)
|
||||
switch Type {
|
||||
case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT":
|
||||
if bs, ok := col.([]byte); ok {
|
||||
ssql += string(bs)
|
||||
} else {
|
||||
ssql += fmt.Sprintf("%d", col)
|
||||
}
|
||||
case "FLOAT", "DOUBLE":
|
||||
if bs, ok := col.([]byte); ok {
|
||||
ssql += string(bs)
|
||||
} else {
|
||||
ssql += fmt.Sprintf("%f", col)
|
||||
}
|
||||
case "DECIMAL", "DEC":
|
||||
ssql += fmt.Sprintf("%s", col)
|
||||
|
||||
case "DATE":
|
||||
t, ok := col.(time.Time)
|
||||
if !ok {
|
||||
global.LOG.Errorf("the DATE type conversion failed, err value: %v", col)
|
||||
return ""
|
||||
}
|
||||
ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02"))
|
||||
case "DATETIME":
|
||||
t, ok := col.(time.Time)
|
||||
if !ok {
|
||||
global.LOG.Errorf("the DATETIME type conversion failed, err value: %v", col)
|
||||
return ""
|
||||
}
|
||||
ssql += fmt.Sprintf("'%s'", t.Format(constant.DateTimeLayout))
|
||||
case "TIMESTAMP":
|
||||
t, ok := col.(time.Time)
|
||||
if !ok {
|
||||
global.LOG.Errorf("the TIMESTAMP type conversion failed, err value: %v", col)
|
||||
return ""
|
||||
}
|
||||
ssql += fmt.Sprintf("'%s'", t.Format(constant.DateTimeLayout))
|
||||
case "TIME":
|
||||
t, ok := col.([]byte)
|
||||
if !ok {
|
||||
global.LOG.Errorf("the TIME type conversion failed, err value: %v", col)
|
||||
return ""
|
||||
}
|
||||
ssql += fmt.Sprintf("'%s'", string(t))
|
||||
case "YEAR":
|
||||
t, ok := col.([]byte)
|
||||
if !ok {
|
||||
global.LOG.Errorf("the YEAR type conversion failed, err value: %v", col)
|
||||
return ""
|
||||
}
|
||||
ssql += string(t)
|
||||
case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT":
|
||||
ssql += fmt.Sprintf("'%s'", strings.Replace(fmt.Sprintf("%s", col), "'", "''", -1))
|
||||
case "BIT", "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB":
|
||||
ssql += fmt.Sprintf("0x%X", col)
|
||||
case "ENUM", "SET":
|
||||
ssql += fmt.Sprintf("'%s'", col)
|
||||
case "BOOL", "BOOLEAN":
|
||||
if col.(bool) {
|
||||
ssql += "true"
|
||||
} else {
|
||||
ssql += "false"
|
||||
}
|
||||
case "JSON":
|
||||
ssql += fmt.Sprintf("'%s'", col)
|
||||
default:
|
||||
global.LOG.Errorf("unsupported colume type: %s", Type)
|
||||
return ""
|
||||
}
|
||||
}
|
||||
if i < len(row)-1 {
|
||||
ssql += ","
|
||||
}
|
||||
}
|
||||
ssql += ");\n"
|
||||
return ssql
|
||||
}
|
||||
@ -1,244 +0,0 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/1Panel-dev/1Panel/backend/constant"
|
||||
"github.com/1Panel-dev/1Panel/backend/global"
|
||||
)
|
||||
|
||||
type sourceOption struct {
|
||||
dryRun bool
|
||||
mergeInsert int
|
||||
debug bool
|
||||
}
|
||||
type SourceOption func(*sourceOption)
|
||||
|
||||
func WithMergeInsert(size int) SourceOption {
|
||||
return func(o *sourceOption) {
|
||||
o.mergeInsert = size
|
||||
}
|
||||
}
|
||||
|
||||
type dbWrapper struct {
|
||||
DB *sql.DB
|
||||
debug bool
|
||||
dryRun bool
|
||||
}
|
||||
|
||||
func newDBWrapper(db *sql.DB, dryRun, debug bool) *dbWrapper {
|
||||
|
||||
return &dbWrapper{
|
||||
DB: db,
|
||||
dryRun: dryRun,
|
||||
debug: debug,
|
||||
}
|
||||
}
|
||||
|
||||
func (db *dbWrapper) Exec(query string, args ...interface{}) (sql.Result, error) {
|
||||
if db.debug {
|
||||
global.LOG.Debugf("query %s", query)
|
||||
}
|
||||
|
||||
if db.dryRun {
|
||||
return nil, nil
|
||||
}
|
||||
return db.DB.Exec(query, args...)
|
||||
}
|
||||
|
||||
func Source(dns string, reader io.Reader, opts ...SourceOption) error {
|
||||
start := time.Now()
|
||||
global.LOG.Infof("source start at %s", start.Format(constant.DateTimeLayout))
|
||||
defer func() {
|
||||
end := time.Now()
|
||||
global.LOG.Infof("source end at %s, cost %s", end.Format(constant.DateTimeLayout), end.Sub(start))
|
||||
}()
|
||||
|
||||
var err error
|
||||
var db *sql.DB
|
||||
var o sourceOption
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
|
||||
dbName, err := getDBNameFromDNS(dns)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("get db name from dns failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
db, err = sql.Open("mysql", dns)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("open mysql db failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
dbWrapper := newDBWrapper(db, o.dryRun, o.debug)
|
||||
|
||||
_, err = dbWrapper.Exec(fmt.Sprintf("USE `%s`;", dbName))
|
||||
if err != nil {
|
||||
global.LOG.Errorf("exec `use %s` failed, err: %v", dbName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
db.SetConnMaxLifetime(3600)
|
||||
|
||||
r := bufio.NewReader(reader)
|
||||
_, err = dbWrapper.Exec("SET autocommit=0;")
|
||||
if err != nil {
|
||||
global.LOG.Errorf("exec `set autocommit=0` failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
line, err := readLine(r)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
global.LOG.Errorf("read sql failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
ssql, err := trim(line)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("trim sql failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
afterInsertSql := ""
|
||||
if o.mergeInsert > 1 && strings.HasPrefix(ssql, "INSERT INTO") {
|
||||
var insertSQLs []string
|
||||
insertSQLs = append(insertSQLs, ssql)
|
||||
for i := 0; i < o.mergeInsert-1; i++ {
|
||||
line, err := readLine(r)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
ssql2, err := trim(line)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("trim merge insert sql failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
if strings.HasPrefix(ssql2, "INSERT INTO") {
|
||||
insertSQLs = append(insertSQLs, ssql2)
|
||||
continue
|
||||
}
|
||||
afterInsertSql = ssql2
|
||||
break
|
||||
}
|
||||
ssql, err = mergeInsert(insertSQLs)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("do merge insert failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = dbWrapper.Exec(ssql)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("exec sql failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
if len(afterInsertSql) != 0 {
|
||||
_, err = dbWrapper.Exec(afterInsertSql)
|
||||
if err != nil {
|
||||
global.LOG.Errorf("exec sql failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_, err = dbWrapper.Exec("COMMIT;")
|
||||
if err != nil {
|
||||
global.LOG.Errorf("exec `commit` failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = dbWrapper.Exec("SET autocommit=1;")
|
||||
if err != nil {
|
||||
global.LOG.Errorf("exec `autocommit=1` failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeInsert(insertSQLs []string) (string, error) {
|
||||
if len(insertSQLs) == 0 {
|
||||
return "", errors.New("no input provided")
|
||||
}
|
||||
builder := strings.Builder{}
|
||||
sql1 := insertSQLs[0]
|
||||
sql1 = strings.TrimSuffix(sql1, ";")
|
||||
builder.WriteString(sql1)
|
||||
for i, insertSQL := range insertSQLs[1:] {
|
||||
if i < len(insertSQLs)-1 {
|
||||
builder.WriteString(",")
|
||||
}
|
||||
|
||||
valuesIdx := strings.Index(insertSQL, "VALUES")
|
||||
if valuesIdx == -1 {
|
||||
return "", errors.New("invalid SQL: missing VALUES keyword")
|
||||
}
|
||||
sqln := insertSQL[valuesIdx:]
|
||||
sqln = strings.TrimPrefix(sqln, "VALUES")
|
||||
sqln = strings.TrimSuffix(sqln, ";")
|
||||
builder.WriteString(sqln)
|
||||
|
||||
}
|
||||
builder.WriteString(";")
|
||||
|
||||
return builder.String(), nil
|
||||
}
|
||||
|
||||
func trim(s string) (string, error) {
|
||||
s = strings.TrimLeft(s, "\n")
|
||||
s = strings.TrimSpace(s)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func getDBNameFromDNS(dns string) (string, error) {
|
||||
ss1 := strings.Split(dns, "/")
|
||||
if len(ss1) == 2 {
|
||||
ss2 := strings.Split(ss1[1], "?")
|
||||
if len(ss2) == 2 {
|
||||
return ss2[0], nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("dns error: %s", dns)
|
||||
}
|
||||
|
||||
func readLine(r *bufio.Reader) (string, error) {
|
||||
lineItem, err := r.ReadString('\n')
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return lineItem, err
|
||||
}
|
||||
global.LOG.Errorf("read merge insert sql failed, err: %v", err)
|
||||
return "", err
|
||||
}
|
||||
if strings.HasSuffix(lineItem, ";\n") {
|
||||
return lineItem, nil
|
||||
}
|
||||
lineAppend, err := readLine(r)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return lineItem, err
|
||||
}
|
||||
global.LOG.Errorf("read merge insert sql failed, err: %v", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
return lineItem + lineAppend, nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user