diff --git a/backend/utils/mysql/helper/dump.go b/backend/utils/mysql/helper/dump.go deleted file mode 100644 index df0144592..000000000 --- a/backend/utils/mysql/helper/dump.go +++ /dev/null @@ -1,303 +0,0 @@ -package helper - -import ( - "bufio" - "database/sql" - "fmt" - "io" - "os" - "strings" - "time" - - "github.com/1Panel-dev/1Panel/backend/constant" - "github.com/1Panel-dev/1Panel/backend/global" - _ "github.com/go-sql-driver/mysql" -) - -func init() {} - -type dumpOption struct { - isData bool - - tables []string - isAllTable bool - isDropTable bool - writer io.Writer -} - -type DumpOption func(*dumpOption) - -func WithDropTable() DumpOption { - return func(option *dumpOption) { - option.isDropTable = true - } -} - -func WithData() DumpOption { - return func(option *dumpOption) { - option.isData = true - } -} - -func WithWriter(writer io.Writer) DumpOption { - return func(option *dumpOption) { - option.writer = writer - } -} - -func Dump(dns string, opts ...DumpOption) error { - start := time.Now() - global.LOG.Infof("dump start at %s\n", start.Format(constant.DateTimeLayout)) - defer func() { - end := time.Now() - global.LOG.Infof("dump end at %s, cost %s\n", end.Format(constant.DateTimeLayout), end.Sub(start)) - }() - - var err error - - var o dumpOption - - for _, opt := range opts { - opt(&o) - } - - if len(o.tables) == 0 { - o.isAllTable = true - } - - if o.writer == nil { - o.writer = os.Stdout - } - - buf := bufio.NewWriter(o.writer) - defer buf.Flush() - - itemFile, lineNumber := "", 0 - - itemFile += "-- ----------------------------\n" - itemFile += "-- MySQL Database Dump\n" - itemFile += "-- Start Time: " + start.Format(constant.DateTimeLayout) + "\n" - itemFile += "-- ----------------------------\n\n\n" - - db, err := sql.Open("mysql", dns) - if err != nil { - global.LOG.Errorf("open mysql db failed, err: %v", err) - return err - } - defer db.Close() - - dbName, err := getDBNameFromDNS(dns) - if err != nil { - global.LOG.Errorf("get db name from dns failed, err: %v", err) - return err - } - _, err = db.Exec(fmt.Sprintf("USE `%s`", dbName)) - if err != nil { - global.LOG.Errorf("exec `use %s` failed, err: %v", dbName, err) - return err - } - - var tables []string - if o.isAllTable { - tmp, err := getAllTables(db) - if err != nil { - global.LOG.Errorf("get all tables failed, err: %v", err) - return err - } - tables = tmp - } else { - tables = o.tables - } - - for _, table := range tables { - if o.isDropTable { - itemFile += fmt.Sprintf("DROP TABLE IF EXISTS `%s`;\n", table) - } - - itemFile += "-- ----------------------------\n" - itemFile += fmt.Sprintf("-- Table structure for %s\n", table) - itemFile += "-- ----------------------------\n" - - createTableSQL, err := getCreateTableSQL(db, table) - if err != nil { - global.LOG.Errorf("get create table sql failed, err: %v", err) - return err - } - itemFile += createTableSQL - itemFile += ";\n\n\n\n" - - if o.isData { - itemFile += "-- ----------------------------\n" - itemFile += fmt.Sprintf("-- Records of %s\n", table) - itemFile += "-- ----------------------------\n" - - lineRows, err := db.Query(fmt.Sprintf("SELECT * FROM `%s`", table)) - if err != nil { - global.LOG.Errorf("exec `select * from %s` failed, err: %v", table, err) - return err - } - defer lineRows.Close() - - var columns []string - columns, err = lineRows.Columns() - if err != nil { - global.LOG.Errorf("get columes failed, err: %v", err) - return err - } - columnTypes, err := lineRows.ColumnTypes() - if err != nil { - global.LOG.Errorf("get colume types failed, err: %v", err) - return err - } - for lineRows.Next() { - row := make([]interface{}, len(columns)) - rowPointers := make([]interface{}, len(columns)) - for i := range columns { - rowPointers[i] = &row[i] - } - if err = lineRows.Scan(rowPointers...); err != nil { - global.LOG.Errorf("scan row data failed, err: %v", err) - return err - } - ssql := loadDataSql(row, columnTypes, table) - if len(ssql) != 0 { - itemFile += ssql - lineNumber++ - } - if lineNumber > 500 { - _, _ = buf.WriteString(itemFile) - itemFile = "" - lineNumber = 0 - } - } - - itemFile += "\n\n" - } - } - - itemFile += "-- ----------------------------\n" - itemFile += "-- Dumped by mysqldump\n" - itemFile += "-- Cost Time: " + time.Since(start).String() + "\n" - itemFile += "-- ----------------------------\n" - - _, _ = buf.WriteString(itemFile) - _ = buf.Flush() - - return nil -} - -func getCreateTableSQL(db *sql.DB, table string) (string, error) { - var createTableSQL string - err := db.QueryRow(fmt.Sprintf("SHOW CREATE TABLE `%s`", table)).Scan(&table, &createTableSQL) - if err != nil { - return "", err - } - createTableSQL = strings.Replace(createTableSQL, "CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1) - return createTableSQL, nil -} - -func getAllTables(db *sql.DB) ([]string, error) { - var tables []string - rows, err := db.Query("SHOW TABLES") - if err != nil { - return nil, err - } - defer rows.Close() - - for rows.Next() { - var table string - err = rows.Scan(&table) - if err != nil { - return nil, err - } - tables = append(tables, table) - } - return tables, nil -} - -func loadDataSql(row []interface{}, columnTypes []*sql.ColumnType, table string) string { - ssql := "INSERT INTO `" + table + "` VALUES (" - for i, col := range row { - if col == nil { - ssql += "NULL" - } else { - Type := columnTypes[i].DatabaseTypeName() - Type = strings.Replace(Type, "UNSIGNED", "", -1) - Type = strings.Replace(Type, " ", "", -1) - switch Type { - case "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT": - if bs, ok := col.([]byte); ok { - ssql += string(bs) - } else { - ssql += fmt.Sprintf("%d", col) - } - case "FLOAT", "DOUBLE": - if bs, ok := col.([]byte); ok { - ssql += string(bs) - } else { - ssql += fmt.Sprintf("%f", col) - } - case "DECIMAL", "DEC": - ssql += fmt.Sprintf("%s", col) - - case "DATE": - t, ok := col.(time.Time) - if !ok { - global.LOG.Errorf("the DATE type conversion failed, err value: %v", col) - return "" - } - ssql += fmt.Sprintf("'%s'", t.Format("2006-01-02")) - case "DATETIME": - t, ok := col.(time.Time) - if !ok { - global.LOG.Errorf("the DATETIME type conversion failed, err value: %v", col) - return "" - } - ssql += fmt.Sprintf("'%s'", t.Format(constant.DateTimeLayout)) - case "TIMESTAMP": - t, ok := col.(time.Time) - if !ok { - global.LOG.Errorf("the TIMESTAMP type conversion failed, err value: %v", col) - return "" - } - ssql += fmt.Sprintf("'%s'", t.Format(constant.DateTimeLayout)) - case "TIME": - t, ok := col.([]byte) - if !ok { - global.LOG.Errorf("the TIME type conversion failed, err value: %v", col) - return "" - } - ssql += fmt.Sprintf("'%s'", string(t)) - case "YEAR": - t, ok := col.([]byte) - if !ok { - global.LOG.Errorf("the YEAR type conversion failed, err value: %v", col) - return "" - } - ssql += string(t) - case "CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT": - ssql += fmt.Sprintf("'%s'", strings.Replace(fmt.Sprintf("%s", col), "'", "''", -1)) - case "BIT", "BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB": - ssql += fmt.Sprintf("0x%X", col) - case "ENUM", "SET": - ssql += fmt.Sprintf("'%s'", col) - case "BOOL", "BOOLEAN": - if col.(bool) { - ssql += "true" - } else { - ssql += "false" - } - case "JSON": - ssql += fmt.Sprintf("'%s'", col) - default: - global.LOG.Errorf("unsupported colume type: %s", Type) - return "" - } - } - if i < len(row)-1 { - ssql += "," - } - } - ssql += ");\n" - return ssql -} diff --git a/backend/utils/mysql/helper/source.go b/backend/utils/mysql/helper/source.go deleted file mode 100644 index b15a5cd97..000000000 --- a/backend/utils/mysql/helper/source.go +++ /dev/null @@ -1,244 +0,0 @@ -package helper - -import ( - "bufio" - "database/sql" - "errors" - "fmt" - "io" - "strings" - "time" - - "github.com/1Panel-dev/1Panel/backend/constant" - "github.com/1Panel-dev/1Panel/backend/global" -) - -type sourceOption struct { - dryRun bool - mergeInsert int - debug bool -} -type SourceOption func(*sourceOption) - -func WithMergeInsert(size int) SourceOption { - return func(o *sourceOption) { - o.mergeInsert = size - } -} - -type dbWrapper struct { - DB *sql.DB - debug bool - dryRun bool -} - -func newDBWrapper(db *sql.DB, dryRun, debug bool) *dbWrapper { - - return &dbWrapper{ - DB: db, - dryRun: dryRun, - debug: debug, - } -} - -func (db *dbWrapper) Exec(query string, args ...interface{}) (sql.Result, error) { - if db.debug { - global.LOG.Debugf("query %s", query) - } - - if db.dryRun { - return nil, nil - } - return db.DB.Exec(query, args...) -} - -func Source(dns string, reader io.Reader, opts ...SourceOption) error { - start := time.Now() - global.LOG.Infof("source start at %s", start.Format(constant.DateTimeLayout)) - defer func() { - end := time.Now() - global.LOG.Infof("source end at %s, cost %s", end.Format(constant.DateTimeLayout), end.Sub(start)) - }() - - var err error - var db *sql.DB - var o sourceOption - for _, opt := range opts { - opt(&o) - } - - dbName, err := getDBNameFromDNS(dns) - if err != nil { - global.LOG.Errorf("get db name from dns failed, err: %v", err) - return err - } - - db, err = sql.Open("mysql", dns) - if err != nil { - global.LOG.Errorf("open mysql db failed, err: %v", err) - return err - } - defer db.Close() - - dbWrapper := newDBWrapper(db, o.dryRun, o.debug) - - _, err = dbWrapper.Exec(fmt.Sprintf("USE `%s`;", dbName)) - if err != nil { - global.LOG.Errorf("exec `use %s` failed, err: %v", dbName, err) - return err - } - - db.SetConnMaxLifetime(3600) - - r := bufio.NewReader(reader) - _, err = dbWrapper.Exec("SET autocommit=0;") - if err != nil { - global.LOG.Errorf("exec `set autocommit=0` failed, err: %v", err) - return err - } - - for { - line, err := readLine(r) - if err != nil { - if err == io.EOF { - break - } - global.LOG.Errorf("read sql failed, err: %v", err) - return err - } - - ssql, err := trim(line) - if err != nil { - global.LOG.Errorf("trim sql failed, err: %v", err) - return err - } - - afterInsertSql := "" - if o.mergeInsert > 1 && strings.HasPrefix(ssql, "INSERT INTO") { - var insertSQLs []string - insertSQLs = append(insertSQLs, ssql) - for i := 0; i < o.mergeInsert-1; i++ { - line, err := readLine(r) - if err != nil { - if err == io.EOF { - break - } - return err - } - ssql2, err := trim(line) - if err != nil { - global.LOG.Errorf("trim merge insert sql failed, err: %v", err) - return err - } - if strings.HasPrefix(ssql2, "INSERT INTO") { - insertSQLs = append(insertSQLs, ssql2) - continue - } - afterInsertSql = ssql2 - break - } - ssql, err = mergeInsert(insertSQLs) - if err != nil { - global.LOG.Errorf("do merge insert failed, err: %v", err) - return err - } - } - - _, err = dbWrapper.Exec(ssql) - if err != nil { - global.LOG.Errorf("exec sql failed, err: %v", err) - return err - } - if len(afterInsertSql) != 0 { - _, err = dbWrapper.Exec(afterInsertSql) - if err != nil { - global.LOG.Errorf("exec sql failed, err: %v", err) - return err - } - } - } - - _, err = dbWrapper.Exec("COMMIT;") - if err != nil { - global.LOG.Errorf("exec `commit` failed, err: %v", err) - return err - } - - _, err = dbWrapper.Exec("SET autocommit=1;") - if err != nil { - global.LOG.Errorf("exec `autocommit=1` failed, err: %v", err) - return err - } - - return nil -} - -func mergeInsert(insertSQLs []string) (string, error) { - if len(insertSQLs) == 0 { - return "", errors.New("no input provided") - } - builder := strings.Builder{} - sql1 := insertSQLs[0] - sql1 = strings.TrimSuffix(sql1, ";") - builder.WriteString(sql1) - for i, insertSQL := range insertSQLs[1:] { - if i < len(insertSQLs)-1 { - builder.WriteString(",") - } - - valuesIdx := strings.Index(insertSQL, "VALUES") - if valuesIdx == -1 { - return "", errors.New("invalid SQL: missing VALUES keyword") - } - sqln := insertSQL[valuesIdx:] - sqln = strings.TrimPrefix(sqln, "VALUES") - sqln = strings.TrimSuffix(sqln, ";") - builder.WriteString(sqln) - - } - builder.WriteString(";") - - return builder.String(), nil -} - -func trim(s string) (string, error) { - s = strings.TrimLeft(s, "\n") - s = strings.TrimSpace(s) - return s, nil -} - -func getDBNameFromDNS(dns string) (string, error) { - ss1 := strings.Split(dns, "/") - if len(ss1) == 2 { - ss2 := strings.Split(ss1[1], "?") - if len(ss2) == 2 { - return ss2[0], nil - } - } - - return "", fmt.Errorf("dns error: %s", dns) -} - -func readLine(r *bufio.Reader) (string, error) { - lineItem, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - return lineItem, err - } - global.LOG.Errorf("read merge insert sql failed, err: %v", err) - return "", err - } - if strings.HasSuffix(lineItem, ";\n") { - return lineItem, nil - } - lineAppend, err := readLine(r) - if err != nil { - if err == io.EOF { - return lineItem, err - } - global.LOG.Errorf("read merge insert sql failed, err: %v", err) - return "", err - } - - return lineItem + lineAppend, nil -}