import data from text files

src/test/import/comm/consts.go

+package comm
+
+const (
+	TruncateTable = `truncate table %s IF EXISTS;`
+
+	CreateTableTempl = `CREATE TABLE IF NOT EXISTS %s (
+		id bigint auto_increment,
+		content varchar(1000) not null unique,
+		tag varchar(50),
+		primary key(id)
+	) engine=innodb default charset=utf8 auto_increment=1;`
+
+	InsertTemplate = "INSERT INTO %s (content) VALUES %s;"
+)

src/test/import/comm/dao.go

+package comm
+
+import (
+	"fmt"
+	"github.com/easysoft/zendata/src/test/import/model"
+	"github.com/fatih/color"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/plugin/dbresolver"
+	"time"
+)
+
+const (
+	dbUser   = "root"
+	dbUrl    = "127.0.0.1:3306"
+	dbPasswd = "P2ssw0rd"
+	dbName   = "zendata"
+)
+
+var (
+	DB *gorm.DB
+)
+
+func GetDB() *gorm.DB {
+	if DB != nil {
+		return DB
+	}
+
+	DB = GormMySQL()
+	_ = DB.Use(
+		dbresolver.Register(
+			dbresolver.Config{}).
+			SetConnMaxIdleTime(time.Hour).
+			SetConnMaxLifetime(24 * time.Hour).
+			SetMaxIdleConns(100).
+			SetMaxOpenConns(200),
+	)
+
+	DB.Session(&gorm.Session{FullSaveAssociations: true, AllowGlobalUpdate: false})
+
+	err := DB.AutoMigrate(
+		&model.DataCategory1{},
+		&model.DataCategory2{},
+
+		&model.DataCity{},
+	)
+	if err != nil {
+		fmt.Printf(color.RedString("migrate models failed, error: %s.", err.Error()))
+	}
+
+	return DB
+}
+
+func GormMySQL() *gorm.DB {
+	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?%s", dbUser, dbPasswd, dbUrl, dbName, "charset=utf8mb4&parseTime=True&loc=Local")
+
+	mysqlConfig := mysql.Config{
+		DSN:                       dsn,   // DSN data source name
+		DefaultStringSize:         191,   // string 类型字段的默认长度
+		DisableDatetimePrecision:  true,  // 禁用 datetime 精度，MySQL 5.6 之前的数据库不支持
+		DontSupportRenameIndex:    true,  // 重命名索引时采用删除并新建的方式，MySQL 5.7 之前的数据库和 MariaDB 不支持重命名索引
+		DontSupportRenameColumn:   true,  // 用 `change` 重命名列，MySQL 8 之前的数据库和 MariaDB 不支持重命名列
+		SkipInitializeWithVersion: false, // 根据版本自动配置
+	}
+	if db, err := gorm.Open(mysql.New(mysqlConfig)); err != nil {
+		return nil
+	} else {
+		sqlDB, _ := db.DB()
+		sqlDB.SetMaxIdleConns(0)
+		sqlDB.SetMaxOpenConns(0)
+		return db
+	}
+}

src/test/import/comm/excel.go

+package comm
+
+import (
+	"fmt"
+	"github.com/360EntSecGroup-Skylar/excelize/v2"
+	fileUtils "github.com/easysoft/zendata/src/utils/file"
+	"strings"
+)
+
+func GetExcelTable() (records []map[string]interface{}) {
+	filePath := "data/address/cn.v1.xlsx"
+	sheetName := "china"
+
+	excel, err := excelize.OpenFile(filePath)
+	if err != nil {
+		fmt.Printf("fail to read file %s, error: %s", filePath, err.Error())
+		return
+	}
+
+	fileName := fileUtils.GetFileName(filePath)
+	fileName = strings.TrimSuffix(fileName, "词库")
+
+	for _, sheet := range excel.GetSheetList() {
+		if sheetName != sheet {
+			continue
+		}
+
+		rows, _ := excel.GetRows(sheet)
+		if len(rows) == 0 {
+			continue
+		}
+
+		colMap := map[int]string{}
+		colCount := 0
+		for colIndex, colVal := range rows[0] {
+			colMap[colIndex] = strings.TrimSpace(colVal)
+			colCount++
+		}
+
+		for rowIndex, row := range rows {
+			if rowIndex == 0 { // ignore header
+				continue
+			}
+
+			record := map[string]interface{}{}
+
+			for colIndex, col := range row {
+				if colIndex >= colCount {
+					break
+				}
+
+				colName := colMap[colIndex]
+				colVal := strings.TrimSpace(col)
+				record[colName] = colVal
+			}
+
+			records = append(records, record)
+		}
+	}
+
+	return
+}

src/test/import/import-address.go

+package main
+
+import (
+	"fmt"
+	"github.com/easysoft/zendata/src/test/import/comm"
+	"github.com/easysoft/zendata/src/test/import/model"
+)
+
+func main() {
+	tableName := model.DataCity{}.TableName()
+	db := comm.GetDB()
+
+	truncateTableSql := fmt.Sprintf(comm.TruncateTable, tableName)
+	db.Exec(truncateTableSql)
+
+	createTableSql := fmt.Sprintf(comm.CreateTableTempl, tableName)
+	err := db.Exec(createTableSql).Error
+	if err != nil {
+		fmt.Printf("create table %s failed, err %s", tableName, err.Error())
+		return
+	}
+
+	records := comm.GetExcelTable()
+	fmt.Sprintf("%d", len(records))
+
+}

src/test/import/import-text.go

@@ -3,38 +3,12 @@ package main
 import (
 	"flag"
 	"fmt"
+	"github.com/easysoft/zendata/src/test/import/comm"
 	fileUtils "github.com/easysoft/zendata/src/utils/file"
-	"github.com/fatih/color"
-	"gorm.io/driver/mysql"
-	"gorm.io/gorm"
-	"gorm.io/plugin/dbresolver"
 	"strings"
-	"time"
-)
-
-const (
-	dbUser   = "root"
-	dbUrl    = "127.0.0.1:3306"
-	dbPasswd = "P2ssw0rd"
-	dbName   = "zendata"
-
-	truncateTable = `truncate table %s;`
-
-	createTableTempl = `CREATE TABLE IF NOT EXISTS %s (
-		id bigint auto_increment,
-		content varchar(1000) not null unique,
-		tag varchar(50),
-		primary key(id)
-	) engine=innodb default charset=utf8 auto_increment=1;`
-
-	//deleteAllSql = "DELETE FROM %s WHERE 1=1;"
-
-	insertTemplate = "INSERT INTO %s (content) VALUES %s;"
 )
 
 var (
-	db *gorm.DB
-
 	tableName string
 	filePath  string
 	colNum    int
@@ -48,31 +22,20 @@ func main() {
 	flag.Parse()
 
 	tableName = "biz_" + strings.TrimLeft(tableName, "biz_")
-	db := GetDB()
+	db := comm.GetDB()
 
-	truncateTableSql := fmt.Sprintf(truncateTable, tableName)
-	err := db.Exec(truncateTableSql).Error
-	if err != nil {
-		fmt.Printf("create table %s failed, err %s", tableName, err.Error())
-		return
-	}
+	truncateTableSql := fmt.Sprintf(comm.TruncateTable, tableName)
+	db.Exec(truncateTableSql)
 
-	createTableSql := fmt.Sprintf(createTableTempl, tableName)
-	err = db.Exec(createTableSql).Error
+	createTableSql := fmt.Sprintf(comm.CreateTableTempl, tableName)
+	err := db.Exec(createTableSql).Error
 	if err != nil {
 		fmt.Printf("create table %s failed, err %s", tableName, err.Error())
 		return
 	}
 
-	//deleteDataSql := fmt.Sprintf(deleteAllSql, tableName)
-	//err = db.Exec(deleteDataSql).Error
-	//if err != nil {
-	//	fmt.Printf("insert data failed, err %s", err.Error())
-	//	return
-	//}
-
 	content := fileUtils.ReadFile(filePath)
-	insertSqls := make([]string, 0)
+	insertSqlArr := make([]string, 0)
 
 	for _, line := range strings.Split(content, "\n") {
 		arr := strings.Split(strings.TrimSpace(line), " ")
@@ -87,110 +50,28 @@ func main() {
 		}
 
 		insert := fmt.Sprintf("('%s')", content)
-		insertSqls = append(insertSqls, insert)
+		insertSqlArr = append(insertSqlArr, insert)
 	}
 
 	for i := 0; i < 1000; i++ {
 		start := i * 10000
 		end := (i + 1) * 10000
 
-		if end > len(insertSqls) {
-			end = len(insertSqls)
+		if end > len(insertSqlArr) {
+			end = len(insertSqlArr)
 		}
 
-		arr := insertSqls[start:end]
+		arr := insertSqlArr[start:end]
 
-		sql := fmt.Sprintf(insertTemplate, tableName, strings.Join(arr, ","))
+		sql := fmt.Sprintf(comm.InsertTemplate, tableName, strings.Join(arr, ","))
 		err = db.Exec(sql).Error
 		if err != nil {
 			fmt.Printf("insert data failed, err %s", err.Error())
 			return
 		}
 
-		if end >= len(insertSqls) {
+		if end >= len(insertSqlArr) {
 			break
 		}
 	}
 }
-
-func GetDB() *gorm.DB {
-	if db != nil {
-		return db
-	}
-
-	db = GormMySQL()
-	_ = db.Use(
-		dbresolver.Register(
-			dbresolver.Config{}).
-			SetConnMaxIdleTime(time.Hour).
-			SetConnMaxLifetime(24 * time.Hour).
-			SetMaxIdleConns(100).
-			SetMaxOpenConns(200),
-	)
-
-	db.Session(&gorm.Session{FullSaveAssociations: true, AllowGlobalUpdate: false})
-
-	err := db.AutoMigrate(
-		&DataCategory1{},
-		&DataCategory2{},
-	)
-	if err != nil {
-		fmt.Printf(color.RedString("migrate models failed, error: %s.", err.Error()))
-	}
-
-	return db
-}
-
-func GormMySQL() *gorm.DB {
-	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?%s", dbUser, dbPasswd, dbUrl, dbName, "charset=utf8mb4&parseTime=True&loc=Local")
-
-	mysqlConfig := mysql.Config{
-		DSN:                       dsn,   // DSN data source name
-		DefaultStringSize:         191,   // string 类型字段的默认长度
-		DisableDatetimePrecision:  true,  // 禁用 datetime 精度，MySQL 5.6 之前的数据库不支持
-		DontSupportRenameIndex:    true,  // 重命名索引时采用删除并新建的方式，MySQL 5.7 之前的数据库和 MariaDB 不支持重命名索引
-		DontSupportRenameColumn:   true,  // 用 `change` 重命名列，MySQL 8 之前的数据库和 MariaDB 不支持重命名列
-		SkipInitializeWithVersion: false, // 根据版本自动配置
-	}
-	if db, err := gorm.Open(mysql.New(mysqlConfig)); err != nil {
-		return nil
-	} else {
-		sqlDB, _ := db.DB()
-		sqlDB.SetMaxIdleConns(0)
-		sqlDB.SetMaxOpenConns(0)
-		return db
-	}
-}
-
-type DataCategory1 struct {
-	BaseModel
-
-	Name string `json:"name"`
-	Desc string `json:"desc" gorm:"column:descr"`
-}
-
-func (DataCategory1) TableName() string {
-	return "biz_data_category1"
-}
-
-type DataCategory2 struct {
-	BaseModel
-
-	Name      string `json:"name"`
-	Desc      string `json:"desc" gorm:"column:descr"`
-	DataTable string `json:"desc"`
-
-	ParentId uint `json:"parentId"`
-}
-
-func (DataCategory2) TableName() string {
-	return "biz_data_category2"
-}
-
-type BaseModel struct {
-	ID        uint       `gorm:"primary_key" sql:"type:INT(10) UNSIGNED NOT NULL" json:"id"`
-	CreatedAt *time.Time `json:"createdAt,omitempty"`
-	UpdatedAt *time.Time `json:"updatedAt,omitempty"`
-	Deleted   bool       `json:"-" gorm:"default:false"`
-	Disabled  bool       `json:"disabled,omitempty" gorm:"default:false"`
-}

src/test/import/model/model.go

+package model
+
+import "time"
+
+type DataCategory1 struct {
+	BaseModel
+
+	Name string `json:"name"`
+	Desc string `json:"desc" gorm:"column:descr"`
+}
+
+func (DataCategory1) TableName() string {
+	return "biz_data_category1"
+}
+
+type DataCategory2 struct {
+	BaseModel
+
+	Name      string `json:"name"`
+	Desc      string `json:"desc" gorm:"column:descr"`
+	DataTable string `json:"desc"`
+
+	ParentId uint `json:"parentId"`
+}
+
+func (DataCategory2) TableName() string {
+	return "biz_data_category2"
+}
+
+type BaseModel struct {
+	ID        uint       `gorm:"primary_key" sql:"type:INT(10) UNSIGNED NOT NULL" json:"id"`
+	CreatedAt *time.Time `json:"createdAt,omitempty"`
+	UpdatedAt *time.Time `json:"updatedAt,omitempty"`
+	Deleted   bool       `json:"-" gorm:"default:false"`
+	Disabled  bool       `json:"disabled,omitempty" gorm:"default:false"`
+}
+
+type DataCountry struct {
+	BaseModel
+
+	Continent    string `json:"continent"`
+	AreaCode     string `json:"areaCode"`
+	EnglishShort string `json:"englishShort"`
+	EnglishFull  string `json:"englishFull"`
+	ChineseShort string `json:"chineseShort"`
+	ChineseFull  string `json:"chineseFull"`
+}
+
+func (DataCountry) TableName() string {
+	return "biz_data_country"
+}
+
+type DataCity struct {
+	BaseModel
+
+	Name    string `json:"name"`
+	Code    string `json:"code"`
+	ZipCode string `json:"zipCode"`
+	State   string `json:"state"`
+}
+
+func (DataCity) TableName() string {
+	return "biz_data_city"
+}
+
+type DataColor struct {
+	BaseModel
+
+	English string `json:"english"`
+	Chinese string `json:"chinese"`
+	Hex     string `json:"hex"`
+	Rgb     string `json:"rgb"`
+}
+
+func (DataColor) TableName() string {
+	return "biz_data_color"
+}
+
+type DataChineseFamily struct {
+	BaseModel
+
+	Name   string `json:"name"`
+	Pinyin string `json:"pinyin"`
+	Double string `json:"double"`
+}
+
+func (DataChineseFamily) TableName() string {
+	return "biz_data_chinese_family"
+}
+
+type DataChineseGiven struct {
+	BaseModel
+
+	Name   string `json:"name"`
+	Pinyin string `json:"pinyin"`
+	Sex    string `json:"sex"`
+}
+
+func (DataChineseGiven) TableName() string {
+	return "biz_data_chinese_given"
+}
+
+type DataEnglishFamily struct {
+	BaseModel
+
+	Name  string `json:"name"`
+	Index string `json:"index"`
+}
+
+func (DataEnglishFamily) TableName() string {
+	return "biz_data_english_family"
+}
+
+type DataEnglishGiven struct {
+	BaseModel
+
+	Name  string `json:"name"`
+	Index string `json:"index"`
+	Sex   string `json:"sex"`
+}
+
+func (DataEnglishGiven) TableName() string {
+	return "biz_data_english_given"
+}
+
+type DataWordsInternetArgot struct {
+	BaseModel
+}
+type DataWordsPreposition struct {
+	BaseModel
+}
+type DataWordsPronoun struct {
+	BaseModel
+}
+type DataWordsAdverb struct {
+	BaseModel
+}
+type DataWordsVerb struct {
+	BaseModel
+}
+type DataWordsAuxiliary struct {
+	BaseModel
+}
+type DataWordsNoun struct {
+	BaseModel
+}
+type DataWordsAdjectivePredicate struct {
+	BaseModel
+}
+type DataWordsAdjective struct {
+	BaseModel
+}
+type DataWordsNumeral struct {
+	BaseModel
+}
+type DataWordsConjunction struct {
+	BaseModel
+}
+type DataWordsMeasure struct {
+	BaseModel
+}