+feat: import-words

src/test/import/comm/excel.go

@@ -2,9 +2,10 @@ package comm
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/360EntSecGroup-Skylar/excelize/v2"
 	fileUtils "github.com/easysoft/zendata/src/utils/file"
-	"strings"
 )
 
 func GetExcelTable(filePath, sheetName string) (records []map[string]interface{}) {
@@ -58,3 +59,45 @@ func GetExcelTable(filePath, sheetName string) (records []map[string]interface{}
 
 	return
 }
+
+func GetExceTablelHeader(filePath string) []string {
+
+	excel, err := excelize.OpenFile(filePath)
+	if err != nil {
+		fmt.Printf("fail to read file %s, error: %s", filePath, err.Error())
+		return nil
+	}
+
+	sheet := excel.GetSheetList()[0]
+	rows, err := excel.GetRows(sheet)
+
+	if err != nil {
+		fmt.Println(err)
+		return nil
+	}
+
+	if len(rows) == 0 {
+		return nil
+	} else {
+		return rows[0]
+	}
+
+}
+
+func GetExcel1stSheet(filePath string) (sheetName string, rows [][]string) {
+
+	excel, err := excelize.OpenFile(filePath)
+	if err != nil {
+		fmt.Printf("fail to read file %s, error: %s", filePath, err.Error())
+		return
+	}
+
+	sheetName = excel.GetSheetList()[0]
+	rows, err = excel.GetRows(sheetName)
+
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	return
+}

src/test/import/import-words.go

+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/easysoft/zendata/src/test/import/comm"
+	"github.com/easysoft/zendata/src/test/import/model"
+	"gorm.io/gorm"
+)
+
+func main() {
+	filePathArr := []string{
+		"./data/words/v1/互联网黑话.xlsx",
+		"./data/words/v1/介词词库.xlsx",
+		"./data/words/v1/代词词库.xlsx",
+		"./data/words/v1/副词词库.xlsx",
+		"./data/words/v1/动词词库.xlsx",
+		"./data/words/v1/助词词库.xlsx",
+		"./data/words/v1/名词词库.xlsx",
+		"./data/words/v1/形容词做谓语.xlsx",
+		"./data/words/v1/形容词词库.xlsx",
+		"./data/words/v1/数词词库.xlsx",
+		"./data/words/v1/连词词库.xlsx",
+		"./data/words/v1/量词词库.xlsx",
+	}
+
+	db := comm.GetDB()
+	if err := db.AutoMigrate(
+		&model.DataWordTagGroup{}, &model.DataWordTag{}, &model.DataWord{},
+	); err != nil {
+		panic(err)
+	}
+
+	logs := []string{}
+	for i, path := range filePathArr {
+		sheetName, count := ImoprtWordsFromExcel(path, db)
+		str := fmt.Sprintf("%d) Path:[%s] SheetName:[%s], count:[%d]\n", i, path, sheetName, count)
+		fmt.Print(str)
+		logs = append(logs, str)
+	}
+
+	for _, l := range logs {
+		fmt.Print(l)
+	}
+}
+
+func ImoprtWordsFromExcel(filePath string, db *gorm.DB) (sheetName string, count int) {
+	sheetName, rows := comm.GetExcel1stSheet(filePath)
+	fmt.Printf("importing ... : [%s] \n", sheetName)
+	// fmt.Print(rows)
+
+	if len(rows) == 0 {
+		return
+	}
+
+	headers := rows[0]
+	if len(headers) < 1 {
+		return
+	}
+
+	headers = headers[1:]
+
+	// create word-tag-group
+	group := model.DataWordTagGroup{Name: strings.TrimSpace(sheetName)}
+	if err := db.Save(&group).Error; err != nil {
+		fmt.Errorf("creating word-tag-group : \n%v\n", err)
+		return
+	}
+
+	// create word-tags
+	tags := make([]model.DataWordTag, 0, len(headers))
+	for _, v := range headers {
+		tag := model.DataWordTag{Name: strings.TrimSpace(v)}
+
+		ret := db.First(&tag, "name = ?", strings.TrimSpace(v))
+		if ret.RowsAffected == 0 {
+			tag.Groups = []*model.DataWordTagGroup{&group}
+			if err := db.Save(&tag).Error; err != nil {
+				fmt.Errorf("sheetName:[%s], Tag[%s]", sheetName, v)
+				return
+			}
+		} else {
+			if err := db.Model(&tag).Association("Groups").Append(&group); err != nil {
+				fmt.Errorf("sheetName:[%s], Tag[%s]", sheetName, v)
+				return
+			}
+		}
+
+		tags = append(tags, tag)
+	}
+
+	// create word
+
+	words := make([]model.DataWord, 0, len(rows))
+
+	for i, r := range rows[1:] {
+		if len(r) == 0 {
+			fmt.Printf("shettName:[%s}, row index:[%d]", sheetName, i+1)
+			continue
+		}
+
+		word := model.DataWord{Word: strings.TrimSpace(r[0])}
+
+		flag := false
+		for j, v := range r[1:] {
+			if strings.TrimSpace(v) != "" {
+				word.Tags = []*model.DataWordTag{&tags[j]}
+				flag = true
+				break
+			}
+		}
+
+		if !flag {
+			fmt.Errorf("word (%v) is not tag.", word)
+		}
+
+		words = append(words, word)
+	}
+
+	tx := db.CreateInBatches(&words, 1000)
+	if tx.Error != nil {
+		fmt.Errorf("%v", tx.Error)
+	}
+
+	var count64 int64
+	tx.Count(&count64)
+	count = int(count64)
+
+	return
+}

src/test/import/model/model.go

@@ -162,3 +162,35 @@ type DataWordsConjunction struct {
 type DataWordsMeasure struct {
 	BaseModel
 }
+
+type DataWordTagGroup struct {
+	BaseModel
+	Name string         `gorm:"uniqueIndex" json:"name"`
+	Tags []*DataWordTag `gorm:"many2many:biz_data_word_tag_group_biz_data_word_tag" json:"tags"`
+}
+
+func (DataWordTagGroup) TableName() string {
+	return "biz_data_word_tag_group"
+}
+
+type DataWordTag struct {
+	BaseModel
+	Name  string      `gorm:"uniqueIndex" json:"name"`
+	Words []*DataWord `gorm:"many2many:biz_data_word_biz_data_word_tag" json:"words"`
+
+	Groups []*DataWordTagGroup `gorm:"many2many:biz_data_word_tag_group_biz_data_word_tag" json:"tags"`
+}
+
+func (DataWordTag) TableName() string {
+	return "biz_data_word_tag"
+}
+
+type DataWord struct {
+	BaseModel
+	Word string         `json:"word"`
+	Tags []*DataWordTag `gorm:"many2many:biz_data_word_biz_data_word_tag" json:"tags"`
+}
+
+func (DataWord) TableName() string {
+	return "biz_data_word"
+}