diff --git a/go.mod b/go.mod index 6544044afcdf0931648c791da9333cb0fe32110d..b2e572883abc14e900b9750f2a06852d4bf3f87e 100644 --- a/go.mod +++ b/go.mod @@ -24,5 +24,6 @@ require ( golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc // indirect golang.org/x/sys v0.0.0-20200819171115-d785dc25833f // indirect gopkg.in/ini.v1 v1.60.0 + gopkg.in/yaml.v2 v2.2.8 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ) diff --git a/src/model/definition.go b/src/model/definition.go index 76ddc87d9a3dd6367147868ace7be6b5616a26de..fa92e07324fdc28237597f8aca96310f2026b905 100644 --- a/src/model/definition.go +++ b/src/model/definition.go @@ -107,4 +107,20 @@ func (def *DefSimple) Init(tableName, author, desc, version string) { } func (fld *FieldSimple) Init(field string) { fld.Field = field +} + + +type DefExport struct { + ClsBase `yaml:",inline"` + XFields []DefFieldExport `yaml:"xfields,flow"` // control orders +} +type DefFieldExport struct { + Field string `yaml:"field"` + Prefix string `yaml:"prefix"` + Postfix string `yaml:"postfix"` + + Select string `yaml:"select"` + Where string `yaml:"where"` + Rand bool `yaml:"rand"` + Limit int `yaml:"limit"` } \ No newline at end of file diff --git a/test/article/common.go b/test/article/common.go index cab80edb9249322ba68b3b490e9ab601a131204c..fd07a1686eac143d1e29308d52c939e5237100bc 100644 --- a/test/article/common.go +++ b/test/article/common.go @@ -36,7 +36,7 @@ func getFilesInDir(folder, ext string, files *[]string) { filePath := fileUtils.AddSepIfNeeded(folder) + name if fi.IsDir() { getFilesInDir(filePath, ext, files) - } else if strings.Index(name, "~") != 0 && path.Ext(filePath) == ".xlsx" { + } else if strings.Index(name, "~") != 0 && path.Ext(filePath) == ext { *files = append(*files, filePath) } } diff --git a/test/article/convert_test.go b/test/article/convert_test.go index 6209901bac6e2130e3ddf76152f2fa9cfa44cca7..7f82f9dd6bab929601c655a46595bbc0c756b04a 100644 --- a/test/article/convert_test.go +++ b/test/article/convert_test.go @@ -6,8 +6,8 @@ import ( "github.com/360EntSecGroup-Skylar/excelize/v2" constant "github.com/easysoft/zendata/src/utils/const" i118Utils "github.com/easysoft/zendata/src/utils/i118" - logUtils "github.com/easysoft/zendata/src/utils/log" _ "github.com/mattn/go-sqlite3" + "log" "strconv" "strings" "testing" @@ -15,7 +15,7 @@ import ( func TestImportSqlite(t *testing.T) { files := make([]string, 0) - getFilesInDir("xdoc/words-9.3", "xlsx", &files) + getFilesInDir("xdoc/words-9.3", ".xlsx", &files) tableName := "words" seq := 1 @@ -33,7 +33,7 @@ func TestImportSqlite(t *testing.T) { dropSql := `DROP TABLE IF EXISTS ` + tableName + `;` _, err = db.Exec(dropSql) if err != nil { - logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_drop_table", tableName, err.Error())) + log.Println(i118Utils.I118Prt.Sprintf("fail_to_drop_table", tableName, err.Error())) return } @@ -44,14 +44,14 @@ func TestImportSqlite(t *testing.T) { ddlSql := fmt.Sprintf(ddlTemplate, strings.Join(ddlFields, ", \n")) _, err = db.Exec(ddlSql) if err != nil { - logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_create_table", tableName, err.Error())) + log.Println(i118Utils.I118Prt.Sprintf("fail_to_create_table", tableName, err.Error())) return } sql := strings.Join(insertSqls, "\n") _, err = db.Exec(sql) if err != nil { - logUtils.PrintTo(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sql, err.Error())) + log.Println(i118Utils.I118Prt.Sprintf("fail_to_exec_query", sql, err.Error())) return } } @@ -59,7 +59,7 @@ func TestImportSqlite(t *testing.T) { func importExcel(filePath, tableName string, seq *int, ddlFields, insertSqls *[]string, colMap *map[string]bool) { excel, err := excelize.OpenFile(filePath) if err != nil { - logUtils.PrintTo("fail to read file " + filePath + ", error: " + err.Error()) + log.Println("fail to read file " + filePath + ", error: " + err.Error()) return } @@ -90,7 +90,7 @@ func importExcel(filePath, tableName string, seq *int, ddlFields, insertSqls *[] colName = "ci" } if colName != "ci" { - colName = colPrefix + ":" + colName + colName = colPrefix + "-" + colName } if (*colMap)[colName] == false { diff --git a/test/article/generate_test.go b/test/article/generate_test.go index dc125a9d63f4b5483890bc02a4dbf8cbf7cdd776..bebcca90c92c870f7d5df47a4ed4934d506677a8 100644 --- a/test/article/generate_test.go +++ b/test/article/generate_test.go @@ -1,12 +1,25 @@ package main import ( + "github.com/easysoft/zendata/src/model" + constant "github.com/easysoft/zendata/src/utils/const" fileUtils "github.com/easysoft/zendata/src/utils/file" _ "github.com/mattn/go-sqlite3" + "gopkg.in/yaml.v3" + "strconv" + "strings" "testing" ) -func TestGenerate(t *testing.T) { +const ( + strLeft = "“" + strRight = "”" + + expLeft = "（" + expRight = "）" +) + +func TestGenerate(ts *testing.T) { files := make([]string, 0) getFilesInDir("xdoc/words-9.3", ".txt", &files) @@ -20,7 +33,129 @@ func TestGenerate(t *testing.T) { } func convertToYaml(article string) (content string) { + sections := parseSections(article) + + conf := createDef(constant.ConfigTypeArticle, "words.v1") + + prefix := "" + for index, section := range sections { + tye := section["type"] + val := section["val"] + if tye == "exp" { + field := createField(index, prefix, val) + conf.XFields = append(conf.XFields, field) + prefix = "" + } else { + prefix += val + } + } + + bytes, _ := yaml.Marshal(&conf) + content = string(bytes) + + // convert yaml format by using a map + m := make(map[string]interface{}) + yaml.Unmarshal([]byte(content), &m) + bytes, _ = yaml.Marshal(&m) + content = string(bytes) + content = strings.Replace(content, "xfields", "\nfields", -1) + + return +} + +func createDef(typ, table string) (conf model.DefExport) { + conf.Title = "automation" + conf.Author = "zendata" + conf.From = table + conf.Type = typ + conf.Desc = "generated from article text automatically" + + return +} + +func createField(index int, prefix, exp string) (field model.DefFieldExport) { + field.Field = strconv.Itoa(index) + field.Prefix = prefix + + field.Select = getPinyin(exp) + field.Where = "true" + field.Rand = true + field.Limit = 1 + + return +} + +func parseSections(content string) (sections []map[string]string) { + strStart := false + expStart := false + + content = strings.TrimSpace(content) + runeArr := []rune(content) + + section := "" + for i := 0; i < len(runeArr); i++ { + item := runeArr[i] + str := string(item) + + isCouple, duplicateStr := isCouple(i, runeArr) + if isCouple { + section += duplicateStr + i += 1 + } else if strStart && str == strRight { // str close + addSection(section, "str", §ions) + + strStart = false + section = "" + } else if expStart && str == expRight { // exp close + addSection(section, "exp", §ions) + + expStart = false + section = "" + } else if !strStart && !expStart && str == strLeft { // str start + if section != "" && strings.TrimSpace(section) != "+" { + addSection(section, "str", §ions) + } + + strStart = true + section = "" + } else if !strStart && !expStart && str == expLeft { // exp start + if section != "" && strings.TrimSpace(section) != "+" { + addSection(section, "str", §ions) + } + + expStart = true + section = "" + } else { + section += str + } + } + + return +} + +func addSection(str, typ string, arr *[]map[string]string) { + mp := map[string]string{} + mp["type"] = typ + mp["val"] = str + + *arr = append(*arr, mp) +} + +func isCouple(i int, arr []rune) (isCouple bool, duplicateStr string) { + if string(arr[i]) == strLeft && (i + 1 < len(arr) && string(arr[i + 1]) == strLeft) { + isCouple = true + duplicateStr = string(arr[i]) + } else if string(arr[i]) == strRight && (i + 1 < len(arr) && string(arr[i + 1]) == strRight) { + isCouple = true + duplicateStr = string(arr[i]) + } else if string(arr[i]) == expLeft && (i + 1 < len(arr) && string(arr[i + 1]) == expLeft) { + isCouple = true + duplicateStr = string(arr[i]) + } else if string(arr[i]) == expRight && (i + 1 < len(arr) && string(arr[i + 1]) == expRight) { + isCouple = true + duplicateStr = string(arr[i]) + } return } \ No newline at end of file diff --git a/tmp/cache/.words.db b/tmp/cache/.words.db index 8b36fb239e78b6ee345a5908e90e2c2f1cf15b23..c967077fdca15a94e9b1ca9f28dc253e1dfde0e0 100644 Binary files a/tmp/cache/.words.db and b/tmp/cache/.words.db differ